1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.mediav2.cts; 18 19 import static android.media.codec.Flags.apvSupport; 20 import static android.mediav2.common.cts.CodecTestBase.IS_AT_LEAST_V; 21 import static android.mediav2.common.cts.CodecTestBase.SupportClass.CODEC_ALL; 22 import static android.mediav2.common.cts.CodecTestBase.SupportClass.CODEC_OPTIONAL; 23 import static android.mediav2.common.cts.DecodeStreamToYuv.getFormatInStream; 24 import static android.mediav2.common.cts.CodecTestBase.VNDK_IS_AT_MOST_U; 25 26 import static com.android.media.extractor.flags.Flags.extractorMp4EnableApv; 27 28 import android.media.MediaCodec; 29 import android.media.MediaCodecInfo; 30 import android.media.MediaExtractor; 31 import android.media.MediaFormat; 32 import android.mediav2.common.cts.CodecDecoderTestBase; 33 import android.mediav2.common.cts.CodecTestActivity; 34 import android.mediav2.common.cts.OutputManager; 35 import android.util.Pair; 36 37 import androidx.test.ext.junit.rules.ActivityScenarioRule; 38 import androidx.test.filters.LargeTest; 39 40 import com.android.compatibility.common.util.ApiTest; 41 import com.android.compatibility.common.util.CddTest; 42 import com.android.compatibility.common.util.Preconditions; 43 44 import org.junit.Assert; 45 import org.junit.Assume; 46 import org.junit.Before; 47 import org.junit.Rule; 48 import org.junit.Test; 49 import org.junit.runner.RunWith; 50 import org.junit.runners.Parameterized; 51 52 import java.io.File; 53 import java.io.IOException; 54 import java.nio.ByteBuffer; 55 import java.util.ArrayList; 56 import java.util.Arrays; 57 import java.util.Collection; 58 import java.util.HashSet; 59 import java.util.List; 60 61 /** 62 * Test video decoders support for Adaptive Playback. 63 * <p> 64 * Adaptive playback support for video decoders is only activated if the codec is configured to 65 * decode onto a Surface. The getOutputImage() will return null if the codec was configured with 66 * an output surface. Hence any form of checksum validation for the decoded output is ruled out. 67 * The only form of validation this test currently does is, it checks if the output count is same 68 * as input count and output timestamps list and input timestamps list are same. 69 */ 70 @RunWith(Parameterized.class) 71 public class AdaptivePlaybackTest extends CodecDecoderTestBase { 72 private final String[] mSrcFiles; 73 private final SupportClass mSupportRequirements; 74 private static final String MEDIA_DIR = WorkDir.getMediaDirString(); 75 private static final HashSet<String> MUST_SUPPORT_APB = new HashSet<>(); 76 77 static { 78 MUST_SUPPORT_APB.add(MediaFormat.MIMETYPE_VIDEO_VP8); 79 MUST_SUPPORT_APB.add(MediaFormat.MIMETYPE_VIDEO_VP9); 80 MUST_SUPPORT_APB.add(MediaFormat.MIMETYPE_VIDEO_AVC); 81 MUST_SUPPORT_APB.add(MediaFormat.MIMETYPE_VIDEO_HEVC); 82 } 83 AdaptivePlaybackTest(String decoder, String mediaType, String[] srcFiles, SupportClass supportRequirements, String allTestParams)84 public AdaptivePlaybackTest(String decoder, String mediaType, String[] srcFiles, 85 SupportClass supportRequirements, String allTestParams) { 86 super(decoder, mediaType, null, allTestParams); 87 mSrcFiles = new String[srcFiles.length]; 88 for (int i = 0; i < srcFiles.length; i++) { 89 mSrcFiles[i] = MEDIA_DIR + srcFiles[i]; 90 } 91 mSupportRequirements = supportRequirements; 92 } 93 94 @Rule 95 public ActivityScenarioRule<CodecTestActivity> mActivityRule = 96 new ActivityScenarioRule<>(CodecTestActivity.class); 97 98 @Before setUp()99 public void setUp() throws IOException, InterruptedException { 100 mActivityRule.getScenario().onActivity(activity -> mActivity = activity); 101 setUpSurface(mActivity); 102 } 103 104 @Parameterized.Parameters(name = "{index}_{0}_{1}") input()105 public static Collection<Object[]> input() { 106 final boolean isEncoder = false; 107 final boolean needAudio = false; 108 final boolean needVideo = true; 109 // mediaType, array list of test files, SupportClass 110 final List<Object[]> exhaustiveArgsList = new ArrayList<>(Arrays.asList(new Object[][]{ 111 {MediaFormat.MIMETYPE_VIDEO_AVC, new String[]{ 112 "bbb_800x640_768kbps_30fps_avc_2b.mp4", 113 "bbb_800x640_768kbps_30fps_avc_nob.mp4", 114 "bbb_1280x720_1mbps_30fps_avc_2b.mp4", 115 "bbb_640x360_512kbps_30fps_avc_nob.mp4", 116 "bbb_1280x720_1mbps_30fps_avc_nob.mp4", 117 "bbb_640x360_512kbps_30fps_avc_2b.mp4", 118 "bbb_1280x720_1mbps_30fps_avc_nob.mp4", 119 "bbb_640x360_512kbps_30fps_avc_nob.mp4", 120 "bbb_640x360_512kbps_30fps_avc_2b.mp4"}, CODEC_ALL}, 121 {MediaFormat.MIMETYPE_VIDEO_HEVC, new String[]{ 122 "bbb_800x640_768kbps_30fps_hevc_2b.mp4", 123 "bbb_800x640_768kbps_30fps_hevc_nob.mp4", 124 "bbb_1280x720_1mbps_30fps_hevc_2b.mp4", 125 "bbb_640x360_512kbps_30fps_hevc_nob.mp4", 126 "bbb_1280x720_1mbps_30fps_hevc_nob.mp4", 127 "bbb_640x360_512kbps_30fps_hevc_2b.mp4", 128 "bbb_1280x720_1mbps_30fps_hevc_nob.mp4", 129 "bbb_640x360_512kbps_30fps_hevc_nob.mp4", 130 "bbb_640x360_512kbps_30fps_hevc_2b.mp4"}, CODEC_ALL}, 131 {MediaFormat.MIMETYPE_VIDEO_VP8, new String[]{ 132 "bbb_800x640_768kbps_30fps_vp8.webm", 133 "bbb_1280x720_1mbps_30fps_vp8.webm", 134 "bbb_640x360_512kbps_30fps_vp8.webm"}, CODEC_ALL}, 135 {MediaFormat.MIMETYPE_VIDEO_VP9, new String[]{ 136 "bbb_800x640_768kbps_30fps_vp9.webm", 137 "bbb_1280x720_1mbps_30fps_vp9.webm", 138 "bbb_640x360_512kbps_30fps_vp9.webm"}, CODEC_ALL}, 139 {MediaFormat.MIMETYPE_VIDEO_MPEG4, new String[]{ 140 "bbb_128x96_64kbps_12fps_mpeg4.mp4", 141 "bbb_176x144_192kbps_15fps_mpeg4.mp4", 142 "bbb_128x96_64kbps_12fps_mpeg4.mp4"}, CODEC_ALL}, 143 {MediaFormat.MIMETYPE_VIDEO_AV1, new String[]{ 144 "bbb_800x640_768kbps_30fps_av1.webm", 145 "bbb_1280x720_1mbps_30fps_av1.webm", 146 "bbb_640x360_512kbps_30fps_av1.webm"}, CODEC_OPTIONAL}, 147 {MediaFormat.MIMETYPE_VIDEO_MPEG2, new String[]{ 148 "bbb_800x640_768kbps_30fps_mpeg2_2b.mp4", 149 "bbb_800x640_768kbps_30fps_mpeg2_nob.mp4", 150 "bbb_1280x720_1mbps_30fps_mpeg2_2b.mp4", 151 "bbb_640x360_512kbps_30fps_mpeg2_nob.mp4", 152 "bbb_1280x720_1mbps_30fps_mpeg2_nob.mp4", 153 "bbb_640x360_512kbps_30fps_mpeg2_2b.mp4", 154 "bbb_1280x720_1mbps_30fps_mpeg2_nob.mp4", 155 "bbb_640x360_512kbps_30fps_mpeg2_nob.mp4", 156 "bbb_640x360_512kbps_30fps_mpeg2_2b.mp4"}, CODEC_ALL}, 157 })); 158 // P010 support was added in Android T, hence limit the following tests to Android T and 159 // above 160 if (IS_AT_LEAST_T) { 161 exhaustiveArgsList.addAll(Arrays.asList(new Object[][]{ 162 {MediaFormat.MIMETYPE_VIDEO_AVC, new String[]{ 163 "cosmat_800x640_24fps_crf22_avc_10bit_2b.mkv", 164 "cosmat_800x640_24fps_crf22_avc_10bit_nob.mkv", 165 "cosmat_1280x720_24fps_crf22_avc_10bit_2b.mkv", 166 "cosmat_640x360_24fps_crf22_avc_10bit_nob.mkv", 167 "cosmat_1280x720_24fps_crf22_avc_10bit_nob.mkv", 168 "cosmat_640x360_24fps_crf22_avc_10bit_2b.mkv", 169 "cosmat_1280x720_24fps_crf22_avc_10bit_nob.mkv", 170 "cosmat_640x360_24fps_crf22_avc_10bit_nob.mkv", 171 "cosmat_640x360_24fps_crf22_avc_10bit_2b.mkv"}, CODEC_OPTIONAL}, 172 {MediaFormat.MIMETYPE_VIDEO_HEVC, new String[]{ 173 "cosmat_800x640_24fps_crf22_hevc_10bit_2b.mkv", 174 "cosmat_800x640_24fps_crf22_hevc_10bit_nob.mkv", 175 "cosmat_1280x720_24fps_crf22_hevc_10bit_2b.mkv", 176 "cosmat_640x360_24fps_crf22_hevc_10bit_nob.mkv", 177 "cosmat_1280x720_24fps_crf22_hevc_10bit_nob.mkv", 178 "cosmat_640x360_24fps_crf22_hevc_10bit_2b.mkv", 179 "cosmat_1280x720_24fps_crf22_hevc_10bit_nob.mkv", 180 "cosmat_640x360_24fps_crf22_hevc_10bit_nob.mkv", 181 "cosmat_640x360_24fps_crf22_hevc_10bit_2b.mkv"}, CODEC_OPTIONAL}, 182 {MediaFormat.MIMETYPE_VIDEO_VP9, new String[]{ 183 "cosmat_640x360_24fps_crf22_vp9_10bit.mkv", 184 "cosmat_1280x720_24fps_crf22_vp9_10bit.mkv", 185 "cosmat_800x640_24fps_crf22_vp9_10bit.mkv"}, CODEC_OPTIONAL}, 186 {MediaFormat.MIMETYPE_VIDEO_AV1, new String[]{ 187 "cosmat_640x360_24fps_512kbps_av1_10bit.mkv", 188 "cosmat_1280x720_24fps_1200kbps_av1_10bit.mkv", 189 "cosmat_800x640_24fps_768kbps_av1_10bit.mkv"}, CODEC_OPTIONAL}, 190 })); 191 } 192 193 if (IS_AT_LEAST_B && apvSupport() && extractorMp4EnableApv()) { 194 exhaustiveArgsList.addAll( 195 Arrays.asList( 196 new Object[][] { 197 { 198 MediaFormat.MIMETYPE_VIDEO_APV, 199 new String[] { 200 "pattern_640x480_30fps_16mbps_apv_10bit.mp4", 201 "pattern_1280x720_30fps_30mbps_apv_10bit.mp4" 202 }, 203 CODEC_OPTIONAL 204 }, 205 })); 206 } 207 List<Object[]> argsList = prepareParamList(exhaustiveArgsList, isEncoder, needAudio, 208 needVideo, false); 209 if (IS_AT_LEAST_V && android.media.codec.Flags.dynamicColorAspects()) { 210 List<Object[]> dynamicColorAspectsArgs = Arrays.asList(new Object[][]{ 211 {MediaFormat.MIMETYPE_VIDEO_AVC, new String[]{ 212 "bbb_640x360_512kbps_30fps_avc_nob.mp4", 213 "cosmat_1280x720_24fps_crf22_avc_10bit_nob.mkv", 214 "bbb_800x640_768kbps_30fps_avc_nob.mp4", 215 "cosmat_640x360_24fps_crf22_avc_10bit_nob.mkv", 216 "bbb_1280x720_1mbps_30fps_avc_2b.mp4", 217 "cosmat_800x640_24fps_crf22_avc_10bit_2b.mkv"}, CODEC_OPTIONAL}, 218 {MediaFormat.MIMETYPE_VIDEO_HEVC, new String[]{ 219 "bbb_640x360_512kbps_30fps_hevc_nob.mp4", 220 "cosmat_1280x720_24fps_crf22_hevc_10bit_nob.mkv", 221 "cosmat_352x288_hdr10_only_stream_hevc.mkv", 222 "bbb_800x640_768kbps_30fps_hevc_nob.mp4", 223 "cosmat_640x360_24fps_crf22_hevc_10bit_2b.mkv", 224 "bbb_1280x720_1mbps_30fps_hevc_2b.mp4", 225 "cosmat_352x288_hdr10plus_hevc.mp4", 226 "cosmat_800x640_24fps_crf22_hevc_10bit_nob.mkv"}, CODEC_OPTIONAL}, 227 {MediaFormat.MIMETYPE_VIDEO_VP9, new String[]{ 228 "bbb_640x360_512kbps_30fps_vp9.webm", 229 "cosmat_1280x720_24fps_crf22_vp9_10bit.mkv", 230 "cosmat_352x288_hdr10_only_container_vp9.mkv", 231 "bbb_800x640_768kbps_30fps_vp9.webm", 232 "cosmat_640x360_24fps_crf22_vp9_10bit.mkv", 233 "bbb_1280x720_1mbps_30fps_vp9.webm", 234 "cosmat_800x640_24fps_crf22_vp9_10bit.mkv"}, CODEC_OPTIONAL}, 235 {MediaFormat.MIMETYPE_VIDEO_AV1, new String[]{ 236 "bbb_640x360_512kbps_30fps_av1.webm", 237 "cosmat_1280x720_24fps_1200kbps_av1_10bit.mkv", 238 "cosmat_352x288_hdr10_stream_and_container_correct_av1.mkv", 239 "bbb_800x640_768kbps_30fps_av1.webm", 240 "cosmat_640x360_24fps_512kbps_av1_10bit.mkv", 241 "bbb_1280x720_1mbps_30fps_av1.webm", 242 "cosmat_352x288_hdr10plus_av1.mkv", 243 "cosmat_800x640_24fps_768kbps_av1_10bit.mkv"}, CODEC_OPTIONAL}, 244 }); 245 argsList.addAll(prepareParamList(dynamicColorAspectsArgs, isEncoder, needAudio, 246 needVideo, false /* mustTestAllCodecs */, ComponentClass.ALL, 247 new String[]{MediaCodecInfo.CodecCapabilities.FEATURE_DynamicColorAspects})); 248 } 249 return argsList; 250 } 251 252 @Override dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)253 protected void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info) { 254 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 255 mSawOutputEOS = true; 256 } 257 if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { 258 mOutputBuff.saveOutPTS(info.presentationTimeUs); 259 mOutputCount++; 260 } 261 mCodec.releaseOutputBuffer(bufferIndex, mSurface != null); 262 } 263 getSupportedFiles(String[] srcFiles, String codecName, String mediaType)264 static List<String> getSupportedFiles(String[] srcFiles, String codecName, String mediaType) 265 throws IOException { 266 List<String> supportedClips = new ArrayList<>(); 267 for (String srcFile : srcFiles) { 268 MediaFormat format = getFormatInStream(mediaType, srcFile); 269 if (isFormatSupported(codecName, mediaType, format)) { 270 supportedClips.add(srcFile); 271 } 272 } 273 return supportedClips; 274 } 275 createInputList(String srcFile, String mediaType, ByteBuffer buffer, List<MediaCodec.BufferInfo> list, int offset, long ptsOffset)276 static Pair<MediaFormat, Long> createInputList(String srcFile, String mediaType, 277 ByteBuffer buffer, List<MediaCodec.BufferInfo> list, int offset, long ptsOffset) 278 throws IOException { 279 Preconditions.assertTestFileExists(srcFile); 280 MediaExtractor extractor = new MediaExtractor(); 281 extractor.setDataSource(srcFile); 282 MediaFormat format = null; 283 for (int trackID = 0; trackID < extractor.getTrackCount(); trackID++) { 284 MediaFormat fmt = extractor.getTrackFormat(trackID); 285 if (mediaType.equalsIgnoreCase(fmt.getString(MediaFormat.KEY_MIME))) { 286 format = fmt; 287 extractor.selectTrack(trackID); 288 break; 289 } 290 } 291 if (format == null) { 292 extractor.release(); 293 throw new IllegalArgumentException( 294 "No track with mediaType: " + mediaType + " found in file: " + srcFile); 295 } 296 if (hasCSD(format)) { 297 MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); 298 bufferInfo.offset = offset; 299 bufferInfo.size = 0; 300 // For some devices with VNDK versions till Android U, sending a zero 301 // timestamp for CSD results in out of order timestamps at the output. 302 // For devices with VNDK versions > Android U, codecs are expected to 303 // handle CSD buffers with timestamp set to zero. 304 bufferInfo.presentationTimeUs = VNDK_IS_AT_MOST_U ? ptsOffset : 0; 305 bufferInfo.flags = MediaCodec.BUFFER_FLAG_CODEC_CONFIG; 306 for (int i = 0; ; i++) { 307 String csdKey = "csd-" + i; 308 if (format.containsKey(csdKey)) { 309 ByteBuffer csdBuffer = format.getByteBuffer(csdKey); 310 bufferInfo.size += csdBuffer.limit(); 311 buffer.put(csdBuffer); 312 format.removeKey(csdKey); 313 } else break; 314 } 315 list.add(bufferInfo); 316 offset += bufferInfo.size; 317 } 318 long maxPts = ptsOffset; 319 while (true) { 320 MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); 321 bufferInfo.size = extractor.readSampleData(buffer, offset); 322 if (bufferInfo.size < 0) break; 323 bufferInfo.offset = offset; 324 bufferInfo.presentationTimeUs = ptsOffset + extractor.getSampleTime(); 325 maxPts = Math.max(maxPts, bufferInfo.presentationTimeUs); 326 int flags = extractor.getSampleFlags(); 327 bufferInfo.flags = 0; 328 if ((flags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) { 329 bufferInfo.flags |= MediaCodec.BUFFER_FLAG_KEY_FRAME; 330 } 331 list.add(bufferInfo); 332 extractor.advance(); 333 offset += bufferInfo.size; 334 } 335 buffer.clear(); 336 buffer.position(offset); 337 extractor.release(); 338 return Pair.create(format, maxPts); 339 } 340 341 /** 342 * Test video decoder for seamless resolution changes. 343 */ 344 @CddTest(requirements = {"5.3/C-1-1"}) 345 @ApiTest(apis = {"android.media.MediaCodecInfo.CodecCapabilities#FEATURE_AdaptivePlayback", 346 "android.media.MediaCodecInfo.CodecCapabilities#FEATURE_DynamicColorAspects"}) 347 @LargeTest 348 @Test(timeout = PER_TEST_TIMEOUT_LARGE_TEST_MS) testAdaptivePlayback()349 public void testAdaptivePlayback() throws IOException, InterruptedException { 350 boolean hasSupport = isFeatureSupported(mCodecName, mMediaType, 351 MediaCodecInfo.CodecCapabilities.FEATURE_AdaptivePlayback); 352 if (MUST_SUPPORT_APB.contains(mMediaType)) { 353 Assert.assertTrue("codec: " + mCodecName + " is required to support " 354 + "FEATURE_AdaptivePlayback" + " for mediaType: " + mMediaType, hasSupport); 355 } else { 356 Assume.assumeTrue("codec: " + mCodecName + " does not support FEATURE_AdaptivePlayback", 357 hasSupport); 358 } 359 List<String> resFiles = getSupportedFiles(mSrcFiles, mCodecName, mMediaType); 360 if (mSupportRequirements.equals(CODEC_ALL)) { 361 Assert.assertEquals("codec: " + mCodecName + " does not support all files in the" 362 + " input list", resFiles.size(), mSrcFiles.length); 363 } 364 Assume.assumeTrue("none of the given test clips are supported by the codec: " 365 + mCodecName, !resFiles.isEmpty()); 366 ArrayList<MediaFormat> formats = new ArrayList<>(); 367 int totalSize = 0; 368 for (String resFile : resFiles) { 369 File file = new File(resFile); 370 totalSize += (int) file.length(); 371 } 372 long ptsOffset = 0; 373 int buffOffset = 0; 374 ArrayList<MediaCodec.BufferInfo> list = new ArrayList<>(); 375 ByteBuffer buffer = ByteBuffer.allocate(totalSize); 376 for (String file : resFiles) { 377 Pair<MediaFormat, Long> metadata = 378 createInputList(file, mMediaType, buffer, list, buffOffset, ptsOffset); 379 formats.add(metadata.first); 380 ptsOffset = metadata.second + 1000000L; 381 buffOffset = (list.get(list.size() - 1).offset) + (list.get(list.size() - 1).size); 382 } 383 mOutputBuff = new OutputManager(); 384 { 385 mCodec = MediaCodec.createByCodecName(mCodecName); 386 MediaFormat format = formats.get(0); 387 mActivity.setScreenParams(getWidth(format), getHeight(format), true); 388 mOutputBuff.reset(); 389 configureCodec(format, true, false, false); 390 mCodec.start(); 391 doWork(buffer, list); 392 queueEOS(); 393 waitForAllOutputs(); 394 mCodec.reset(); 395 mCodec.release(); 396 } 397 } 398 } 399