1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.video.cts; 18 19 import static android.video.cts.CodecPerformanceTestBase.ScalingFactor.Mode; 20 21 import static org.junit.Assert.assertTrue; 22 import static org.junit.Assert.fail; 23 import static org.junit.Assume.assumeTrue; 24 25 import android.media.MediaCodec; 26 import android.media.MediaCodecInfo; 27 import android.media.MediaCodecList; 28 import android.media.MediaExtractor; 29 import android.media.MediaFormat; 30 import android.media.cts.TestArgs; 31 import android.os.Build; 32 import android.os.SystemProperties; 33 import android.util.Range; 34 import android.view.Surface; 35 36 import org.junit.Before; 37 38 import java.io.File; 39 import java.io.IOException; 40 import java.nio.ByteBuffer; 41 import java.util.ArrayList; 42 import java.util.List; 43 44 class CodecPerformanceTestBase { 45 private static final String LOG_TAG = CodecPerformanceTestBase.class.getSimpleName(); 46 static final long Q_DEQ_TIMEOUT_US = 5000; // block at most 5ms while looking for io buffers 47 static final int PER_TEST_TIMEOUT_LARGE_TEST_MS = 300000; 48 static final int MIN_FRAME_COUNT = 500; 49 static final int SELECT_ALL = 0; // Select all codecs 50 static final int SELECT_HARDWARE = 1; // Select Hardware codecs only 51 static final int SELECT_SOFTWARE = 2; // Select Software codecs only 52 // allowed tolerance in measured fps vs expected fps, i.e. codecs achieving fps 53 // that is greater than (FPS_TOLERANCE_FACTOR * expectedFps) will be considered as 54 // passing the test 55 static final double FPS_TOLERANCE_FACTOR; 56 static final boolean IS_AT_LEAST_VNDK_S; 57 58 static final int DEVICE_INITIAL_SDK; 59 static final int VNDK_VERSION; 60 61 // Some older devices can not support concurrent instances of both decoder and encoder 62 // at max resolution. To handle such cases, this test is limited to test the 63 // resolutions that are less than half of max supported frame sizes of encoder. 64 static final boolean EXCLUDE_ENCODER_MAX_RESOLUTION; 65 66 // Some older devices can not support concurrent instances of both decoder and encoder 67 // for operating rates > 0 and < 30 68 static final boolean EXCLUDE_ENCODER_OPRATE_0_TO_30; 69 70 static final String mInputPrefix = WorkDir.getMediaDirString(); 71 72 ArrayList<MediaCodec.BufferInfo> mBufferInfos; 73 ByteBuffer mBuff; 74 75 final String mDecoderName; 76 final String mTestFile; 77 final int mKeyPriority; 78 final float mMaxOpRateScalingFactor; 79 final Mode mMaxOpRateScalingFactorMode; 80 81 String mDecoderMime; 82 int mWidth; 83 int mHeight; 84 int mFrameRate; 85 86 boolean mSawDecInputEOS = false; 87 boolean mSawDecOutputEOS = false; 88 int mDecInputNum = 0; 89 int mDecOutputNum = 0; 90 int mSampleIndex = 0; 91 92 MediaCodec mDecoder; 93 MediaFormat mDecoderFormat; 94 Surface mSurface; 95 double mOperatingRateExpected; 96 97 static class ScalingFactor{ 98 public enum Mode {ORDINARY, NEGATIVE, VERY_LARGE}; 99 100 private Mode mMode; 101 private float mValue; 102 ScalingFactor(Mode mode, float value)103 public ScalingFactor(Mode mode, float value) { 104 mMode = mode; 105 mValue = value; 106 } 107 getValue()108 public float getValue() { 109 return mValue; 110 } 111 getMode()112 public Mode getMode() { 113 return mMode; 114 } 115 } 116 117 static final ScalingFactor[] SCALING_FACTORS_LIST = { 118 new ScalingFactor(Mode.ORDINARY, 2.5f), 119 new ScalingFactor(Mode.ORDINARY, 1.25f), 120 new ScalingFactor(Mode.ORDINARY, 1.0f), 121 new ScalingFactor(Mode.ORDINARY, 0.75f), 122 new ScalingFactor(Mode.ORDINARY, 0.0f), 123 new ScalingFactor(Mode.NEGATIVE, -1.0f), 124 new ScalingFactor(Mode.VERY_LARGE, 0.0f)}; 125 static final int[] KEY_PRIORITIES_LIST = new int[]{1, 0}; 126 127 static { 128 // os.Build.VERSION.DEVICE_INITIAL_SDK_INT can be used here, but it was called 129 // os.Build.VERSION.FIRST_SDK_INT in Android R and below. Using DEVICE_INITIAL_SDK_INT 130 // will mean that the tests built in Android S can't be run on Android R and below. 131 DEVICE_INITIAL_SDK = SystemProperties.getInt("ro.product.first_api_level", 0); 132 133 VNDK_VERSION = SystemProperties.getInt("ro.vndk.version", 134 Build.VERSION_CODES.CUR_DEVELOPMENT); 135 136 // fps tolerance factor is kept quite low for devices with Android R VNDK or lower 137 FPS_TOLERANCE_FACTOR = VNDK_VERSION <= Build.VERSION_CODES.R ? 0.67 : 0.95; 138 139 IS_AT_LEAST_VNDK_S = VNDK_VERSION > Build.VERSION_CODES.R; 140 141 // Encoders on devices launched on Android Q and lower aren't tested at maximum resolution 142 EXCLUDE_ENCODER_MAX_RESOLUTION = DEVICE_INITIAL_SDK <= Build.VERSION_CODES.Q; 143 144 // Encoders on devices launched on Android R and lower aren't tested when operating rate 145 // that is set is > 0 and < 30. 146 // This includes devices launched on Android S with R or lower vendor partition. 147 EXCLUDE_ENCODER_OPRATE_0_TO_30 = 148 !IS_AT_LEAST_VNDK_S || (DEVICE_INITIAL_SDK <= Build.VERSION_CODES.R); 149 } 150 151 @Before prologue()152 public void prologue() { 153 assumeTrue("For VNDK R and below, operating rate <= 0 isn't tested", 154 IS_AT_LEAST_VNDK_S || mMaxOpRateScalingFactor > 0.0); 155 156 assumeTrue("For devices launched on Android P and below, operating rate tests are disabled", 157 DEVICE_INITIAL_SDK > Build.VERSION_CODES.P); 158 159 if (DEVICE_INITIAL_SDK <= Build.VERSION_CODES.Q) { 160 assumeTrue("For devices launched with Android Q and below, operating rate tests are " + 161 "limited to operating rate scaling factor > 0.0 and <= 1.25", 162 mMaxOpRateScalingFactor > 0.0 && mMaxOpRateScalingFactor <= 1.25); 163 } 164 } 165 CodecPerformanceTestBase(String decoderName, String testFile, int keyPriority, float maxOpRateScalingFactor, Mode maxOpRateScalingFactorMode)166 public CodecPerformanceTestBase(String decoderName, String testFile, int keyPriority, 167 float maxOpRateScalingFactor, Mode maxOpRateScalingFactorMode) { 168 mDecoderName = decoderName; 169 mTestFile = testFile; 170 mKeyPriority = keyPriority; 171 mMaxOpRateScalingFactor = maxOpRateScalingFactor; 172 mMaxOpRateScalingFactorMode = maxOpRateScalingFactorMode; 173 mBufferInfos = new ArrayList<>(); 174 } 175 getVideoFormat(String filePath)176 static MediaFormat getVideoFormat(String filePath) throws IOException { 177 final String input = mInputPrefix + filePath; 178 MediaExtractor extractor = new MediaExtractor(); 179 extractor.setDataSource(input); 180 for (int trackID = 0; trackID < extractor.getTrackCount(); trackID++) { 181 MediaFormat format = extractor.getTrackFormat(trackID); 182 if (format.getString(MediaFormat.KEY_MIME).startsWith("video/")) { 183 extractor.release(); 184 return format; 185 } 186 } 187 extractor.release(); 188 return null; 189 } 190 selectCodecs(String mime, ArrayList<MediaFormat> formats, String[] features, boolean isEncoder)191 static ArrayList<String> selectCodecs(String mime, ArrayList<MediaFormat> formats, 192 String[] features, boolean isEncoder) { 193 return selectCodecs(mime, formats, features, isEncoder, SELECT_ALL); 194 } 195 selectHardwareCodecs(String mime, ArrayList<MediaFormat> formats, String[] features, boolean isEncoder)196 static ArrayList<String> selectHardwareCodecs(String mime, ArrayList<MediaFormat> formats, 197 String[] features, boolean isEncoder) { 198 return selectCodecs(mime, formats, features, isEncoder, SELECT_HARDWARE); 199 } 200 selectCodecs(String mime, ArrayList<MediaFormat> formats, String[] features, boolean isEncoder, int selectCodecOption)201 static ArrayList<String> selectCodecs(String mime, ArrayList<MediaFormat> formats, 202 String[] features, boolean isEncoder, int selectCodecOption) { 203 ArrayList<String> listOfCodecs = new ArrayList<>(); 204 if (TestArgs.shouldSkipMediaType(mime)) { 205 return listOfCodecs; 206 } 207 MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 208 MediaCodecInfo[] codecInfos = codecList.getCodecInfos(); 209 210 for (MediaCodecInfo codecInfo : codecInfos) { 211 if (TestArgs.shouldSkipCodec(codecInfo.getName())) { 212 continue; 213 } 214 if (codecInfo.isEncoder() != isEncoder) continue; 215 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && codecInfo.isAlias()) continue; 216 if (selectCodecOption == SELECT_HARDWARE && !codecInfo.isHardwareAccelerated()) 217 continue; 218 else if (selectCodecOption == SELECT_SOFTWARE && !codecInfo.isSoftwareOnly()) 219 continue; 220 String[] types = codecInfo.getSupportedTypes(); 221 for (String type : types) { 222 if (type.equalsIgnoreCase(mime)) { 223 boolean isOk = true; 224 MediaCodecInfo.CodecCapabilities codecCapabilities = 225 codecInfo.getCapabilitiesForType(type); 226 if (formats != null) { 227 for (MediaFormat format : formats) { 228 if (!codecCapabilities.isFormatSupported(format)) { 229 isOk = false; 230 break; 231 } 232 } 233 } 234 if (features != null) { 235 for (String feature : features) { 236 if (!codecCapabilities.isFeatureSupported(feature)) { 237 isOk = false; 238 break; 239 } 240 } 241 } 242 if (isOk) listOfCodecs.add(codecInfo.getName()); 243 } 244 } 245 } 246 return listOfCodecs; 247 } 248 setUpDecoderInput()249 MediaFormat setUpDecoderInput() throws IOException { 250 final String input = mInputPrefix + mTestFile; 251 MediaExtractor extractor = new MediaExtractor(); 252 extractor.setDataSource(input); 253 for (int trackID = 0; trackID < extractor.getTrackCount(); trackID++) { 254 MediaFormat format = extractor.getTrackFormat(trackID); 255 if (format.getString(MediaFormat.KEY_MIME).startsWith("video/")) { 256 extractor.selectTrack(trackID); 257 File file = new File(input); 258 int bufferSize = (int) file.length(); 259 mBuff = ByteBuffer.allocate(bufferSize); 260 int offset = 0; 261 long maxPTS = 0; 262 while (true) { 263 MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); 264 bufferInfo.size = extractor.readSampleData(mBuff, offset); 265 if (bufferInfo.size < 0) break; 266 bufferInfo.offset = offset; 267 bufferInfo.presentationTimeUs = extractor.getSampleTime(); 268 maxPTS = Math.max(maxPTS, bufferInfo.presentationTimeUs); 269 int flags = extractor.getSampleFlags(); 270 bufferInfo.flags = 0; 271 if ((flags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) { 272 bufferInfo.flags |= MediaCodec.BUFFER_FLAG_KEY_FRAME; 273 } 274 mBufferInfos.add(bufferInfo); 275 extractor.advance(); 276 offset += bufferInfo.size; 277 } 278 279 // If the clip doesn't have sufficient frames, loopback by copying bufferInfos 280 // from the start of the list and incrementing the timestamp. 281 int actualBufferInfosCount = mBufferInfos.size(); 282 long ptsOffset; 283 while (mBufferInfos.size() < MIN_FRAME_COUNT) { 284 ptsOffset = maxPTS + 1000000L; 285 for (int i = 0; i < actualBufferInfosCount; i++) { 286 MediaCodec.BufferInfo tmpBufferInfo = mBufferInfos.get(i); 287 MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); 288 bufferInfo.set(tmpBufferInfo.offset, tmpBufferInfo.size, 289 ptsOffset + tmpBufferInfo.presentationTimeUs, 290 tmpBufferInfo.flags); 291 maxPTS = Math.max(maxPTS, bufferInfo.presentationTimeUs); 292 mBufferInfos.add(bufferInfo); 293 if (mBufferInfos.size() >= MIN_FRAME_COUNT) break; 294 } 295 } 296 MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); 297 bufferInfo.set(0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); 298 mBufferInfos.add(bufferInfo); 299 mDecoderMime = format.getString(MediaFormat.KEY_MIME); 300 mWidth = format.getInteger(MediaFormat.KEY_WIDTH); 301 mHeight = format.getInteger(MediaFormat.KEY_HEIGHT); 302 mFrameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE, 30); 303 extractor.release(); 304 return format; 305 } 306 } 307 extractor.release(); 308 fail("No video track found in file: " + mTestFile); 309 return null; 310 } 311 312 // TODO (b/193458026) Limit max expected fps getMaxExpectedFps(int width, int height)313 static int getMaxExpectedFps(int width, int height) { 314 int numSamples = width * height; 315 if (numSamples > 3840 * 2160 * 2) { // 8K 316 return 30; 317 } else if (numSamples > 1920 * 1088 * 2) { // 4K 318 return 120; 319 } else { 320 return 240; 321 } 322 } 323 getMaxOperatingRate(String codecName, String mime)324 int getMaxOperatingRate(String codecName, String mime) throws IOException { 325 MediaCodec codec = MediaCodec.createByCodecName(codecName); 326 MediaCodecInfo mediaCodecInfo = codec.getCodecInfo(); 327 List<MediaCodecInfo.VideoCapabilities.PerformancePoint> pps = mediaCodecInfo 328 .getCapabilitiesForType(mime).getVideoCapabilities() 329 .getSupportedPerformancePoints(); 330 assertTrue(pps.size() > 0); 331 MediaCodecInfo.VideoCapabilities.PerformancePoint cpp = 332 new MediaCodecInfo.VideoCapabilities.PerformancePoint(mWidth, mHeight, mFrameRate); 333 int macroblocks = cpp.getMaxMacroBlocks(); 334 int maxOperatingRate = -1; 335 for (MediaCodecInfo.VideoCapabilities.PerformancePoint pp : pps) { 336 if (pp.covers(cpp)) { 337 maxOperatingRate = Math.max(Math.min(pp.getMaxFrameRate(), 338 (int) pp.getMaxMacroBlockRate() / macroblocks), maxOperatingRate); 339 } 340 } 341 codec.release(); 342 assumeTrue("Codec doesn't advertise performance point for " + mWidth + "x" + mHeight, 343 maxOperatingRate != -1); 344 return maxOperatingRate; 345 } 346 getEncoderMinComplexity(String codecName, String mime)347 int getEncoderMinComplexity(String codecName, String mime) throws IOException { 348 MediaCodec codec = MediaCodec.createByCodecName(codecName); 349 MediaCodecInfo mediaCodecInfo = codec.getCodecInfo(); 350 int minComplexity = -1; 351 if (mediaCodecInfo.isEncoder()) { 352 Range<Integer> complexityRange = mediaCodecInfo 353 .getCapabilitiesForType(mime).getEncoderCapabilities() 354 .getComplexityRange(); 355 minComplexity = complexityRange.getLower(); 356 } 357 codec.release(); 358 return minComplexity; 359 } 360 getMaxFrameSize(String codecName, String mime)361 static int getMaxFrameSize(String codecName, String mime) throws IOException { 362 MediaCodec codec = MediaCodec.createByCodecName(codecName); 363 MediaCodecInfo.CodecCapabilities codecCapabilities = 364 codec.getCodecInfo().getCapabilitiesForType(mime); 365 MediaCodecInfo.VideoCapabilities vc = codecCapabilities.getVideoCapabilities(); 366 Range<Integer> heights = vc.getSupportedHeights(); 367 Range<Integer> widths = vc.getSupportedWidthsFor(heights.getUpper()); 368 int maxFrameSize = heights.getUpper() * widths.getUpper(); 369 codec.release(); 370 return maxFrameSize; 371 } 372 enqueueDecoderInput(int bufferIndex)373 void enqueueDecoderInput(int bufferIndex) { 374 MediaCodec.BufferInfo info = mBufferInfos.get(mSampleIndex++); 375 if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { 376 ByteBuffer dstBuf = mDecoder.getInputBuffer(bufferIndex); 377 dstBuf.put(mBuff.array(), info.offset, info.size); 378 mDecInputNum++; 379 } 380 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 381 mSawDecInputEOS = true; 382 } 383 mDecoder.queueInputBuffer(bufferIndex, 0, info.size, info.presentationTimeUs, info.flags); 384 } 385 dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info, boolean render)386 void dequeueDecoderOutput(int bufferIndex, MediaCodec.BufferInfo info, boolean render) { 387 if (info.size > 0) { 388 mDecOutputNum++; 389 } 390 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 391 mSawDecOutputEOS = true; 392 } 393 mDecoder.releaseOutputBuffer(bufferIndex, render); 394 } 395 } 396