1 /* 2 * Copyright (C) 2025 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.google.android.angleallowlists.vts; 18 19 import static org.junit.Assert.assertFalse; 20 import static org.junit.Assert.assertTrue; 21 import static org.junit.Assert.fail; 22 23 import com.android.compatibility.common.util.FeatureUtil; 24 import com.android.compatibility.common.util.PropertyUtil; 25 import com.android.compatibility.common.util.VsrTest; 26 import com.android.tradefed.config.Option; 27 import com.android.tradefed.device.DeviceNotAvailableException; 28 import com.android.tradefed.device.ITestDevice; 29 import com.android.tradefed.log.LogUtil; 30 import com.android.tradefed.testtype.DeviceJUnit4ClassRunner; 31 import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test; 32 import com.android.tradefed.util.RunUtil; 33 import com.google.common.io.Files; 34 import java.io.File; 35 import java.io.IOException; 36 import java.nio.charset.StandardCharsets; 37 import java.util.ArrayList; 38 import java.util.HashMap; 39 import java.util.HashSet; 40 import java.util.List; 41 import java.util.Map; 42 import java.util.Set; 43 import java.util.StringTokenizer; 44 import java.util.regex.Matcher; 45 import java.util.regex.Pattern; 46 import org.junit.After; 47 import org.junit.Assume; 48 import org.junit.Before; 49 import org.junit.Rule; 50 import org.junit.Test; 51 import org.junit.rules.TemporaryFolder; 52 import org.junit.rules.TestName; 53 import org.junit.runner.RunWith; 54 55 @RunWith(DeviceJUnit4ClassRunner.class) 56 public class AngleAllowlistTraceTest extends BaseHostJUnit4Test { 57 // Object that invokes adb commands and interacts with test devices 58 private Helper mTestHelper; 59 60 // Multi-user system property 61 private String mCurrentUser; 62 63 // ANGLE trace app directory. The directory path is affected by the value of mCurrentUser 64 private String mAngleTraceTestAppHomeDir; 65 private String mAngleTraceTestBlobCacheDir; 66 67 // Properties used for Vulkan feature checks 68 private static final int VULKAN_1_1 = 0x00401000; // 1.1.0 69 private static final String VULKAN_VERSION_FEATURE = "feature:android.hardware.vulkan.version"; 70 private static final String VULKAN_LEVEL_FEATURE = "feature:android.hardware.vulkan.level"; 71 72 // Package install attempts and intervals before install retries 73 private static final int NUM_ATTEMPTS = 5; 74 private static final int APP_INSTALL_REATTEMPT_SLEEP_MSEC = 5000; 75 76 // Trace test max runs 77 private static final int MAX_TRACE_RUN_COUNT = 5; 78 79 // Trace test FPS requirement 80 private static final double FPS_REQUIREMENT = 60.0; 81 82 // Comparison threshold when NATIVE FPS > 60 and ANGLE FPS < 60 83 // Allow 10% threshold when measuring ANGLE FPS against 60, so that we consider below case as 84 // passing: 85 // NATIVE: 61.18 86 // ANGLE: 59.81 87 private static final double FPS_THRESHOLD = 0.9; 88 89 private static enum DriverType { ANGLE, NATIVE } 90 ; 91 92 // Properties used for ANGLE Trace test 93 @Rule public final TemporaryFolder mTemporaryFolder = new TemporaryFolder(); 94 95 @Rule 96 public final DeviceJUnit4ClassRunner.TestMetrics mMetrics = 97 new DeviceJUnit4ClassRunner.TestMetrics(); 98 99 @Rule 100 public final DeviceJUnit4ClassRunner.TestLogData mLogData = 101 new DeviceJUnit4ClassRunner.TestLogData(); 102 103 @Rule public final TestName mTestName = new TestName(); 104 105 @Option(name = "angle_trace_package_path", description = "path to angle trace package files") 106 private String mANGLETracePackagePath = null; 107 108 // Allows partners to run tests on devices that haven't fully configured with proper vendor api 109 // b/377337787#comment25 110 @Option(name = "bypass-vendor-api-requirement", 111 description = "whether to bypass the vendor api requirement check") 112 private boolean mBypassVendorApiRequirement = false; 113 114 private static final String ANGLE_TRACE_TEST_PACKAGE_NAME = "com.android.angle.test"; 115 private static final String ANGLE_TRACE_DATA_ON_DEVICE_DIR = 116 "/storage/emulated/0/chromium_tests_root"; 117 118 private static final int WAIT_RUN_TRACES_MILLIS = 5 * 60 * 1000; 119 private HashMap<String, Double> mTracePerfANGLEFPS = new HashMap<>(); 120 private HashMap<String, Double> mTracePerfNativeFPS = new HashMap<>(); 121 122 private HashSet<String> mSkippedTrace = new HashSet<>(); 123 private HashSet<String> mTracePerfANGLEBelowRequiredFPS = new HashSet<>(); 124 125 // Group 1: e.g. "wall_time" 126 // Group 2: e.g. "1945_air_force". 127 // Group 3: time in ms. e.g. "11.5506817933". 128 private static final Pattern PATTERN_METRICS = Pattern.compile( 129 "TracePerf_(?:vulkan|native)\\.(wall_time|gpu_time): ([^\\s=]*)= ([^\\s]*) ms"); 130 private static final Pattern PATTERN_TRACE_NAMES = Pattern.compile("TraceTest.(.*?)\n"); 131 getDefaultANGLETracePathDir()132 private String getDefaultANGLETracePathDir() { 133 return System.getProperty("user.dir").concat("/angle_traces"); 134 } 135 setANGLETracePackagePath()136 private void setANGLETracePackagePath() { 137 if (mANGLETracePackagePath == null) { 138 mANGLETracePackagePath = getDefaultANGLETracePathDir(); 139 } 140 } 141 142 /** 143 * Invokes BaseHostJUnit4Test installPackage() API, with NUM_ATTEMPTS of retries 144 * Difference between this function and Helper.installApkFile() is this function can only 145 * install apks that exist in the same test module (e.g. apks that are specified under 146 * device_common_data or data field in Android.bp), while installApkFile() can install apks from 147 * any directory. 148 */ installTestApp(String appName)149 private void installTestApp(String appName) throws Exception { 150 for (int i = 0; i < NUM_ATTEMPTS; i++) { 151 try { 152 installPackage(appName); 153 return; 154 } catch (Exception e) { 155 LogUtil.CLog.e("Exception in installing the app: %s, error message: %s", appName, 156 e.getMessage()); 157 if (i < NUM_ATTEMPTS - 1) { 158 RunUtil.getDefault().sleep(APP_INSTALL_REATTEMPT_SLEEP_MSEC); 159 } else { 160 throw e; 161 } 162 } 163 } 164 } 165 getAngleInstrumentCommand(final String gtestArguments)166 private String getAngleInstrumentCommand(final String gtestArguments) { 167 return String.format("am instrument -w -e" 168 + " org.chromium.native_test.NativeTestInstrumentationTestRunner.StdoutFile" 169 + " %s/files/out.txt -e" 170 + " org.chromium.native_test.NativeTest.CommandLineFlags \"%s\" -e" 171 + " org.chromium.native_test." 172 + "NativeTestInstrumentationTestRunner.ShardNanoTimeout" 173 + " 1000000000000000000 -e" 174 + " org.chromium.native_test." 175 + "NativeTestInstrumentationTestRunner.NativeTestActivity" 176 + " com.android.angle.test.AngleUnitTestActivity " 177 + " com.android.angle.test/" 178 + "org.chromium.build.gtest_apk.NativeTestInstrumentationTestRunner", 179 mAngleTraceTestAppHomeDir, gtestArguments); 180 } 181 runAndBlockAngleTestApp(final Helper helper, final String gtestArguments)182 private void runAndBlockAngleTestApp(final Helper helper, final String gtestArguments) 183 throws CommandException, InstrumentationCrashException { 184 helper.adbShellInstrumentationCommandCheck( 185 WAIT_RUN_TRACES_MILLIS, getAngleInstrumentCommand(gtestArguments)); 186 187 // Cat the stdout file. This will be logged. 188 helper.adbShellCommandCheck(Helper.WAIT_ADB_SHELL_FILE_OP_MILLIS, 189 String.format("run-as %s cat %s/files/out.txt", ANGLE_TRACE_TEST_PACKAGE_NAME, 190 mAngleTraceTestAppHomeDir)); 191 } 192 193 /** Run angle_trace_tests app to get the list of traces */ runAngleListTrace(final Helper helper, final File gtestStdoutFile)194 private List<String> runAngleListTrace(final Helper helper, final File gtestStdoutFile) 195 throws CommandException, InstrumentationCrashException, IOException, 196 DeviceNotAvailableException { 197 // verify the device state 198 helper.assertDeviceStateOk(); 199 200 // Remove previous stdout file on the device, if present. 201 helper.adbShellCommandCheck(Helper.WAIT_ADB_SHELL_FILE_OP_MILLIS, 202 String.format("run-as %s rm -f %s/files/out.txt", ANGLE_TRACE_TEST_PACKAGE_NAME, 203 mAngleTraceTestAppHomeDir)); 204 205 // Check file has gone but the directory exists. 206 helper.adbShellCommandCheck(Helper.WAIT_ADB_SHELL_FILE_OP_MILLIS, 207 String.format("run-as %s test ! -f %s/files/out.txt && run-as %s test -d %s", 208 ANGLE_TRACE_TEST_PACKAGE_NAME, mAngleTraceTestAppHomeDir, 209 ANGLE_TRACE_TEST_PACKAGE_NAME, mAngleTraceTestAppHomeDir)); 210 211 // run angle_trace_tests app with --list-tests arg 212 runAndBlockAngleTestApp(helper, "--list-tests"); 213 214 // pull the test output file 215 helper.adbShellCommandWithStdout(Helper.WAIT_ADB_SHELL_FILE_OP_MILLIS, gtestStdoutFile, 216 String.format("run-as %s cat %s/files/out.txt", ANGLE_TRACE_TEST_PACKAGE_NAME, 217 mAngleTraceTestAppHomeDir)); 218 219 // Log it. 220 helper.logTextFile("ListTraceOutput", gtestStdoutFile); 221 222 // Read it. 223 final String stdout = Files.asCharSource(gtestStdoutFile, StandardCharsets.UTF_8).read(); 224 225 // Find list of traces to run 226 final Matcher traceNameMatcher = PATTERN_TRACE_NAMES.matcher(stdout); 227 228 // Store the trace names in an ArrayList 229 final ArrayList<String> traceNames = new ArrayList<>(); 230 231 while (traceNameMatcher.find()) { 232 final String traceName = traceNameMatcher.group(1); 233 traceNames.add(traceName); 234 } 235 return traceNames; 236 } 237 238 /** 239 * Execute angle trace test on trace with traceName until either of below conditions is met: 240 * 1) trace reaches FPS_REQUIREMENT fps 241 * 2) trace is ran for totalTraceRunCount times 242 */ runAngleTracePerfMultiTimes(final String traceName, final File gtestStdoutFile, final Helper helper, final DriverType driverType, final int totalTraceRunCount)243 private Double runAngleTracePerfMultiTimes(final String traceName, final File gtestStdoutFile, 244 final Helper helper, final DriverType driverType, final int totalTraceRunCount) 245 throws Throwable { 246 assertTrue("totalTraceRunCount must be greater than 0", totalTraceRunCount > 0); 247 Double traceFPS = null; 248 int traceRunCount = 0; 249 do { 250 runAngleTracePerf(traceName, gtestStdoutFile, helper, driverType); 251 switch (driverType) { 252 case ANGLE: 253 traceFPS = mTracePerfANGLEFPS.get(traceName); 254 break; 255 case NATIVE: 256 traceFPS = mTracePerfNativeFPS.get(traceName); 257 break; 258 default: 259 fail("must specify either ANGLE or NATIVE as the driverType"); 260 } 261 assertTrue(traceFPS != null); 262 } while ((Double.compare(traceFPS.doubleValue(), FPS_REQUIREMENT) < 0) 263 && ++traceRunCount < totalTraceRunCount); 264 265 return traceFPS; 266 } 267 268 /** 269 * Execute angle trace test on trace with traceName 270 * This function invokes trace test packaged in com.android.angle.test apk through 271 * instrumentation commands 272 */ runAngleTracePerf(final String testName, final File gtestStdoutFile, final Helper helper, final DriverType driverType)273 private void runAngleTracePerf(final String testName, final File gtestStdoutFile, 274 final Helper helper, final DriverType driverType) 275 throws CommandException, InstrumentationCrashException, IOException, 276 DeviceNotAvailableException { 277 // verify device state 278 helper.assertDeviceStateOk(); 279 280 // Remove previous stdout file on the device, if present. 281 helper.adbShellCommandCheck(Helper.WAIT_ADB_SHELL_FILE_OP_MILLIS, 282 String.format("run-as %s rm -f %s/files/out.txt", ANGLE_TRACE_TEST_PACKAGE_NAME, 283 mAngleTraceTestAppHomeDir)); 284 285 // Check file has gone but the directory exists. 286 helper.adbShellCommandCheck(Helper.WAIT_ADB_SHELL_FILE_OP_MILLIS, 287 String.format("run-as %s test ! -f %s/files/out.txt && run-as %s test -d %s", 288 ANGLE_TRACE_TEST_PACKAGE_NAME, mAngleTraceTestAppHomeDir, 289 ANGLE_TRACE_TEST_PACKAGE_NAME, mAngleTraceTestAppHomeDir)); 290 291 // Remove previous stdout file on the host, if present. 292 // noinspection ResultOfMethodCallIgnored 293 gtestStdoutFile.delete(); 294 295 // Check file has gone. 296 assertFalse("Failed to delete " + gtestStdoutFile, gtestStdoutFile.exists()); 297 298 // Clear blob cache 299 helper.adbShellCommandCheck(Helper.WAIT_ADB_LARGE_FILE_OP_MILLIS, 300 String.format("run-as %s rm -rf %s", ANGLE_TRACE_TEST_PACKAGE_NAME, 301 mAngleTraceTestBlobCacheDir)); 302 303 // Run the trace. 304 switch (driverType) { 305 case ANGLE: 306 // Set trace to run with System ANGLE 307 mTestHelper.adbShellCommandCheck(mTestHelper.WAIT_SET_GLOBAL_SETTING_MILLIS, 308 "settings put global angle_gl_driver_selection_pkgs" 309 + " com.android.angle.test"); 310 mTestHelper.adbShellCommandCheck(mTestHelper.WAIT_SET_GLOBAL_SETTING_MILLIS, 311 "settings put global angle_gl_driver_selection_values angle"); 312 break; 313 case NATIVE: 314 // Delete global vars so that trace run on default native driver 315 mTestHelper.adbShellCommandCheck(mTestHelper.WAIT_SET_GLOBAL_SETTING_MILLIS, 316 "settings delete global angle_gl_driver_selection_pkgs"); 317 mTestHelper.adbShellCommandCheck(mTestHelper.WAIT_SET_GLOBAL_SETTING_MILLIS, 318 "settings delete global angle_gl_driver_selection_values"); 319 break; 320 default: 321 fail("must specify either ANGLE or NATIVE as the driverType"); 322 break; 323 } 324 runAndBlockAngleTestApp(helper, 325 String.format("--gtest_filter=TraceTest.%s " 326 + "--use-gl=native " 327 + "--verbose " 328 + "--verbose-logging " 329 + "--fps-limit=100 " 330 + "--fixed-test-time-with-warmup " 331 + "10", 332 testName)); 333 334 helper.assertDeviceStateOk(); 335 336 getAndLogTraceMetrics(testName, gtestStdoutFile, helper, driverType); 337 } 338 339 /** 340 * Parse the trace test result and store the result 341 */ getAndLogTraceMetrics(final String testName, final File gtestStdoutFile, final Helper helper, final DriverType driverType)342 private void getAndLogTraceMetrics(final String testName, final File gtestStdoutFile, 343 final Helper helper, final DriverType driverType) throws CommandException, IOException { 344 String renderer = driverType.toString(); 345 346 // cat the test output file 347 helper.adbShellCommandWithStdout(Helper.WAIT_ADB_SHELL_FILE_OP_MILLIS, gtestStdoutFile, 348 String.format("run-as %s cat %s/files/out.txt", ANGLE_TRACE_TEST_PACKAGE_NAME, 349 mAngleTraceTestAppHomeDir)); 350 351 // Log it. 352 helper.logTextFile(String.format("%s_stdout", testName), gtestStdoutFile); 353 354 // Read it. 355 final String stdout = Files.asCharSource(gtestStdoutFile, StandardCharsets.UTF_8).read(); 356 357 boolean isTraceSkipped = false; 358 359 if (stdout.contains("Test skipped due to missing extension")) { 360 LogUtil.CLog.d("ANGLE trace test skipped: missing ext"); 361 isTraceSkipped = true; 362 } 363 364 if (stdout.contains("[ SKIPPED ] 1 test, listed below:")) { 365 LogUtil.CLog.d("ANGLE trace test skipped"); 366 isTraceSkipped = true; 367 } 368 369 if (isTraceSkipped) { 370 mSkippedTrace.add(testName); 371 helper.logMetricString(testName, "skipped"); 372 return; 373 } 374 375 // Find all metrics of interest in the stdout file and store them into metricsMap. 376 final Matcher metricsMatcher = PATTERN_METRICS.matcher(stdout); 377 final HashMap<String, String> metricsMap = new HashMap<>(); 378 379 // Keep a list as well, so that we process the metrics deterministically, in order. 380 final ArrayList<String> metricNames = new ArrayList<>(); 381 382 while (metricsMatcher.find()) { 383 final String metricName = metricsMatcher.group(1); 384 final String metricValue = metricsMatcher.group(3); 385 386 if (!metricsMap.containsKey(metricName)) { 387 metricNames.add(metricName); 388 } 389 metricsMap.put(metricName, metricValue); 390 } 391 392 assertTrue("We expect at least one metric.", metricNames.size() >= 1); 393 394 // Add each time as a metric 395 for (final String metricName : metricNames) { 396 final String metricValue = metricsMap.get(metricName); 397 398 // E.g. "1945_air_force.angle.wall_time" 399 // E.g. "1945_air_force.native.wall_time" 400 String fullMetricName = String.format("%s.%s.%s", testName, renderer, metricName); 401 402 helper.logMetricDouble(String.format("%s_ms", fullMetricName), metricValue, "ms"); 403 404 if (metricName.equals("wall_time")) { 405 // Calculate FPS 406 double wallTime = Double.parseDouble(metricValue); 407 assertTrue("wallTime should be bigger than 0", Double.compare(wallTime, 0.0) > 0); 408 double fps = 1000.0 / wallTime; 409 switch (driverType) { 410 case ANGLE: 411 mTracePerfANGLEFPS.put(testName, Double.valueOf(fps)); 412 break; 413 case NATIVE: 414 mTracePerfNativeFPS.put(testName, Double.valueOf(fps)); 415 break; 416 default: 417 fail("must specify either ANGLE or NATIVE as the driverType"); 418 } 419 420 // Log FPS in metrics, which will be saved as a tradefed result file later with 421 // mTestHelper.saveMetricsAsArtifact() 422 fullMetricName = String.format("%s.%s.fps", testName, renderer); 423 helper.logMetricDouble(fullMetricName, String.valueOf(fps), "fps"); 424 } 425 } 426 } 427 uninstallTestApps()428 private void uninstallTestApps() throws CommandException { 429 // Remove the existing ANGLE allowlist trace test apk from the device, if present. 430 mTestHelper.uninstallAppIgnoreErrors(ANGLE_TRACE_TEST_PACKAGE_NAME); 431 432 // Remove previous ANGLE trace data directory, if present 433 mTestHelper.adbShellCommandCheck(mTestHelper.WAIT_ADB_LARGE_FILE_OP_MILLIS, 434 String.format("rm -rf %s", ANGLE_TRACE_DATA_ON_DEVICE_DIR)); 435 436 // Remove the existing ANGLE allowlist driver check apk from the device, if present 437 mTestHelper.uninstallAppIgnoreErrors(AngleCommon.ANGLE_TEST_PKG); 438 } 439 isLowRamDevice(ITestDevice device)440 private boolean isLowRamDevice(ITestDevice device) throws Exception { 441 return "true".equals(device.getProperty("ro.config.low_ram")); 442 } 443 444 /** 445 * Check if device supports vulkan 1.1. 446 * If the device includes a Vulkan driver, feature list returned by 447 * "adb shell pm list features" should contain 448 * "feature:android.hardware.vulkan.level" (FEATURE_VULKAN_HARDWARE_LEVEL) and 449 * "feature:android.hardware.vulkan.version" (FEATURE_VULKAN_HARDWARE_VERSION) 450 * reference: https://source.android.com/docs/core/graphics/implement-vulkan 451 */ isVulkan11Supported(ITestDevice device)452 private boolean isVulkan11Supported(ITestDevice device) throws Exception { 453 final String features = device.executeShellCommand("pm list features"); 454 455 StringTokenizer featureToken = new StringTokenizer(features, "\n"); 456 457 boolean isVulkanLevelFeatureSupported = false; 458 459 boolean isVulkanVersionFeatureSupported = false; 460 461 boolean isVulkan_1_1_Supported = false; 462 463 while (featureToken.hasMoreTokens()) { 464 String currentFeature = featureToken.nextToken(); 465 466 // Check if currentFeature strings starts with "feature:android.hardware.vulkan.level" 467 // Check that currentFeature string length is at least the length of 468 // "feature:android.hardware.vulkan.level" before calling substring so that the endIndex 469 // is not out of bound. 470 if (currentFeature.length() >= VULKAN_LEVEL_FEATURE.length() 471 && currentFeature.substring(0, VULKAN_LEVEL_FEATURE.length()) 472 .equals(VULKAN_LEVEL_FEATURE)) { 473 isVulkanLevelFeatureSupported = true; 474 } 475 476 // Check if currentFeature strings starts with "feature:android.hardware.vulkan.version" 477 // Check that currentFeature string length is at least the length of 478 // "feature:android.hardware.vulkan.version" before calling substring so that the 479 // endIndex is not out of bound. 480 if (currentFeature.length() >= VULKAN_VERSION_FEATURE.length() 481 && currentFeature.substring(0, VULKAN_VERSION_FEATURE.length()) 482 .equals(VULKAN_VERSION_FEATURE)) { 483 isVulkanVersionFeatureSupported = true; 484 485 // If android.hardware.vulkan.version feature is supported by the device, 486 // check if the vulkan version supported is at least vulkan 1.1. 487 // ANGLE is only intended to work properly with vulkan version >= vulkan 1.1 488 String[] currentFeatureAndValue = currentFeature.split("="); 489 if (currentFeatureAndValue.length > 1) { 490 int vulkanVersionLevelSupported = Integer.parseInt(currentFeatureAndValue[1]); 491 isVulkan_1_1_Supported = vulkanVersionLevelSupported >= VULKAN_1_1; 492 } 493 } 494 495 if (isVulkanLevelFeatureSupported && isVulkanVersionFeatureSupported 496 && isVulkan_1_1_Supported) { 497 return true; 498 } 499 } 500 501 return false; 502 } 503 isVendorAPILevelMeetingA16Requirement(ITestDevice device)504 private boolean isVendorAPILevelMeetingA16Requirement(ITestDevice device) throws Exception { 505 if (mBypassVendorApiRequirement) { 506 return true; 507 } 508 final int vendorApiLevel = PropertyUtil.getVsrApiLevel(device); 509 return vendorApiLevel >= 202504; 510 } 511 verifyTraceList(List<String> traceNames)512 private void verifyTraceList(List<String> traceNames) { 513 Set<String> traceNamesSet = new HashSet<>(); 514 for (String traceName : traceNames) { 515 traceNamesSet.add(traceName); 516 } 517 for (String requiredAppName : AngleAllowlist.apps.values()) { 518 assertTrue(String.format("app %s must be included in the angle trace package", 519 requiredAppName), 520 traceNamesSet.contains(requiredAppName)); 521 } 522 } 523 524 @Before setUp()525 public void setUp() throws Exception { 526 // Instantiate a Helper object, which also calls Helper.preTestSetup() 527 // that sets the device ready for tests 528 mTestHelper = new Helper(getTestInformation(), mTemporaryFolder, mMetrics, mLogData, 529 mTestName.getMethodName()); 530 531 // Query current_user 532 final File cmdStdOutFile = new File(mTemporaryFolder.getRoot(), "cmdStdOut.txt"); 533 mCurrentUser = mTestHelper.adbShellCommandWithStdout( 534 mTestHelper.WAIT_SET_GLOBAL_SETTING_MILLIS, cmdStdOutFile, "am get-current-user"); 535 536 LogUtil.CLog.d("mCurrentUser is: %s", mCurrentUser); 537 538 mAngleTraceTestAppHomeDir = 539 String.format("/data/user/%s/com.android.angle.test", mCurrentUser); 540 mAngleTraceTestBlobCacheDir = 541 String.format("/data/user_de/%s/com.android.angle.test/cache", mCurrentUser); 542 543 setANGLETracePackagePath(); 544 545 uninstallTestApps(); 546 547 AngleCommon.clearSettings(getDevice()); 548 } 549 550 @After tearDown()551 public void tearDown() throws Exception { 552 uninstallTestApps(); 553 554 AngleCommon.clearSettings(getDevice()); 555 } 556 557 @VsrTest(requirements = {"VSR-5.1"}) 558 @Test testAngleTraces()559 public void testAngleTraces() throws Throwable { 560 Assume.assumeFalse(isLowRamDevice(getDevice())); 561 Assume.assumeFalse(FeatureUtil.isTV(getDevice())); 562 Assume.assumeTrue(isVulkan11Supported(getDevice())); 563 Assume.assumeTrue(isVendorAPILevelMeetingA16Requirement(getDevice())); 564 // Firstly check ANGLE is available in System Partition 565 // Install driver check app 566 installTestApp(AngleCommon.ANGLE_TEST_APP); 567 // Verify ANGLE is available in system partition 568 runDeviceTests(AngleCommon.ANGLE_TEST_PKG, 569 AngleCommon.ANGLE_TEST_PKG + "." + AngleCommon.ANGLE_DRIVER_TEST_CLASS, 570 AngleCommon.ANGLE_DRIVER_TEST_LOCATION_METHOD); 571 572 // Secondly run trace tests with System ANGLE 573 // We will copy the stdout file content from the device to here. 574 final File gtestStdoutFile = new File(mTemporaryFolder.getRoot(), "out.txt"); 575 576 try { 577 LogUtil.CLog.d("Installing angle trace app and pushing trace data to the device."); 578 579 // Create trace data directory on the device. 580 mTestHelper.deviceMkDirP(ANGLE_TRACE_DATA_ON_DEVICE_DIR); 581 582 final File angleTraceTestPackage = new File(mANGLETracePackagePath); 583 584 // Install the ANGLE APK. 585 final File angleApkFile = mTestHelper.path(angleTraceTestPackage, "out", 586 "AndroidPerformance", "angle_trace_tests_apk", "angle_trace_tests-debug.apk"); 587 588 mTestHelper.installApkFile(angleApkFile); 589 590 // grant test apk permissions 591 mTestHelper.adbShellCommandCheck(mTestHelper.WAIT_ADB_SHELL_FILE_OP_MILLIS, 592 String.format("appops set %s MANAGE_EXTERNAL_STORAGE allow || true", 593 ANGLE_TRACE_TEST_PACKAGE_NAME)); 594 595 // Push trace_list.json 596 final File angleTraceListJson = mTestHelper.path( 597 angleTraceTestPackage, "out", "AndroidPerformance", "gen", "trace_list.json"); 598 mTestHelper.adbCommandCheck(mTestHelper.WAIT_ADB_SHELL_FILE_OP_MILLIS, "push", 599 angleTraceListJson.toString(), 600 String.format("%s/gen/trace_list.json", ANGLE_TRACE_DATA_ON_DEVICE_DIR)); 601 602 // Create a src/tests/restricted_traces directory on test device, this is required in 603 // order for angle_trace_tests app process to launch successfully 604 mTestHelper.deviceMkDirP(String.format( 605 "%s/src/tests/restricted_traces", ANGLE_TRACE_DATA_ON_DEVICE_DIR)); 606 607 // Launch angle_trace_tests app with --list-test argument to get the list of trace names 608 List<String> traceNames = runAngleListTrace(mTestHelper, gtestStdoutFile); 609 610 // Verify the traces in angle_traces package contains all required ANGLE allowlist apps 611 verifyTraceList(traceNames); 612 613 // Delete angle_debug_package global settings so that when trace is set to run 614 // with DriverType.ANGLE, trace will use system ANGLE, not ANGLE debug apk. 615 mTestHelper.adbShellCommandCheck(mTestHelper.WAIT_SET_GLOBAL_SETTING_MILLIS, 616 "settings delete global angle_debug_package"); 617 618 // Run all the trace test of apps required on ANGLE allowlist. 619 for (final String traceName : AngleAllowlist.apps.values()) { 620 // push the "<traceName>.json" onto the device 621 String traceJsonFileName = String.format("%s.json", traceName); 622 final File traceJsonFile = mTestHelper.path(angleTraceTestPackage, "src", "tests", 623 "restricted_traces", traceName, traceJsonFileName); 624 mTestHelper.adbCommandCheck(mTestHelper.WAIT_ADB_LARGE_FILE_OP_MILLIS, "push", 625 traceJsonFile.toString(), 626 String.format("%s/src/tests/restricted_traces/%s/%s", 627 ANGLE_TRACE_DATA_ON_DEVICE_DIR, traceName, traceJsonFileName)); 628 629 // push the "<traceName>.angledata.gz" file onto the device 630 String traceDataFileName = String.format("%s.angledata.gz", traceName); 631 final File traceDataFile = mTestHelper.path(angleTraceTestPackage, "src", "tests", 632 "restricted_traces", traceName, traceDataFileName); 633 mTestHelper.adbCommandCheck(mTestHelper.WAIT_ADB_LARGE_FILE_OP_MILLIS, "push", 634 traceDataFile.toString(), 635 String.format("%s/src/tests/restricted_traces/%s/%s", 636 ANGLE_TRACE_DATA_ON_DEVICE_DIR, traceName, traceDataFileName)); 637 638 // Run trace test on angle until either of below conditions is met: 639 // 1) trace reaches FPS_REQUIREMENT fps 640 // 2) trace is ran MAX_TRACE_RUN_COUNT times 641 Double currentTraceAngleFPS = runAngleTracePerfMultiTimes(traceName, 642 gtestStdoutFile, mTestHelper, DriverType.ANGLE, MAX_TRACE_RUN_COUNT); 643 644 // If trace fails to reach FPS_REQUIREMENT fps on ANGLE, run trace on native driver, 645 // too. If trace also fails to reach FPS_REQUIREMENT fps on native, we treat this 646 // trace test passes on ANGLE, as ANGLE doesn't make the trace perform worse. 647 if (Double.compare(currentTraceAngleFPS.doubleValue(), FPS_REQUIREMENT) < 0) { 648 mTracePerfANGLEBelowRequiredFPS.add(traceName); 649 runAngleTracePerfMultiTimes(traceName, gtestStdoutFile, mTestHelper, 650 DriverType.NATIVE, MAX_TRACE_RUN_COUNT); 651 } 652 } 653 654 // Check all required traces completed successfully 655 assertTrue(String.format("Not all required traces are ran, traces that are skipped: %s", 656 mSkippedTrace.toString()), 657 mTracePerfANGLEFPS.size() == AngleAllowlist.apps.size()); 658 659 // Check trace test result 660 Set<String> failedTraceList = new HashSet<String>(); 661 for (String traceName : mTracePerfANGLEBelowRequiredFPS) { 662 final boolean isNativeTraceDataAvailable = 663 mTracePerfNativeFPS.containsKey(traceName); 664 assertTrue(String.format( 665 "trace %s runs slower than %f fps on ANGLE, we expect the trace" 666 + " to also execute on native driver, but there is no " 667 + "data from native driver run", 668 traceName, FPS_REQUIREMENT), 669 isNativeTraceDataAvailable); 670 Double nativeFps = mTracePerfNativeFPS.get(traceName); 671 boolean nativeFpsReachesRequiredFPS = 672 Double.compare(nativeFps.doubleValue(), FPS_REQUIREMENT) >= 0; 673 if (!nativeFpsReachesRequiredFPS) { 674 LogUtil.CLog.d( 675 "trace %s doesn't reach %f FPS on both ANGLE and NATIVE GLES driver", 676 traceName, FPS_REQUIREMENT); 677 } else { 678 Double angleFps = mTracePerfANGLEFPS.get(traceName); 679 if (angleFps < FPS_REQUIREMENT * FPS_THRESHOLD) { 680 failedTraceList.add(traceName); 681 } 682 } 683 } 684 if (!failedTraceList.isEmpty()) { 685 for (String failedTraceName : failedTraceList) { 686 LogUtil.CLog.e("trace %s reaches %f FPS on NATIVE, native fps: %f, but fails " 687 + "on ANGLE, angle fps: %f, and FPS on ANGLE is less than %f " 688 + "of %f", 689 failedTraceName, FPS_REQUIREMENT, 690 mTracePerfNativeFPS.get(failedTraceName), 691 mTracePerfANGLEFPS.get(failedTraceName), FPS_THRESHOLD, 692 FPS_REQUIREMENT); 693 } 694 fail(String.format("There are traces that reaches %f FPS on NATIVE, but fails to " 695 + "reach %f FPS on ANGLE: %s, and FPS on ANGLE is less than %f " 696 + "of %f", 697 FPS_REQUIREMENT, FPS_REQUIREMENT, failedTraceList.toString(), FPS_THRESHOLD, 698 FPS_REQUIREMENT)); 699 } 700 } finally { 701 mTestHelper.saveMetricsAsArtifact(); 702 } 703 } 704 } 705