1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.drawelements.deqp.runner; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.compatibility.common.tradefed.targetprep.IncrementalDeqpPreparer; 20 import com.android.ddmlib.IDevice; 21 import com.android.ddmlib.IShellOutputReceiver; 22 import com.android.tradefed.build.IFolderBuildInfo; 23 import com.android.tradefed.config.ConfigurationException; 24 import com.android.tradefed.config.OptionSetter; 25 import com.android.tradefed.device.DeviceNotAvailableException; 26 import com.android.tradefed.device.IManagedTestDevice; 27 import com.android.tradefed.device.ITestDevice; 28 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 29 import com.android.tradefed.result.ITestInvocationListener; 30 import com.android.tradefed.result.TestDescription; 31 import com.android.tradefed.result.error.InfraErrorIdentifier; 32 import com.android.tradefed.testtype.Abi; 33 import com.android.tradefed.testtype.IAbi; 34 import com.android.tradefed.testtype.IRemoteTest; 35 import com.android.tradefed.testtype.IRuntimeHintProvider; 36 import com.android.tradefed.util.AbiUtils; 37 import com.android.tradefed.util.FileUtil; 38 import com.android.tradefed.util.IRunUtil; 39 import com.android.tradefed.util.RunInterruptedException; 40 import java.io.File; 41 import java.io.FileNotFoundException; 42 import java.io.FileWriter; 43 import java.io.IOException; 44 import java.io.PrintWriter; 45 import java.io.StringWriter; 46 import java.util.ArrayList; 47 import java.util.Collection; 48 import java.util.Collections; 49 import java.util.HashMap; 50 import java.util.HashSet; 51 import java.util.List; 52 import java.util.Map; 53 import java.util.Set; 54 import java.util.concurrent.TimeUnit; 55 import junit.framework.TestCase; 56 import org.easymock.EasyMock; 57 import org.easymock.IAnswer; 58 import org.easymock.IMocksControl; 59 60 /** 61 * Unit tests for {@link DeqpTestRunner}. 62 */ 63 public class DeqpTestRunnerTest extends TestCase { 64 private static final IAbi ABI = new Abi("armeabi-v7a", "32"); 65 private static final String APP_DIR = "/sdcard/"; 66 private static final String CASE_LIST_FILE_NAME = "dEQP-TestCaseList.txt"; 67 private static final String LOG_FILE_NAME = "TestLog.qpa"; 68 private static final String INSTRUMENTATION_NAME = 69 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation"; 70 private static final String QUERY_INSTRUMENTATION_NAME = 71 "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation"; 72 private static final String ONLY_LANDSCAPE_FEATURES = 73 "feature:" + DeqpTestRunner.FEATURE_LANDSCAPE; 74 private static final String ALL_FEATURES = 75 ONLY_LANDSCAPE_FEATURES + 76 "\nfeature:" + DeqpTestRunner.FEATURE_PORTRAIT; 77 private static List<Map<String, String>> DEFAULT_INSTANCE_ARGS; 78 79 private static String ASSUMPTION_FAILURE_MESSAGE = "Assumption Failure"; 80 private static String PASS_MESSAGE = "Pass"; 81 82 static { 83 DEFAULT_INSTANCE_ARGS = new ArrayList<>(1); DEFAULT_INSTANCE_ARGS.add(new HashMap<String, String>())84 DEFAULT_INSTANCE_ARGS.add(new HashMap<String, String>()); 85 DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", 86 "rgba8888d24s8"); 87 DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified"); 88 DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window"); 89 } 90 91 private File mTestsDir = null; 92 93 public static class BuildHelperMock extends CompatibilityBuildHelper { 94 private File mTestsDir = null; BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir)95 public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) { 96 super(buildInfo); 97 mTestsDir = testsDir; 98 } 99 @Override getTestsDir()100 public File getTestsDir() throws FileNotFoundException { 101 return mTestsDir; 102 } 103 } 104 105 /** 106 * {@inheritDoc} 107 */ 108 @Override setUp()109 protected void setUp() throws Exception { 110 super.setUp(); 111 mTestsDir = FileUtil.createTempDir("deqp-test-cases"); 112 } 113 114 /** 115 * {@inheritDoc} 116 */ 117 @Override tearDown()118 protected void tearDown() throws Exception { 119 FileUtil.recursiveDelete(mTestsDir); 120 super.tearDown(); 121 } 122 123 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, int minorVersion, Collection<TestDescription> tests, File testsDir)124 buildGlesTestRunner(int majorVersion, int minorVersion, 125 Collection<TestDescription> tests, File testsDir) 126 throws ConfigurationException, IOException { 127 return buildGlesTestRunner(majorVersion, minorVersion, tests, testsDir, false, 128 new ArrayList<TestDescription>()); 129 } 130 131 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, int minorVersion, Collection<TestDescription> tests, File testsDir, boolean incrementalDeqpEnabled, Collection<TestDescription> incrementalTests)132 buildGlesTestRunner(int majorVersion, int minorVersion, Collection<TestDescription> tests, 133 File testsDir, boolean incrementalDeqpEnabled, 134 Collection<TestDescription> incrementalTests) 135 throws ConfigurationException, IOException { 136 137 StringWriter testlist = new StringWriter(); 138 for (TestDescription test : tests) { 139 testlist.write(test.getClassName() + "." + test.getTestName() + "\n"); 140 } 141 StringWriter incrementalTestlist = new StringWriter(); 142 for (TestDescription test : incrementalTests) { 143 incrementalTestlist.write(test.getClassName() + "." + test.getTestName() + "\n"); 144 } 145 return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString(), testsDir, 146 incrementalDeqpEnabled, incrementalTestlist.toString()); 147 } 148 getMockBuildHelper(File testsDir)149 private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) { 150 IFolderBuildInfo mockIFolderBuildInfo = 151 EasyMock.createMock(IFolderBuildInfo.class); 152 EasyMock.expect(mockIFolderBuildInfo.getBuildAttributes()) 153 .andReturn(new HashMap<>()) 154 .anyTimes(); 155 EasyMock.replay(mockIFolderBuildInfo); 156 return new BuildHelperMock(mockIFolderBuildInfo, testsDir); 157 } 158 159 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, int minorVersion, String testlist, File testsDir, boolean incrementalDeqpEnabled, String incrementalTestlist)160 buildGlesTestRunner(int majorVersion, int minorVersion, String testlist, File testsDir, 161 boolean incrementalDeqpEnabled, String incrementalTestlist) 162 throws ConfigurationException, IOException { 163 164 DeqpTestRunner runner = new DeqpTestRunner(); 165 OptionSetter setter = new OptionSetter(runner); 166 167 String deqpPackage = 168 "dEQP-GLES" + majorVersion + 169 (minorVersion > 0 ? Integer.toString(minorVersion) : ""); 170 171 final File caselistsFile = new File(testsDir, "gles3-caselist.txt"); 172 FileUtil.writeToFile(testlist, caselistsFile); 173 174 if (!incrementalTestlist.isEmpty()) { 175 final File incrementalCaselistsFile = new File(testsDir, 176 "gles3-incremental-caselist.txt"); 177 FileUtil.writeToFile(incrementalTestlist, incrementalCaselistsFile); 178 setter.setOptionValue( 179 "incremental-deqp-include-file", incrementalCaselistsFile.getName()); 180 } 181 182 setter.setOptionValue("deqp-package", deqpPackage); 183 setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8"); 184 setter.setOptionValue("deqp-caselist-file", caselistsFile.getName()); 185 setter.setOptionValue("deqp-screen-rotation", "unspecified"); 186 setter.setOptionValue("deqp-surface-type", "window"); 187 setter.setOptionValue("enable-incremental-deqp", String.valueOf(incrementalDeqpEnabled)); 188 runner.setAbi(ABI); 189 runner.setBuildHelper(getMockBuildHelper(testsDir)); 190 191 return runner; 192 } 193 getTestId(DeqpTestRunner runner)194 private static String getTestId(DeqpTestRunner runner) { 195 return AbiUtils.createId(ABI.getName(), runner.getPackageName()); 196 } 197 198 /** 199 * Test version of OpenGL ES. 200 */ testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion, String resultCode)201 private void testGlesVersion(int requiredMajorVersion, 202 int requiredMinorVersion, int majorVersion, 203 int minorVersion, String resultCode) 204 throws Exception { 205 final TestDescription testId = new TestDescription( 206 "dEQP-GLES" + Integer.toString(requiredMajorVersion) + 207 Integer.toString(requiredMinorVersion) + ".info", 208 "version"); 209 210 final String testPath = 211 "dEQP-GLES" + Integer.toString(requiredMajorVersion) + 212 Integer.toString(requiredMinorVersion) + ".info.version"; 213 214 final String testTrie = 215 "{dEQP-GLES" + Integer.toString(requiredMajorVersion) + 216 Integer.toString(requiredMinorVersion) + "{info{version}}}"; 217 218 String resultDetails = resultCode; 219 if (resultCode.equals(ASSUMPTION_FAILURE_MESSAGE)) { 220 resultDetails = 221 DeqpTestRunner.ASSUMPTION_FAILURE_DEQP_LEVEL_LOG_MESSAGE; 222 } 223 224 /* MultiLineReceiver expects "\r\n" line ending. */ 225 final String output = 226 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 227 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 228 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 229 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 230 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 231 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 232 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 233 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 234 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 235 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 236 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 237 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 238 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 239 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 240 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 241 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + 242 testPath + "\r\n" 243 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 244 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + 245 "\r\n" 246 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + 247 resultDetails + "\r\n" 248 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 249 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 250 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 251 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 252 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 253 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 254 + "INSTRUMENTATION_CODE: 0\r\n"; 255 256 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 257 ITestInvocationListener mockListener = 258 EasyMock.createStrictMock(ITestInvocationListener.class); 259 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 260 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 261 tests.add(testId); 262 263 DeqpTestRunner deqpTest = buildGlesTestRunner( 264 requiredMajorVersion, requiredMinorVersion, tests, mTestsDir); 265 266 int version = (majorVersion << 16) | minorVersion; 267 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 268 .andReturn(Integer.toString(version)) 269 .atLeastOnce(); 270 271 if (majorVersion > requiredMajorVersion || 272 (majorVersion == requiredMajorVersion && 273 minorVersion >= requiredMinorVersion)) { 274 275 // Expect the calls twice: setupTestEnvironment() and 276 // teardownTestEnvironment() 277 EasyMock 278 .expect(mockDevice.executeShellCommand(EasyMock.eq( 279 "settings delete global angle_gl_driver_selection_pkgs"))) 280 .andReturn("") 281 .once(); 282 EasyMock 283 .expect(mockDevice.executeShellCommand(EasyMock.eq( 284 "settings delete global angle_gl_driver_selection_values"))) 285 .andReturn("") 286 .once(); 287 EasyMock 288 .expect(mockDevice.executeShellCommand(EasyMock.eq( 289 "settings delete global angle_gl_driver_selection_pkgs"))) 290 .andReturn("") 291 .once(); 292 EasyMock 293 .expect(mockDevice.executeShellCommand(EasyMock.eq( 294 "settings delete global angle_gl_driver_selection_values"))) 295 .andReturn("") 296 .once(); 297 298 expectRenderConfigQuery(mockDevice, requiredMajorVersion, 299 requiredMinorVersion); 300 301 String commandLine = String.format( 302 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 303 + "--deqp-screen-rotation=unspecified " 304 + "--deqp-surface-type=window " 305 + "--deqp-log-images=disable " 306 + "--deqp-watchdog=enable", 307 APP_DIR + CASE_LIST_FILE_NAME); 308 309 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, 310 commandLine, output); 311 } 312 313 mockListener.testRunStarted(getTestId(deqpTest), 1); 314 EasyMock.expectLastCall().once(); 315 316 mockListener.testStarted(EasyMock.eq(testId)); 317 EasyMock.expectLastCall().once(); 318 319 if (resultCode.equals(ASSUMPTION_FAILURE_MESSAGE)) { 320 mockListener.testAssumptionFailure( 321 EasyMock.eq(testId), 322 EasyMock.eq( 323 DeqpTestRunner.ASSUMPTION_FAILURE_DEQP_LEVEL_LOG_MESSAGE)); 324 EasyMock.expectLastCall().once(); 325 } 326 327 mockListener.testEnded(EasyMock.eq(testId), 328 EasyMock.<HashMap<String, Metric>>notNull()); 329 EasyMock.expectLastCall().once(); 330 331 mockListener.testRunEnded(EasyMock.anyLong(), 332 EasyMock.<HashMap<String, Metric>>notNull()); 333 EasyMock.expectLastCall().once(); 334 335 EasyMock.replay(mockDevice, mockIDevice); 336 EasyMock.replay(mockListener); 337 338 deqpTest.setDevice(mockDevice); 339 deqpTest.run(mockListener); 340 341 EasyMock.verify(mockListener); 342 EasyMock.verify(mockDevice, mockIDevice); 343 } 344 expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, int minorVersion)345 private void expectRenderConfigQuery(ITestDevice mockDevice, 346 int majorVersion, int minorVersion) 347 throws Exception { 348 expectRenderConfigQuery( 349 mockDevice, 350 String.format("--deqp-gl-config-name=rgba8888d24s8 " 351 + "--deqp-screen-rotation=unspecified " 352 + "--deqp-surface-type=window " 353 + "--deqp-gl-major-version=%d " 354 + "--deqp-gl-minor-version=%d", 355 majorVersion, minorVersion)); 356 } 357 expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)358 private void expectRenderConfigQuery(ITestDevice mockDevice, 359 String commandLine) throws Exception { 360 expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes"); 361 } 362 expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, String output)363 private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, 364 String commandLine, 365 String output) 366 throws Exception { 367 final String queryOutput = 368 "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n" 369 + "INSTRUMENTATION_CODE: 0\r\n"; 370 final String command = String.format( 371 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine " 372 + "\"%s\" %s", 373 AbiUtils.createAbiFlag(ABI.getName()), commandLine, 374 QUERY_INSTRUMENTATION_NAME); 375 376 mockDevice.executeShellCommand( 377 EasyMock.eq(command), EasyMock.<IShellOutputReceiver>notNull()); 378 379 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 380 @Override 381 public Object answer() { 382 IShellOutputReceiver receiver = 383 (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 384 385 receiver.addOutput(queryOutput.getBytes(), 0, 386 queryOutput.length()); 387 receiver.flush(); 388 389 return null; 390 } 391 }); 392 } 393 394 /** 395 * Test that result code produces correctly pass or fail. 396 */ testResultCode(final String resultCode, boolean pass)397 private void testResultCode(final String resultCode, boolean pass) 398 throws Exception { 399 final TestDescription testId = 400 new TestDescription("dEQP-GLES3.info", "version"); 401 final String testPath = "dEQP-GLES3.info.version"; 402 final String testTrie = "{dEQP-GLES3{info{version}}}"; 403 404 /* MultiLineReceiver expects "\r\n" line ending. */ 405 final String output = 406 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 407 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 408 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 409 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 410 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 411 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 412 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 413 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 414 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 415 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 416 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 417 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 418 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 419 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 420 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 421 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + 422 testPath + "\r\n" 423 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 424 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + 425 "\r\n" 426 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + 427 resultCode + "\r\n" 428 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 429 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 430 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 431 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 432 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 433 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 434 + "INSTRUMENTATION_CODE: 0\r\n"; 435 436 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 437 ITestInvocationListener mockListener = 438 EasyMock.createStrictMock(ITestInvocationListener.class); 439 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 440 441 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 442 tests.add(testId); 443 444 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 445 446 int version = 3 << 16; 447 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 448 .andReturn(Integer.toString(version)) 449 .atLeastOnce(); 450 451 expectRenderConfigQuery(mockDevice, 3, 0); 452 453 String commandLine = String.format( 454 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 455 + "--deqp-screen-rotation=unspecified " 456 + "--deqp-surface-type=window " 457 + "--deqp-log-images=disable " 458 + "--deqp-watchdog=enable", 459 APP_DIR + CASE_LIST_FILE_NAME); 460 461 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, 462 commandLine, output); 463 464 mockListener.testRunStarted(getTestId(deqpTest), 1); 465 EasyMock.expectLastCall().once(); 466 467 // Expect the calls twice: setupTestEnvironment() and 468 // teardownTestEnvironment() 469 EasyMock 470 .expect(mockDevice.executeShellCommand(EasyMock.eq( 471 "settings delete global angle_gl_driver_selection_pkgs"))) 472 .andReturn("") 473 .once(); 474 EasyMock 475 .expect(mockDevice.executeShellCommand(EasyMock.eq( 476 "settings delete global angle_gl_driver_selection_values"))) 477 .andReturn("") 478 .once(); 479 EasyMock 480 .expect(mockDevice.executeShellCommand(EasyMock.eq( 481 "settings delete global angle_gl_driver_selection_pkgs"))) 482 .andReturn("") 483 .once(); 484 EasyMock 485 .expect(mockDevice.executeShellCommand(EasyMock.eq( 486 "settings delete global angle_gl_driver_selection_values"))) 487 .andReturn("") 488 .once(); 489 490 mockListener.testStarted(EasyMock.eq(testId)); 491 EasyMock.expectLastCall().once(); 492 493 if (!pass) { 494 mockListener.testFailed( 495 testId, 496 "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n" + 497 resultCode + ": Detail" + resultCode); 498 499 EasyMock.expectLastCall().once(); 500 } 501 502 mockListener.testEnded(EasyMock.eq(testId), 503 EasyMock.<HashMap<String, Metric>>notNull()); 504 EasyMock.expectLastCall().once(); 505 506 mockListener.testRunEnded(EasyMock.anyLong(), 507 EasyMock.<HashMap<String, Metric>>notNull()); 508 EasyMock.expectLastCall().once(); 509 510 EasyMock.replay(mockDevice, mockIDevice); 511 EasyMock.replay(mockListener); 512 513 deqpTest.setDevice(mockDevice); 514 deqpTest.run(mockListener); 515 516 EasyMock.verify(mockListener); 517 EasyMock.verify(mockDevice, mockIDevice); 518 } 519 520 /** 521 * Test running multiple test cases. 522 */ testRun_multipleTests()523 public void testRun_multipleTests() throws Exception { 524 /* MultiLineReceiver expects "\r\n" line ending. */ 525 final String output = 526 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 527 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 528 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 529 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 530 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 531 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 532 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 533 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 534 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 535 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 536 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 537 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 538 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 539 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 540 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 541 + 542 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 543 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 544 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 545 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 546 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 547 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 548 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 549 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 550 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 551 + 552 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 553 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 554 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 555 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 556 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 557 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 558 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 559 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 560 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 561 + 562 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 563 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 564 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 565 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 566 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 567 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 568 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 569 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 570 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 571 + 572 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 573 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 574 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 575 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 576 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 577 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 578 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 579 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 580 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 581 + 582 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 583 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 584 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 585 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 586 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 587 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 588 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 589 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 590 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 591 + 592 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 593 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 594 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 595 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 596 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 597 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 598 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 599 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 600 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 601 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 602 + "INSTRUMENTATION_CODE: 0\r\n"; 603 604 final TestDescription[] testIds = { 605 new TestDescription("dEQP-GLES3.info", "vendor"), 606 new TestDescription("dEQP-GLES3.info", "renderer"), 607 new TestDescription("dEQP-GLES3.info", "version"), 608 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 609 new TestDescription("dEQP-GLES3.info", "extensions"), 610 new TestDescription("dEQP-GLES3.info", "render_target")}; 611 612 final String[] testPaths = {"dEQP-GLES3.info.vendor", 613 "dEQP-GLES3.info.renderer", 614 "dEQP-GLES3.info.version", 615 "dEQP-GLES3.info.shading_language_version", 616 "dEQP-GLES3.info.extensions", 617 "dEQP-GLES3.info.render_target"}; 618 619 final String testTrie = 620 "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 621 622 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 623 ITestInvocationListener mockListener = 624 EasyMock.createStrictMock(ITestInvocationListener.class); 625 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 626 627 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 628 629 for (TestDescription id : testIds) { 630 tests.add(id); 631 } 632 633 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 634 635 int version = 3 << 16; 636 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 637 .andReturn(Integer.toString(version)) 638 .atLeastOnce(); 639 640 expectRenderConfigQuery(mockDevice, 3, 0); 641 642 String commandLine = String.format( 643 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 644 + "--deqp-screen-rotation=unspecified " 645 + "--deqp-surface-type=window " 646 + "--deqp-log-images=disable " 647 + "--deqp-watchdog=enable", 648 APP_DIR + CASE_LIST_FILE_NAME); 649 650 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, 651 commandLine, output); 652 653 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 654 EasyMock.expectLastCall().once(); 655 656 // Expect the calls twice: setupTestEnvironment() and 657 // teardownTestEnvironment() 658 EasyMock 659 .expect(mockDevice.executeShellCommand(EasyMock.eq( 660 "settings delete global angle_gl_driver_selection_pkgs"))) 661 .andReturn("") 662 .once(); 663 EasyMock 664 .expect(mockDevice.executeShellCommand(EasyMock.eq( 665 "settings delete global angle_gl_driver_selection_values"))) 666 .andReturn("") 667 .once(); 668 EasyMock 669 .expect(mockDevice.executeShellCommand(EasyMock.eq( 670 "settings delete global angle_gl_driver_selection_pkgs"))) 671 .andReturn("") 672 .once(); 673 EasyMock 674 .expect(mockDevice.executeShellCommand(EasyMock.eq( 675 "settings delete global angle_gl_driver_selection_values"))) 676 .andReturn("") 677 .once(); 678 679 for (int i = 0; i < testPaths.length; i++) { 680 mockListener.testStarted(EasyMock.eq(testIds[i])); 681 EasyMock.expectLastCall().once(); 682 683 mockListener.testEnded(EasyMock.eq(testIds[i]), 684 EasyMock.<HashMap<String, Metric>>notNull()); 685 686 EasyMock.expectLastCall().once(); 687 } 688 689 mockListener.testRunEnded(EasyMock.anyLong(), 690 EasyMock.<HashMap<String, Metric>>notNull()); 691 EasyMock.expectLastCall().once(); 692 693 EasyMock.replay(mockDevice, mockIDevice); 694 EasyMock.replay(mockListener); 695 696 deqpTest.setDevice(mockDevice); 697 deqpTest.run(mockListener); 698 699 EasyMock.verify(mockListener); 700 EasyMock.verify(mockDevice, mockIDevice); 701 } 702 buildTestProcessOutput(List<TestDescription> tests)703 static private String buildTestProcessOutput(List<TestDescription> tests) { 704 /* MultiLineReceiver expects "\r\n" line ending. */ 705 final String outputHeader = 706 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 707 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 708 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 709 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 710 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 711 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 712 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 713 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 714 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 715 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 716 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 717 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 718 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 719 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; 720 721 final String outputEnd = 722 "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 723 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 724 + "INSTRUMENTATION_CODE: 0\r\n"; 725 726 StringWriter output = new StringWriter(); 727 output.write(outputHeader); 728 for (TestDescription test : tests) { 729 output.write( 730 "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"); 731 output.write( 732 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath="); 733 output.write(test.getClassName()); 734 output.write("."); 735 output.write(test.getTestName()); 736 output.write("\r\n"); 737 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 738 output.write( 739 "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"); 740 output.write( 741 "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"); 742 output.write( 743 "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"); 744 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 745 output.write( 746 "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"); 747 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 748 } 749 output.write(outputEnd); 750 return output.toString(); 751 } 752 testFiltering(DeqpTestRunner deqpTest, String expectedTrie, List<TestDescription> expectedTests)753 private void testFiltering(DeqpTestRunner deqpTest, String expectedTrie, 754 List<TestDescription> expectedTests) 755 throws Exception { 756 int version = 3 << 16; 757 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 758 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 759 .andReturn(Integer.toString(version)) 760 .atLeastOnce(); 761 762 boolean thereAreTests = !expectedTests.isEmpty(); 763 764 ITestInvocationListener mockListener = 765 EasyMock.createStrictMock(ITestInvocationListener.class); 766 mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size()); 767 EasyMock.expectLastCall().once(); 768 769 // Expect the calls twice: setupTestEnvironment() and 770 // teardownTestEnvironment() 771 EasyMock 772 .expect(mockDevice.executeShellCommand(EasyMock.eq( 773 "settings delete global angle_gl_driver_selection_pkgs"))) 774 .andReturn("") 775 .once(); 776 EasyMock 777 .expect(mockDevice.executeShellCommand(EasyMock.eq( 778 "settings delete global angle_gl_driver_selection_values"))) 779 .andReturn("") 780 .once(); 781 EasyMock 782 .expect(mockDevice.executeShellCommand(EasyMock.eq( 783 "settings delete global angle_gl_driver_selection_pkgs"))) 784 .andReturn("") 785 .once(); 786 EasyMock 787 .expect(mockDevice.executeShellCommand(EasyMock.eq( 788 "settings delete global angle_gl_driver_selection_values"))) 789 .andReturn("") 790 .once(); 791 792 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 793 if (thereAreTests) { 794 expectRenderConfigQuery(mockDevice, 3, 0); 795 796 String testOut = buildTestProcessOutput(expectedTests); 797 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); 798 799 for (int i = 0; i < expectedTests.size(); i++) { 800 mockListener.testStarted(EasyMock.eq(expectedTests.get(i))); 801 EasyMock.expectLastCall().once(); 802 803 mockListener.testEnded( 804 EasyMock.eq(expectedTests.get(i)), 805 EasyMock.<HashMap<String, Metric>>notNull()); 806 807 EasyMock.expectLastCall().once(); 808 } 809 } 810 811 mockListener.testRunEnded(EasyMock.anyLong(), 812 EasyMock.<HashMap<String, Metric>>notNull()); 813 EasyMock.expectLastCall().once(); 814 815 EasyMock.replay(mockDevice, mockIDevice); 816 EasyMock.replay(mockListener); 817 818 deqpTest.setDevice(mockDevice); 819 deqpTest.run(mockListener); 820 821 EasyMock.verify(mockListener); 822 EasyMock.verify(mockDevice, mockIDevice); 823 } 824 testRun_trivialIncludeFilter()825 public void testRun_trivialIncludeFilter() throws Exception { 826 final TestDescription[] testIds = { 827 new TestDescription("dEQP-GLES3.missing", "no"), 828 new TestDescription("dEQP-GLES3.missing", "nope"), 829 new TestDescription("dEQP-GLES3.missing", "donotwant"), 830 new TestDescription("dEQP-GLES3.pick_me", "yes"), 831 new TestDescription("dEQP-GLES3.pick_me", "ok"), 832 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 833 }; 834 835 List<TestDescription> allTests = new ArrayList<TestDescription>(); 836 for (TestDescription id : testIds) { 837 allTests.add(id); 838 } 839 840 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 841 activeTests.add(testIds[3]); 842 activeTests.add(testIds[4]); 843 activeTests.add(testIds[5]); 844 845 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 846 847 DeqpTestRunner deqpTest = 848 buildGlesTestRunner(3, 0, allTests, mTestsDir); 849 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me#*"); 850 testFiltering(deqpTest, expectedTrie, activeTests); 851 } 852 testRun_trivialExcludeFilter()853 public void testRun_trivialExcludeFilter() throws Exception { 854 final TestDescription[] testIds = { 855 new TestDescription("dEQP-GLES3.missing", "no"), 856 new TestDescription("dEQP-GLES3.missing", "nope"), 857 new TestDescription("dEQP-GLES3.missing", "donotwant"), 858 new TestDescription("dEQP-GLES3.pick_me", "yes"), 859 new TestDescription("dEQP-GLES3.pick_me", "ok"), 860 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 861 }; 862 863 List<TestDescription> allTests = new ArrayList<TestDescription>(); 864 for (TestDescription id : testIds) { 865 allTests.add(id); 866 } 867 868 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 869 activeTests.add(testIds[3]); 870 activeTests.add(testIds[4]); 871 activeTests.add(testIds[5]); 872 873 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 874 875 DeqpTestRunner deqpTest = 876 buildGlesTestRunner(3, 0, allTests, mTestsDir); 877 deqpTest.addExcludeFilter("dEQP-GLES3.missing#*"); 878 testFiltering(deqpTest, expectedTrie, activeTests); 879 } 880 testRun_includeAndExcludeFilter()881 public void testRun_includeAndExcludeFilter() throws Exception { 882 final TestDescription[] testIds = { 883 new TestDescription("dEQP-GLES3.group1", "foo"), 884 new TestDescription("dEQP-GLES3.group1", "nope"), 885 new TestDescription("dEQP-GLES3.group1", "donotwant"), 886 new TestDescription("dEQP-GLES3.group2", "foo"), 887 new TestDescription("dEQP-GLES3.group2", "yes"), 888 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"), 889 }; 890 891 List<TestDescription> allTests = new ArrayList<TestDescription>(); 892 for (TestDescription id : testIds) { 893 allTests.add(id); 894 } 895 896 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 897 activeTests.add(testIds[4]); 898 899 String expectedTrie = "{dEQP-GLES3{group2{yes}}}"; 900 901 DeqpTestRunner deqpTest = 902 buildGlesTestRunner(3, 0, allTests, mTestsDir); 903 904 Set<String> includes = new HashSet<>(); 905 includes.add("dEQP-GLES3.group2#*"); 906 deqpTest.addAllIncludeFilters(includes); 907 908 Set<String> excludes = new HashSet<>(); 909 excludes.add("*foo"); 910 excludes.add("*thoushallnotpass"); 911 deqpTest.addAllExcludeFilters(excludes); 912 testFiltering(deqpTest, expectedTrie, activeTests); 913 } 914 testRun_includeAll()915 public void testRun_includeAll() throws Exception { 916 final TestDescription[] testIds = { 917 new TestDescription("dEQP-GLES3.group1", "mememe"), 918 new TestDescription("dEQP-GLES3.group1", "yeah"), 919 new TestDescription("dEQP-GLES3.group1", "takeitall"), 920 new TestDescription("dEQP-GLES3.group2", "jeba"), 921 new TestDescription("dEQP-GLES3.group2", "yes"), 922 new TestDescription("dEQP-GLES3.group2", "granted"), 923 }; 924 925 List<TestDescription> allTests = new ArrayList<TestDescription>(); 926 for (TestDescription id : testIds) { 927 allTests.add(id); 928 } 929 930 String expectedTrie = 931 "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}"; 932 933 DeqpTestRunner deqpTest = 934 buildGlesTestRunner(3, 0, allTests, mTestsDir); 935 deqpTest.addIncludeFilter("*"); 936 testFiltering(deqpTest, expectedTrie, allTests); 937 } 938 testRun_excludeAll()939 public void testRun_excludeAll() throws Exception { 940 final TestDescription[] testIds = { 941 new TestDescription("dEQP-GLES3.group1", "no"), 942 new TestDescription("dEQP-GLES3.group1", "nope"), 943 new TestDescription("dEQP-GLES3.group1", "nottoday"), 944 new TestDescription("dEQP-GLES3.group2", "banned"), 945 new TestDescription("dEQP-GLES3.group2", "notrecognized"), 946 new TestDescription("dEQP-GLES3.group2", "-2"), 947 }; 948 949 List<TestDescription> allTests = new ArrayList<TestDescription>(); 950 for (TestDescription id : testIds) { 951 allTests.add(id); 952 } 953 954 DeqpTestRunner deqpTest = 955 buildGlesTestRunner(3, 0, allTests, mTestsDir); 956 deqpTest.addExcludeFilter("*"); 957 ITestInvocationListener mockListener = 958 EasyMock.createStrictMock(ITestInvocationListener.class); 959 mockListener.testRunStarted(getTestId(deqpTest), 0); 960 EasyMock.expectLastCall().once(); 961 mockListener.testRunEnded(EasyMock.anyLong(), 962 EasyMock.<HashMap<String, Metric>>notNull()); 963 EasyMock.expectLastCall().once(); 964 965 EasyMock.replay(mockListener); 966 deqpTest.run(mockListener); 967 EasyMock.verify(mockListener); 968 } 969 testRun_incrementalDeqpAttributeSet()970 public void testRun_incrementalDeqpAttributeSet() throws Exception { 971 final TestDescription[] testIds = { 972 new TestDescription("dEQP-GLES3.group1", "no"), 973 new TestDescription("dEQP-GLES3.group1", "nope"), 974 new TestDescription("dEQP-GLES3.group1", "nottoday"), 975 new TestDescription("dEQP-GLES3.group2", "banned"), 976 new TestDescription("dEQP-GLES3.group2", "notrecognized"), 977 new TestDescription("dEQP-GLES3.group2", "-2"), 978 }; 979 980 List<TestDescription> allTests = new ArrayList<TestDescription>(); 981 for (TestDescription id : testIds) { 982 allTests.add(id); 983 } 984 985 DeqpTestRunner deqpTest = 986 buildGlesTestRunner(3, 0, allTests, mTestsDir); 987 988 HashMap attributes = new HashMap<>(); 989 attributes.put(IncrementalDeqpPreparer.INCREMENTAL_DEQP_ATTRIBUTE_NAME, 990 ""); 991 IFolderBuildInfo mockBuildInfo = 992 EasyMock.createMock(IFolderBuildInfo.class); 993 EasyMock.expect(mockBuildInfo.getBuildAttributes()) 994 .andReturn(attributes) 995 .atLeastOnce(); 996 CompatibilityBuildHelper helper = 997 new BuildHelperMock(mockBuildInfo, mTestsDir); 998 deqpTest.setBuildHelper(helper); 999 EasyMock.replay(mockBuildInfo); 1000 1001 ITestInvocationListener mockListener = 1002 EasyMock.createStrictMock(ITestInvocationListener.class); 1003 mockListener.testRunStarted(getTestId(deqpTest), 0); 1004 EasyMock.expectLastCall().once(); 1005 mockListener.testRunEnded(EasyMock.anyLong(), 1006 EasyMock.<HashMap<String, Metric>>notNull()); 1007 EasyMock.expectLastCall().once(); 1008 1009 EasyMock.replay(mockListener); 1010 deqpTest.run(mockListener); 1011 EasyMock.verify(mockListener); 1012 } 1013 testRun_incrementalDeqpEnabled()1014 public void testRun_incrementalDeqpEnabled() throws Exception { 1015 final String output = 1016 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1017 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1018 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1019 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1020 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1021 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1022 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1023 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1024 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1025 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1026 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1027 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1028 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1029 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1030 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1031 + 1032 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.incremental-deqp.should-run-1\r\n" 1033 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1034 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1035 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1036 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1037 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1038 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1039 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1040 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1041 + 1042 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.incremental-deqp.should-run-2\r\n" 1043 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1044 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1045 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1046 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1047 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1048 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1049 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1050 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1051 + 1052 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.incremental-deqp.should-run-3\r\n" 1053 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1054 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1055 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1056 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1057 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1058 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1059 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1060 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1061 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1062 + "INSTRUMENTATION_CODE: 0\r\n"; 1063 final TestDescription[] testIds = { 1064 new TestDescription("dEQP-GLES3.non-incremental-deqp", 1065 "should-skip-1"), 1066 new TestDescription("dEQP-GLES3.non-incremental-deqp", 1067 "should-skip-2"), 1068 new TestDescription("dEQP-GLES3.non-incremental-deqp", 1069 "should-skip-3"), 1070 new TestDescription("dEQP-GLES3.incremental-deqp", "should-run-1"), 1071 new TestDescription("dEQP-GLES3.incremental-deqp", "should-run-2"), 1072 new TestDescription("dEQP-GLES3.incremental-deqp", "should-run-3"), 1073 }; 1074 1075 List<TestDescription> allTests = new ArrayList<>(); 1076 Collections.addAll(allTests, testIds); 1077 1078 List<TestDescription> incrementalTests = new ArrayList<>(); 1079 incrementalTests.add(testIds[3]); 1080 incrementalTests.add(testIds[4]); 1081 incrementalTests.add(testIds[5]); 1082 1083 DeqpTestRunner deqpTest = 1084 buildGlesTestRunner(3, 0, allTests, mTestsDir, true, incrementalTests); 1085 1086 String testTrie = 1087 "{dEQP-GLES3{incremental-deqp{should-run-1,should-run-2,should-run-3}}}"; 1088 1089 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1090 ITestInvocationListener mockListener = 1091 EasyMock.createStrictMock(ITestInvocationListener.class); 1092 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1093 int version = 3 << 16; 1094 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1095 .andReturn(Integer.toString(version)) 1096 .atLeastOnce(); 1097 1098 expectRenderConfigQuery(mockDevice, 3, 0); 1099 1100 String commandLine = String.format( 1101 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1102 + "--deqp-screen-rotation=unspecified " 1103 + "--deqp-surface-type=window " 1104 + "--deqp-log-images=disable " 1105 + "--deqp-watchdog=enable", 1106 APP_DIR + CASE_LIST_FILE_NAME); 1107 1108 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, 1109 commandLine, output); 1110 1111 mockListener.testRunStarted(getTestId(deqpTest), incrementalTests.size()); 1112 EasyMock.expectLastCall().once(); 1113 1114 // Expect the calls twice: setupTestEnvironment() and 1115 // teardownTestEnvironment() 1116 EasyMock 1117 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1118 "settings delete global angle_gl_driver_selection_pkgs"))) 1119 .andReturn("") 1120 .once(); 1121 EasyMock 1122 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1123 "settings delete global angle_gl_driver_selection_values"))) 1124 .andReturn("") 1125 .once(); 1126 EasyMock 1127 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1128 "settings delete global angle_gl_driver_selection_pkgs"))) 1129 .andReturn("") 1130 .once(); 1131 EasyMock 1132 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1133 "settings delete global angle_gl_driver_selection_values"))) 1134 .andReturn("") 1135 .once(); 1136 1137 for (TestDescription incrementalTest : incrementalTests) { 1138 mockListener.testStarted(EasyMock.eq(incrementalTest)); 1139 EasyMock.expectLastCall().once(); 1140 1141 mockListener.testEnded(EasyMock.eq(incrementalTest), 1142 EasyMock.<HashMap<String, Metric>>notNull()); 1143 1144 EasyMock.expectLastCall().once(); 1145 } 1146 1147 mockListener.testRunEnded(EasyMock.anyLong(), 1148 EasyMock.<HashMap<String, Metric>>notNull()); 1149 EasyMock.expectLastCall().once(); 1150 1151 EasyMock.replay(mockDevice, mockIDevice); 1152 EasyMock.replay(mockListener); 1153 1154 deqpTest.setDevice(mockDevice); 1155 deqpTest.run(mockListener); 1156 1157 EasyMock.verify(mockListener); 1158 EasyMock.verify(mockDevice, mockIDevice); 1159 } 1160 1161 /** 1162 * Test running a unexecutable test. 1163 */ testRun_unexecutableTests()1164 public void testRun_unexecutableTests() throws Exception { 1165 final String instrumentationAnswerNoExecs = 1166 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1167 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1168 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1169 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1170 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1171 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1172 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1173 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1174 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1175 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1176 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1177 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1178 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1179 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1180 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1181 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1182 + "INSTRUMENTATION_CODE: 0\r\n"; 1183 1184 final TestDescription[] testIds = { 1185 new TestDescription("dEQP-GLES3.missing", "no"), 1186 new TestDescription("dEQP-GLES3.missing", "nope"), 1187 new TestDescription("dEQP-GLES3.missing", "donotwant"), 1188 }; 1189 1190 final String[] testPaths = { 1191 "dEQP-GLES3.missing.no", 1192 "dEQP-GLES3.missing.nope", 1193 "dEQP-GLES3.missing.donotwant", 1194 }; 1195 1196 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1197 ITestInvocationListener mockListener = 1198 EasyMock.createStrictMock(ITestInvocationListener.class); 1199 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1200 1201 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1202 1203 for (TestDescription id : testIds) { 1204 tests.add(id); 1205 } 1206 1207 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1208 1209 int version = 3 << 16; 1210 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1211 .andReturn(Integer.toString(version)) 1212 .atLeastOnce(); 1213 1214 expectRenderConfigQuery(mockDevice, 3, 0); 1215 1216 String commandLine = String.format( 1217 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1218 + "--deqp-screen-rotation=unspecified " 1219 + "--deqp-surface-type=window " 1220 + "--deqp-log-images=disable " 1221 + "--deqp-watchdog=enable", 1222 APP_DIR + CASE_LIST_FILE_NAME); 1223 1224 // first try 1225 runInstrumentationLineAndAnswer( 1226 mockDevice, mockIDevice, "{dEQP-GLES3{missing{no,nope,donotwant}}}", 1227 commandLine, instrumentationAnswerNoExecs); 1228 1229 // splitting begins 1230 runInstrumentationLineAndAnswer( 1231 mockDevice, mockIDevice, "{dEQP-GLES3{missing{no}}}", commandLine, 1232 instrumentationAnswerNoExecs); 1233 runInstrumentationLineAndAnswer( 1234 mockDevice, mockIDevice, "{dEQP-GLES3{missing{nope,donotwant}}}", 1235 commandLine, instrumentationAnswerNoExecs); 1236 runInstrumentationLineAndAnswer( 1237 mockDevice, mockIDevice, "{dEQP-GLES3{missing{nope}}}", commandLine, 1238 instrumentationAnswerNoExecs); 1239 runInstrumentationLineAndAnswer( 1240 mockDevice, mockIDevice, "{dEQP-GLES3{missing{donotwant}}}", 1241 commandLine, instrumentationAnswerNoExecs); 1242 1243 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 1244 EasyMock.expectLastCall().once(); 1245 1246 // Expect the calls twice: setupTestEnvironment() and 1247 // teardownTestEnvironment() 1248 EasyMock 1249 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1250 "settings delete global angle_gl_driver_selection_pkgs"))) 1251 .andReturn("") 1252 .once(); 1253 EasyMock 1254 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1255 "settings delete global angle_gl_driver_selection_values"))) 1256 .andReturn("") 1257 .once(); 1258 EasyMock 1259 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1260 "settings delete global angle_gl_driver_selection_pkgs"))) 1261 .andReturn("") 1262 .once(); 1263 EasyMock 1264 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1265 "settings delete global angle_gl_driver_selection_values"))) 1266 .andReturn("") 1267 .once(); 1268 1269 for (int i = 0; i < testPaths.length; i++) { 1270 mockListener.testStarted(EasyMock.eq(testIds[i])); 1271 EasyMock.expectLastCall().once(); 1272 1273 mockListener.testFailed( 1274 EasyMock.eq(testIds[i]), 1275 EasyMock.eq( 1276 "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n" 1277 + "Abort: Test cannot be executed")); 1278 EasyMock.expectLastCall().once(); 1279 1280 mockListener.testEnded(EasyMock.eq(testIds[i]), 1281 EasyMock.<HashMap<String, Metric>>notNull()); 1282 EasyMock.expectLastCall().once(); 1283 } 1284 1285 mockListener.testRunEnded(EasyMock.anyLong(), 1286 EasyMock.<HashMap<String, Metric>>notNull()); 1287 EasyMock.expectLastCall().once(); 1288 1289 EasyMock.replay(mockDevice, mockIDevice); 1290 EasyMock.replay(mockListener); 1291 1292 deqpTest.setDevice(mockDevice); 1293 deqpTest.run(mockListener); 1294 1295 EasyMock.verify(mockListener); 1296 EasyMock.verify(mockDevice, mockIDevice); 1297 } 1298 1299 /** 1300 * Test that test are left unexecuted if pm list query fails 1301 */ testRun_queryPmListFailure()1302 public void testRun_queryPmListFailure() throws Exception { 1303 final TestDescription testId = 1304 new TestDescription("dEQP-GLES3.orientation", "test"); 1305 1306 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1307 ITestInvocationListener mockListener = 1308 EasyMock.createStrictMock(ITestInvocationListener.class); 1309 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1310 tests.add(testId); 1311 1312 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1313 OptionSetter setter = new OptionSetter(deqpTest); 1314 // Note: If the rotation is the default unspecified, features are not 1315 // queried at all 1316 setter.setOptionValue("deqp-screen-rotation", "90"); 1317 1318 deqpTest.setDevice(mockDevice); 1319 1320 int version = 3 << 16; 1321 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1322 .andReturn(Integer.toString(version)) 1323 .atLeastOnce(); 1324 1325 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 1326 .andReturn("not a valid format"); 1327 1328 mockListener.testRunStarted(getTestId(deqpTest), 1); 1329 EasyMock.expectLastCall().once(); 1330 EasyMock 1331 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1332 "settings delete global angle_gl_driver_selection_pkgs"))) 1333 .andReturn("") 1334 .once(); 1335 EasyMock 1336 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1337 "settings delete global angle_gl_driver_selection_values"))) 1338 .andReturn("") 1339 .once(); 1340 1341 mockListener.testRunEnded(EasyMock.anyLong(), 1342 EasyMock.<HashMap<String, Metric>>notNull()); 1343 EasyMock.expectLastCall().once(); 1344 1345 EasyMock.replay(mockDevice); 1346 EasyMock.replay(mockListener); 1347 deqpTest.run(mockListener); 1348 EasyMock.verify(mockListener); 1349 EasyMock.verify(mockDevice); 1350 } 1351 1352 /** 1353 * Test that test are left unexecuted if renderablity query fails 1354 */ testRun_queryRenderabilityFailure()1355 public void testRun_queryRenderabilityFailure() throws Exception { 1356 final TestDescription testId = 1357 new TestDescription("dEQP-GLES3.orientation", "test"); 1358 1359 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1360 ITestInvocationListener mockListener = 1361 EasyMock.createStrictMock(ITestInvocationListener.class); 1362 1363 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1364 tests.add(testId); 1365 1366 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1367 1368 deqpTest.setDevice(mockDevice); 1369 1370 int version = 3 << 16; 1371 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1372 .andReturn(Integer.toString(version)) 1373 .atLeastOnce(); 1374 1375 expectRenderConfigQueryAndReturn( 1376 mockDevice, 1377 "--deqp-gl-config-name=rgba8888d24s8 " 1378 + "--deqp-screen-rotation=unspecified " 1379 + "--deqp-surface-type=window " 1380 + "--deqp-gl-major-version=3 " 1381 + "--deqp-gl-minor-version=0", 1382 "Maybe?"); 1383 1384 mockListener.testRunStarted(getTestId(deqpTest), 1); 1385 EasyMock.expectLastCall().once(); 1386 1387 EasyMock 1388 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1389 "settings delete global angle_gl_driver_selection_pkgs"))) 1390 .andReturn("") 1391 .once(); 1392 EasyMock 1393 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1394 "settings delete global angle_gl_driver_selection_values"))) 1395 .andReturn("") 1396 .once(); 1397 mockListener.testRunEnded(EasyMock.anyLong(), 1398 EasyMock.<HashMap<String, Metric>>notNull()); 1399 EasyMock.expectLastCall().once(); 1400 1401 EasyMock.replay(mockDevice); 1402 EasyMock.replay(mockListener); 1403 deqpTest.run(mockListener); 1404 EasyMock.verify(mockListener); 1405 EasyMock.verify(mockDevice); 1406 } 1407 1408 /** 1409 * Test that orientation is supplied to runner correctly 1410 */ testOrientation(final String rotation, final String featureString)1411 private void testOrientation(final String rotation, 1412 final String featureString) throws Exception { 1413 final TestDescription testId = 1414 new TestDescription("dEQP-GLES3.orientation", "test"); 1415 final String testPath = "dEQP-GLES3.orientation.test"; 1416 final String testTrie = "{dEQP-GLES3{orientation{test}}}"; 1417 final String output = 1418 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1419 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1420 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1421 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1422 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1423 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1424 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1425 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1426 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1427 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1428 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1429 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1430 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1431 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1432 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1433 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + 1434 testPath + "\r\n" 1435 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1436 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1437 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1438 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1439 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1440 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1441 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1442 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1443 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1444 + "INSTRUMENTATION_CODE: 0\r\n"; 1445 1446 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1447 ITestInvocationListener mockListener = 1448 EasyMock.createStrictMock(ITestInvocationListener.class); 1449 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1450 1451 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1452 tests.add(testId); 1453 1454 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1455 OptionSetter setter = new OptionSetter(deqpTest); 1456 setter.setOptionValue("deqp-screen-rotation", rotation); 1457 1458 deqpTest.setDevice(mockDevice); 1459 1460 int version = 3 << 16; 1461 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1462 .andReturn(Integer.toString(version)) 1463 .atLeastOnce(); 1464 1465 if (!rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED)) { 1466 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 1467 .andReturn(featureString); 1468 } 1469 1470 final boolean isPortraitOrientation = 1471 rotation.equals(BatchRunConfiguration.ROTATION_PORTRAIT) || 1472 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT); 1473 final boolean isLandscapeOrientation = 1474 rotation.equals(BatchRunConfiguration.ROTATION_LANDSCAPE) || 1475 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE); 1476 final boolean executable = 1477 rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED) || 1478 (isPortraitOrientation && 1479 featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) || 1480 (isLandscapeOrientation && 1481 featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE)); 1482 1483 if (executable) { 1484 expectRenderConfigQuery( 1485 mockDevice, 1486 String.format( 1487 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s " 1488 + 1489 "--deqp-surface-type=window --deqp-gl-major-version=3 " 1490 + "--deqp-gl-minor-version=0", 1491 rotation)); 1492 1493 String commandLine = String.format( 1494 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1495 + "--deqp-screen-rotation=%s " 1496 + "--deqp-surface-type=window " 1497 + "--deqp-log-images=disable " 1498 + "--deqp-watchdog=enable", 1499 APP_DIR + CASE_LIST_FILE_NAME, rotation); 1500 1501 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, 1502 commandLine, output); 1503 } 1504 1505 mockListener.testRunStarted(getTestId(deqpTest), 1); 1506 EasyMock.expectLastCall().once(); 1507 1508 // Expect the calls twice: setupTestEnvironment() and 1509 // teardownTestEnvironment() 1510 EasyMock 1511 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1512 "settings delete global angle_gl_driver_selection_pkgs"))) 1513 .andReturn("") 1514 .once(); 1515 EasyMock 1516 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1517 "settings delete global angle_gl_driver_selection_values"))) 1518 .andReturn("") 1519 .once(); 1520 EasyMock 1521 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1522 "settings delete global angle_gl_driver_selection_pkgs"))) 1523 .andReturn("") 1524 .once(); 1525 EasyMock 1526 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1527 "settings delete global angle_gl_driver_selection_values"))) 1528 .andReturn("") 1529 .once(); 1530 1531 mockListener.testStarted(EasyMock.eq(testId)); 1532 EasyMock.expectLastCall().once(); 1533 1534 mockListener.testEnded(EasyMock.eq(testId), 1535 EasyMock.<HashMap<String, Metric>>notNull()); 1536 EasyMock.expectLastCall().once(); 1537 1538 mockListener.testRunEnded(EasyMock.anyLong(), 1539 EasyMock.<HashMap<String, Metric>>notNull()); 1540 EasyMock.expectLastCall().once(); 1541 1542 EasyMock.replay(mockDevice, mockIDevice); 1543 EasyMock.replay(mockListener); 1544 deqpTest.run(mockListener); 1545 EasyMock.verify(mockListener); 1546 EasyMock.verify(mockDevice, mockIDevice); 1547 } 1548 1549 /** 1550 * Test OpeGL ES3 tests on device with OpenGL ES2. 1551 */ testRun_require30DeviceVersion20()1552 public void testRun_require30DeviceVersion20() throws Exception { 1553 testGlesVersion(3, 0, 2, 0, ASSUMPTION_FAILURE_MESSAGE); 1554 } 1555 1556 /** 1557 * Test OpeGL ES3.1 tests on device with OpenGL ES2. 1558 */ testRun_require31DeviceVersion20()1559 public void testRun_require31DeviceVersion20() throws Exception { 1560 testGlesVersion(3, 1, 2, 0, ASSUMPTION_FAILURE_MESSAGE); 1561 } 1562 1563 /** 1564 * Test OpeGL ES3 tests on device with OpenGL ES3. 1565 */ testRun_require30DeviceVersion30()1566 public void testRun_require30DeviceVersion30() throws Exception { 1567 testGlesVersion(3, 0, 3, 0, PASS_MESSAGE); 1568 } 1569 1570 /** 1571 * Test OpeGL ES3.1 tests on device with OpenGL ES3. 1572 */ testRun_require31DeviceVersion30()1573 public void testRun_require31DeviceVersion30() throws Exception { 1574 testGlesVersion(3, 1, 3, 0, ASSUMPTION_FAILURE_MESSAGE); 1575 } 1576 1577 /** 1578 * Test OpeGL ES3 tests on device with OpenGL ES3.1. 1579 */ testRun_require30DeviceVersion31()1580 public void testRun_require30DeviceVersion31() throws Exception { 1581 testGlesVersion(3, 0, 3, 1, PASS_MESSAGE); 1582 } 1583 1584 /** 1585 * Test OpeGL ES3.1 tests on device with OpenGL ES3.1. 1586 */ testRun_require31DeviceVersion31()1587 public void testRun_require31DeviceVersion31() throws Exception { 1588 testGlesVersion(3, 1, 3, 1, PASS_MESSAGE); 1589 } 1590 1591 /** 1592 * Test dEQP Pass result code. 1593 */ testRun_resultPass()1594 public void testRun_resultPass() throws Exception { 1595 testResultCode("Pass", true); 1596 } 1597 1598 /** 1599 * Test dEQP Fail result code. 1600 */ testRun_resultFail()1601 public void testRun_resultFail() throws Exception { 1602 testResultCode("Fail", false); 1603 } 1604 1605 /** 1606 * Test dEQP NotSupported result code. 1607 */ testRun_resultNotSupported()1608 public void testRun_resultNotSupported() throws Exception { 1609 testResultCode("NotSupported", true); 1610 } 1611 1612 /** 1613 * Test dEQP QualityWarning result code. 1614 */ testRun_resultQualityWarning()1615 public void testRun_resultQualityWarning() throws Exception { 1616 testResultCode("QualityWarning", true); 1617 } 1618 1619 /** 1620 * Test dEQP CompatibilityWarning result code. 1621 */ testRun_resultCompatibilityWarning()1622 public void testRun_resultCompatibilityWarning() throws Exception { 1623 testResultCode("CompatibilityWarning", true); 1624 } 1625 1626 /** 1627 * Test dEQP ResourceError result code. 1628 */ testRun_resultResourceError()1629 public void testRun_resultResourceError() throws Exception { 1630 testResultCode("ResourceError", false); 1631 } 1632 1633 /** 1634 * Test dEQP InternalError result code. 1635 */ testRun_resultInternalError()1636 public void testRun_resultInternalError() throws Exception { 1637 testResultCode("InternalError", false); 1638 } 1639 1640 /** 1641 * Test dEQP Crash result code. 1642 */ testRun_resultCrash()1643 public void testRun_resultCrash() throws Exception { 1644 testResultCode("Crash", false); 1645 } 1646 1647 /** 1648 * Test dEQP Timeout result code. 1649 */ testRun_resultTimeout()1650 public void testRun_resultTimeout() throws Exception { 1651 testResultCode("Timeout", false); 1652 } 1653 /** 1654 * Test dEQP Orientation 1655 */ testRun_orientationLandscape()1656 public void testRun_orientationLandscape() throws Exception { 1657 testOrientation("90", ALL_FEATURES); 1658 } 1659 1660 /** 1661 * Test dEQP Orientation 1662 */ testRun_orientationPortrait()1663 public void testRun_orientationPortrait() throws Exception { 1664 testOrientation("0", ALL_FEATURES); 1665 } 1666 1667 /** 1668 * Test dEQP Orientation 1669 */ testRun_orientationReverseLandscape()1670 public void testRun_orientationReverseLandscape() throws Exception { 1671 testOrientation("270", ALL_FEATURES); 1672 } 1673 1674 /** 1675 * Test dEQP Orientation 1676 */ testRun_orientationReversePortrait()1677 public void testRun_orientationReversePortrait() throws Exception { 1678 testOrientation("180", ALL_FEATURES); 1679 } 1680 1681 /** 1682 * Test dEQP Orientation 1683 */ testRun_orientationUnspecified()1684 public void testRun_orientationUnspecified() throws Exception { 1685 testOrientation("unspecified", ALL_FEATURES); 1686 } 1687 1688 /** 1689 * Test dEQP Orientation with limited features 1690 */ testRun_orientationUnspecifiedLimitedFeatures()1691 public void testRun_orientationUnspecifiedLimitedFeatures() 1692 throws Exception { 1693 testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES); 1694 } 1695 1696 /** 1697 * Test dEQP Orientation with limited features 1698 */ testRun_orientationLandscapeLimitedFeatures()1699 public void testRun_orientationLandscapeLimitedFeatures() throws Exception { 1700 testOrientation("90", ONLY_LANDSCAPE_FEATURES); 1701 } 1702 1703 /** 1704 * Test dEQP Orientation with limited features 1705 */ testRun_orientationPortraitLimitedFeatures()1706 public void testRun_orientationPortraitLimitedFeatures() throws Exception { 1707 testOrientation("0", ONLY_LANDSCAPE_FEATURES); 1708 } 1709 1710 /** 1711 * Test dEQP unsupported pixel format 1712 */ testRun_unsupportedPixelFormat()1713 public void testRun_unsupportedPixelFormat() throws Exception { 1714 final String pixelFormat = "rgba5658d16m4"; 1715 final TestDescription testId = 1716 new TestDescription("dEQP-GLES3.pixelformat", "test"); 1717 1718 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1719 ITestInvocationListener mockListener = 1720 EasyMock.createStrictMock(ITestInvocationListener.class); 1721 1722 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1723 tests.add(testId); 1724 1725 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1726 OptionSetter setter = new OptionSetter(deqpTest); 1727 setter.setOptionValue("deqp-gl-config-name", pixelFormat); 1728 1729 deqpTest.setDevice(mockDevice); 1730 1731 int version = 3 << 16; 1732 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1733 .andReturn(Integer.toString(version)) 1734 .atLeastOnce(); 1735 1736 expectRenderConfigQueryAndReturn( 1737 mockDevice, 1738 String.format( 1739 "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified " 1740 + "--deqp-surface-type=window " 1741 + "--deqp-gl-major-version=3 " 1742 + "--deqp-gl-minor-version=0", 1743 pixelFormat), 1744 "No"); 1745 1746 mockListener.testRunStarted(getTestId(deqpTest), 1); 1747 EasyMock.expectLastCall().once(); 1748 1749 // Expect the calls twice: setupTestEnvironment() and 1750 // teardownTestEnvironment() 1751 EasyMock 1752 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1753 "settings delete global angle_gl_driver_selection_pkgs"))) 1754 .andReturn("") 1755 .once(); 1756 EasyMock 1757 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1758 "settings delete global angle_gl_driver_selection_values"))) 1759 .andReturn("") 1760 .once(); 1761 EasyMock 1762 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1763 "settings delete global angle_gl_driver_selection_pkgs"))) 1764 .andReturn("") 1765 .once(); 1766 EasyMock 1767 .expect(mockDevice.executeShellCommand(EasyMock.eq( 1768 "settings delete global angle_gl_driver_selection_values"))) 1769 .andReturn("") 1770 .once(); 1771 1772 mockListener.testStarted(EasyMock.eq(testId)); 1773 EasyMock.expectLastCall().once(); 1774 1775 mockListener.testEnded(EasyMock.eq(testId), 1776 EasyMock.<HashMap<String, Metric>>notNull()); 1777 EasyMock.expectLastCall().once(); 1778 1779 mockListener.testRunEnded(EasyMock.anyLong(), 1780 EasyMock.<HashMap<String, Metric>>notNull()); 1781 EasyMock.expectLastCall().once(); 1782 1783 EasyMock.replay(mockDevice); 1784 EasyMock.replay(mockListener); 1785 deqpTest.run(mockListener); 1786 EasyMock.verify(mockListener); 1787 EasyMock.verify(mockDevice); 1788 } 1789 1790 /** 1791 * Test interface to mock Tradefed device types. 1792 */ 1793 public static interface RecoverableTestDevice 1794 extends ITestDevice, IManagedTestDevice {} 1795 1796 private static enum RecoveryEvent { 1797 PROGRESS, 1798 FAIL_CONNECTION_REFUSED, 1799 FAIL_LINK_KILLED, 1800 } 1801 runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)1802 private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, 1803 RecoveryEvent[] events) 1804 throws DeviceNotAvailableException { 1805 for (RecoveryEvent event : events) { 1806 switch (event) { 1807 case PROGRESS: 1808 recovery.onExecutionProgressed(); 1809 break; 1810 case FAIL_CONNECTION_REFUSED: 1811 recovery.recoverConnectionRefused(); 1812 break; 1813 case FAIL_LINK_KILLED: 1814 recovery.recoverComLinkKilled(); 1815 break; 1816 } 1817 } 1818 } 1819 setRecoveryExpectationWait( DeqpTestRunner.ISleepProvider mockSleepProvider)1820 private void setRecoveryExpectationWait( 1821 DeqpTestRunner.ISleepProvider mockSleepProvider) { 1822 mockSleepProvider.sleep(EasyMock.gt(0)); 1823 EasyMock.expectLastCall().once(); 1824 } 1825 setRecoveryExpectationKillProcess( RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider)1826 private void setRecoveryExpectationKillProcess( 1827 RecoverableTestDevice mockDevice, 1828 DeqpTestRunner.ISleepProvider mockSleepProvider) 1829 throws DeviceNotAvailableException { 1830 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))) 1831 .andReturn("root 1234 com.drawelement.deqp") 1832 .once(); 1833 1834 EasyMock 1835 .expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))) 1836 .andReturn("") 1837 .once(); 1838 1839 // Recovery checks if kill failed 1840 mockSleepProvider.sleep(EasyMock.gt(0)); 1841 EasyMock.expectLastCall().once(); 1842 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))) 1843 .andReturn("") 1844 .once(); 1845 } 1846 1847 private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)1848 setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice) 1849 throws DeviceNotAvailableException { 1850 EasyMock.expect(mockDevice.recoverDevice()).andReturn(true).once(); 1851 } 1852 setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)1853 private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice) 1854 throws DeviceNotAvailableException { 1855 mockDevice.reboot(); 1856 EasyMock.expectLastCall().once(); 1857 } 1858 setRecoveryExpectationOfAConnFailure( RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)1859 private int setRecoveryExpectationOfAConnFailure( 1860 RecoverableTestDevice mockDevice, 1861 DeqpTestRunner.ISleepProvider mockSleepProvider, 1862 int numConsecutiveErrors) throws DeviceNotAvailableException { 1863 switch (numConsecutiveErrors) { 1864 case 0: 1865 case 1: 1866 setRecoveryExpectationRecovery(mockDevice); 1867 return 2; 1868 case 2: 1869 setRecoveryExpectationReboot(mockDevice); 1870 return 3; 1871 default: 1872 return 4; 1873 } 1874 } 1875 setRecoveryExpectationOfAComKilled( RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)1876 private int setRecoveryExpectationOfAComKilled( 1877 RecoverableTestDevice mockDevice, 1878 DeqpTestRunner.ISleepProvider mockSleepProvider, 1879 int numConsecutiveErrors) throws DeviceNotAvailableException { 1880 switch (numConsecutiveErrors) { 1881 case 0: 1882 setRecoveryExpectationWait(mockSleepProvider); 1883 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1884 return 1; 1885 case 1: 1886 setRecoveryExpectationRecovery(mockDevice); 1887 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1888 return 2; 1889 case 2: 1890 setRecoveryExpectationReboot(mockDevice); 1891 return 3; 1892 default: 1893 return 4; 1894 } 1895 } 1896 setRecoveryExpectationsOfAPattern( RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)1897 private void setRecoveryExpectationsOfAPattern( 1898 RecoverableTestDevice mockDevice, 1899 DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events) 1900 throws DeviceNotAvailableException { 1901 int numConsecutiveErrors = 0; 1902 for (RecoveryEvent event : events) { 1903 switch (event) { 1904 case PROGRESS: 1905 numConsecutiveErrors = 0; 1906 break; 1907 case FAIL_CONNECTION_REFUSED: 1908 numConsecutiveErrors = setRecoveryExpectationOfAConnFailure( 1909 mockDevice, mockSleepProvider, numConsecutiveErrors); 1910 break; 1911 case FAIL_LINK_KILLED: 1912 numConsecutiveErrors = setRecoveryExpectationOfAComKilled( 1913 mockDevice, mockSleepProvider, numConsecutiveErrors); 1914 break; 1915 } 1916 } 1917 } 1918 1919 /** 1920 * Test dEQP runner recovery state machine. 1921 */ testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent... pattern)1922 private void testRecoveryWithPattern(boolean expectSuccess, 1923 RecoveryEvent... pattern) 1924 throws Exception { 1925 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 1926 IMocksControl orderedControl = EasyMock.createStrictControl(); 1927 RecoverableTestDevice mockDevice = 1928 orderedControl.createMock(RecoverableTestDevice.class); 1929 EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL"); 1930 DeqpTestRunner.ISleepProvider mockSleepProvider = 1931 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 1932 1933 setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, 1934 pattern); 1935 1936 orderedControl.replay(); 1937 1938 recovery.setDevice(mockDevice); 1939 recovery.setSleepProvider(mockSleepProvider); 1940 try { 1941 runRecoveryWithPattern(recovery, pattern); 1942 if (!expectSuccess) { 1943 fail("Expected DeviceNotAvailableException"); 1944 } 1945 } catch (DeviceNotAvailableException ex) { 1946 if (expectSuccess) { 1947 fail("Did not expect DeviceNotAvailableException"); 1948 } 1949 } 1950 1951 orderedControl.verify(); 1952 } 1953 1954 // basic patterns 1955 testRecovery_NoEvents()1956 public void testRecovery_NoEvents() throws Exception { 1957 testRecoveryWithPattern(true); 1958 } 1959 testRecovery_AllOk()1960 public void testRecovery_AllOk() throws Exception { 1961 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1962 RecoveryEvent.PROGRESS); 1963 } 1964 1965 // conn fail patterns 1966 testRecovery_OneConnectionFailureBegin()1967 public void testRecovery_OneConnectionFailureBegin() throws Exception { 1968 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1969 RecoveryEvent.PROGRESS); 1970 } 1971 testRecovery_TwoConnectionFailuresBegin()1972 public void testRecovery_TwoConnectionFailuresBegin() throws Exception { 1973 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1974 RecoveryEvent.FAIL_CONNECTION_REFUSED, 1975 RecoveryEvent.PROGRESS); 1976 } 1977 testRecovery_ThreeConnectionFailuresBegin()1978 public void testRecovery_ThreeConnectionFailuresBegin() throws Exception { 1979 testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1980 RecoveryEvent.FAIL_CONNECTION_REFUSED, 1981 RecoveryEvent.FAIL_CONNECTION_REFUSED); 1982 } 1983 testRecovery_OneConnectionFailureMid()1984 public void testRecovery_OneConnectionFailureMid() throws Exception { 1985 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1986 RecoveryEvent.FAIL_CONNECTION_REFUSED, 1987 RecoveryEvent.PROGRESS); 1988 } 1989 testRecovery_TwoConnectionFailuresMid()1990 public void testRecovery_TwoConnectionFailuresMid() throws Exception { 1991 testRecoveryWithPattern( 1992 true, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1993 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1994 } 1995 testRecovery_ThreeConnectionFailuresMid()1996 public void testRecovery_ThreeConnectionFailuresMid() throws Exception { 1997 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, 1998 RecoveryEvent.FAIL_CONNECTION_REFUSED, 1999 RecoveryEvent.FAIL_CONNECTION_REFUSED, 2000 RecoveryEvent.FAIL_CONNECTION_REFUSED); 2001 } 2002 2003 // link fail patterns 2004 testRecovery_OneLinkFailureBegin()2005 public void testRecovery_OneLinkFailureBegin() throws Exception { 2006 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 2007 RecoveryEvent.PROGRESS); 2008 } 2009 testRecovery_TwoLinkFailuresBegin()2010 public void testRecovery_TwoLinkFailuresBegin() throws Exception { 2011 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 2012 RecoveryEvent.FAIL_LINK_KILLED, 2013 RecoveryEvent.PROGRESS); 2014 } 2015 testRecovery_ThreeLinkFailuresBegin()2016 public void testRecovery_ThreeLinkFailuresBegin() throws Exception { 2017 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 2018 RecoveryEvent.FAIL_LINK_KILLED, 2019 RecoveryEvent.FAIL_LINK_KILLED, 2020 RecoveryEvent.PROGRESS); 2021 } 2022 testRecovery_FourLinkFailuresBegin()2023 public void testRecovery_FourLinkFailuresBegin() throws Exception { 2024 testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED, 2025 RecoveryEvent.FAIL_LINK_KILLED, 2026 RecoveryEvent.FAIL_LINK_KILLED, 2027 RecoveryEvent.FAIL_LINK_KILLED); 2028 } 2029 testRecovery_OneLinkFailureMid()2030 public void testRecovery_OneLinkFailureMid() throws Exception { 2031 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 2032 RecoveryEvent.FAIL_LINK_KILLED, 2033 RecoveryEvent.PROGRESS); 2034 } 2035 testRecovery_TwoLinkFailuresMid()2036 public void testRecovery_TwoLinkFailuresMid() throws Exception { 2037 testRecoveryWithPattern( 2038 true, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 2039 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 2040 } 2041 testRecovery_ThreeLinkFailuresMid()2042 public void testRecovery_ThreeLinkFailuresMid() throws Exception { 2043 testRecoveryWithPattern( 2044 true, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 2045 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 2046 RecoveryEvent.PROGRESS); 2047 } 2048 testRecovery_FourLinkFailuresMid()2049 public void testRecovery_FourLinkFailuresMid() throws Exception { 2050 testRecoveryWithPattern( 2051 false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 2052 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 2053 RecoveryEvent.FAIL_LINK_KILLED); 2054 } 2055 2056 // mixed patterns 2057 testRecovery_MixedFailuresProgressBetween()2058 public void testRecovery_MixedFailuresProgressBetween() throws Exception { 2059 testRecoveryWithPattern( 2060 true, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 2061 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 2062 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 2063 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 2064 RecoveryEvent.PROGRESS); 2065 } 2066 testRecovery_MixedFailuresNoProgressBetween()2067 public void testRecovery_MixedFailuresNoProgressBetween() throws Exception { 2068 testRecoveryWithPattern( 2069 true, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 2070 RecoveryEvent.FAIL_CONNECTION_REFUSED, 2071 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 2072 } 2073 2074 /** 2075 * Test recovery if process cannot be killed 2076 */ testRecovery_unkillableProcess()2077 public void testRecovery_unkillableProcess() throws Exception { 2078 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 2079 IMocksControl orderedControl = EasyMock.createStrictControl(); 2080 RecoverableTestDevice mockDevice = 2081 orderedControl.createMock(RecoverableTestDevice.class); 2082 DeqpTestRunner.ISleepProvider mockSleepProvider = 2083 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 2084 2085 // recovery attempts to kill the process after a timeout 2086 mockSleepProvider.sleep(EasyMock.gt(0)); 2087 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))) 2088 .andReturn("root 1234 com.drawelement.deqp") 2089 .once(); 2090 EasyMock 2091 .expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))) 2092 .andReturn("") 2093 .once(); 2094 2095 // Recovery checks if kill failed 2096 mockSleepProvider.sleep(EasyMock.gt(0)); 2097 EasyMock.expectLastCall().once(); 2098 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))) 2099 .andReturn("root 1234 com.drawelement.deqp") 2100 .once(); 2101 2102 // Recovery resets the connection 2103 EasyMock.expect(mockDevice.recoverDevice()).andReturn(true); 2104 2105 // and attempts to kill the process again 2106 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))) 2107 .andReturn("root 1234 com.drawelement.deqp") 2108 .once(); 2109 EasyMock 2110 .expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))) 2111 .andReturn("") 2112 .once(); 2113 2114 // Recovery checks if kill failed 2115 mockSleepProvider.sleep(EasyMock.gt(0)); 2116 EasyMock.expectLastCall().once(); 2117 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))) 2118 .andReturn("root 1234 com.drawelement.deqp") 2119 .once(); 2120 2121 // recovery reboots the device 2122 mockDevice.reboot(); 2123 EasyMock.expectLastCall().once(); 2124 2125 orderedControl.replay(); 2126 recovery.setDevice(mockDevice); 2127 recovery.setSleepProvider(mockSleepProvider); 2128 recovery.recoverComLinkKilled(); 2129 orderedControl.verify(); 2130 } 2131 2132 /** 2133 * Test external interruption before batch run. 2134 */ testInterrupt_killBeforeBatch()2135 public void testInterrupt_killBeforeBatch() throws Exception { 2136 final TestDescription testId = 2137 new TestDescription("dEQP-GLES3.interrupt", "test"); 2138 2139 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2140 tests.add(testId); 2141 2142 ITestInvocationListener mockListener = 2143 EasyMock.createStrictMock(ITestInvocationListener.class); 2144 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2145 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 2146 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 2147 2148 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2149 2150 deqpTest.setDevice(mockDevice); 2151 deqpTest.setRunUtil(mockRunUtil); 2152 2153 int version = 3 << 16; 2154 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 2155 .andReturn(Integer.toString(version)) 2156 .atLeastOnce(); 2157 2158 expectRenderConfigQuery( 2159 mockDevice, 2160 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 2161 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 2162 + "--deqp-gl-minor-version=0"); 2163 2164 mockRunUtil.sleep(0); 2165 EasyMock.expectLastCall().andThrow(new RunInterruptedException( 2166 "message", InfraErrorIdentifier.TRADEFED_SHUTTING_DOWN)); 2167 2168 mockListener.testRunStarted(getTestId(deqpTest), 1); 2169 EasyMock.expectLastCall().once(); 2170 EasyMock 2171 .expect(mockDevice.executeShellCommand(EasyMock.eq( 2172 "settings delete global angle_gl_driver_selection_pkgs"))) 2173 .andReturn("") 2174 .once(); 2175 EasyMock 2176 .expect(mockDevice.executeShellCommand(EasyMock.eq( 2177 "settings delete global angle_gl_driver_selection_values"))) 2178 .andReturn("") 2179 .once(); 2180 mockListener.testRunEnded( 2181 EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject()); 2182 EasyMock.expectLastCall().once(); 2183 2184 EasyMock.replay(mockDevice, mockIDevice); 2185 EasyMock.replay(mockListener); 2186 EasyMock.replay(mockRunUtil); 2187 try { 2188 deqpTest.run(mockListener); 2189 fail("expected RunInterruptedException"); 2190 } catch (RunInterruptedException ex) { 2191 // expected 2192 } 2193 EasyMock.verify(mockRunUtil); 2194 EasyMock.verify(mockListener); 2195 EasyMock.verify(mockDevice, mockIDevice); 2196 } 2197 2198 private void runShardedTest(TestDescription[] testIds, ArrayList<ArrayList<TestDescription>> testsForShard)2199 runShardedTest(TestDescription[] testIds, 2200 ArrayList<ArrayList<TestDescription>> testsForShard) 2201 throws Exception { 2202 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2203 for (TestDescription id : testIds) 2204 tests.add(id); 2205 2206 DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests, mTestsDir); 2207 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split(); 2208 2209 for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) { 2210 DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex); 2211 shard.setBuildHelper(getMockBuildHelper(mTestsDir)); 2212 2213 ArrayList<TestDescription> shardTests = 2214 testsForShard.get(shardIndex); 2215 2216 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2217 ITestInvocationListener mockListener = 2218 EasyMock.createStrictMock(ITestInvocationListener.class); 2219 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 2220 int version = 3 << 16; 2221 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 2222 .andReturn(Integer.toString(version)) 2223 .atLeastOnce(); 2224 2225 mockListener.testRunStarted(getTestId(shard), shardTests.size()); 2226 EasyMock.expectLastCall().once(); 2227 2228 // Expect the calls twice: setupTestEnvironment() and 2229 // teardownTestEnvironment() 2230 EasyMock 2231 .expect(mockDevice.executeShellCommand(EasyMock.eq( 2232 "settings delete global angle_gl_driver_selection_pkgs"))) 2233 .andReturn("") 2234 .once(); 2235 EasyMock 2236 .expect(mockDevice.executeShellCommand(EasyMock.eq( 2237 "settings delete global angle_gl_driver_selection_values"))) 2238 .andReturn("") 2239 .once(); 2240 EasyMock 2241 .expect(mockDevice.executeShellCommand(EasyMock.eq( 2242 "settings delete global angle_gl_driver_selection_pkgs"))) 2243 .andReturn("") 2244 .once(); 2245 EasyMock 2246 .expect(mockDevice.executeShellCommand(EasyMock.eq( 2247 "settings delete global angle_gl_driver_selection_values"))) 2248 .andReturn("") 2249 .once(); 2250 2251 expectRenderConfigQuery(mockDevice, 3, 0); 2252 2253 String testOut = buildTestProcessOutput(shardTests); 2254 // NOTE: This assumes that there won't be multiple batches per 2255 // shard! 2256 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); 2257 2258 for (int i = 0; i < shardTests.size(); i++) { 2259 mockListener.testStarted(EasyMock.eq(shardTests.get(i))); 2260 EasyMock.expectLastCall().once(); 2261 2262 mockListener.testEnded( 2263 EasyMock.eq(shardTests.get(i)), 2264 EasyMock.<HashMap<String, Metric>>notNull()); 2265 2266 EasyMock.expectLastCall().once(); 2267 } 2268 2269 mockListener.testRunEnded( 2270 EasyMock.anyLong(), 2271 EasyMock.<HashMap<String, Metric>>notNull()); 2272 EasyMock.expectLastCall().once(); 2273 2274 EasyMock.replay(mockDevice, mockIDevice); 2275 EasyMock.replay(mockListener); 2276 2277 shard.setDevice(mockDevice); 2278 shard.run(mockListener); 2279 2280 EasyMock.verify(mockListener); 2281 EasyMock.verify(mockDevice, mockIDevice); 2282 } 2283 } 2284 testSharding_smallTrivial()2285 public void testSharding_smallTrivial() throws Exception { 2286 final TestDescription[] testIds = { 2287 new TestDescription("dEQP-GLES3.info", "vendor"), 2288 new TestDescription("dEQP-GLES3.info", "renderer"), 2289 new TestDescription("dEQP-GLES3.info", "version"), 2290 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 2291 new TestDescription("dEQP-GLES3.info", "extensions"), 2292 new TestDescription("dEQP-GLES3.info", "render_target")}; 2293 ArrayList<ArrayList<TestDescription>> shardedTests = new ArrayList<>(); 2294 ArrayList<TestDescription> shardOne = new ArrayList<>(); 2295 for (int i = 0; i < testIds.length; i++) { 2296 shardOne.add(testIds[i]); 2297 } 2298 shardedTests.add(shardOne); 2299 runShardedTest(testIds, shardedTests); 2300 } 2301 testSharding_twoShards()2302 public void testSharding_twoShards() throws Exception { 2303 final int TEST_COUNT = 1237; 2304 final int SHARD_SIZE = 1000; 2305 2306 ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT); 2307 for (int i = 0; i < TEST_COUNT; i++) { 2308 testIds.add(new TestDescription("dEQP-GLES3.funny.group", 2309 String.valueOf(i))); 2310 } 2311 2312 ArrayList<ArrayList<TestDescription>> shardedTests = new ArrayList<>(); 2313 ArrayList<TestDescription> shard = new ArrayList<>(); 2314 for (int i = 0; i < testIds.size(); i++) { 2315 if (i == SHARD_SIZE) { 2316 shardedTests.add(shard); 2317 shard = new ArrayList<>(); 2318 } 2319 shard.add(testIds.get(i)); 2320 } 2321 shardedTests.add(shard); 2322 runShardedTest(testIds.toArray(new TestDescription[testIds.size()]), 2323 shardedTests); 2324 } 2325 testSharding_empty()2326 public void testSharding_empty() throws Exception { 2327 DeqpTestRunner runner = buildGlesTestRunner( 2328 3, 0, new ArrayList<TestDescription>(), mTestsDir); 2329 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split(); 2330 // Returns null when cannot be sharded. 2331 assertNull(shards); 2332 } 2333 2334 /** 2335 * Test external interruption in testFailed(). 2336 */ testInterrupt_killReportTestFailed()2337 public void testInterrupt_killReportTestFailed() throws Exception { 2338 final TestDescription testId = 2339 new TestDescription("dEQP-GLES3.interrupt", "test"); 2340 final String testPath = "dEQP-GLES3.interrupt.test"; 2341 final String testTrie = "{dEQP-GLES3{interrupt{test}}}"; 2342 final String output = 2343 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 2344 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 2345 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 2346 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2347 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 2348 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 2349 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 2350 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2351 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 2352 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 2353 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 2354 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2355 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 2356 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2357 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2358 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + 2359 testPath + "\r\n" 2360 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2361 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n" 2362 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n" 2363 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2364 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2365 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2366 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2367 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 2368 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2369 + "INSTRUMENTATION_CODE: 0\r\n"; 2370 2371 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2372 tests.add(testId); 2373 2374 ITestInvocationListener mockListener = 2375 EasyMock.createStrictMock(ITestInvocationListener.class); 2376 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2377 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 2378 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 2379 2380 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2381 2382 deqpTest.setDevice(mockDevice); 2383 deqpTest.setRunUtil(mockRunUtil); 2384 2385 int version = 3 << 16; 2386 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 2387 .andReturn(Integer.toString(version)) 2388 .atLeastOnce(); 2389 2390 expectRenderConfigQuery( 2391 mockDevice, 2392 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 2393 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 2394 + "--deqp-gl-minor-version=0"); 2395 2396 mockRunUtil.sleep(0); 2397 EasyMock.expectLastCall().once(); 2398 2399 String commandLine = String.format( 2400 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2401 + "--deqp-screen-rotation=unspecified " 2402 + "--deqp-surface-type=window " 2403 + "--deqp-log-images=disable " 2404 + "--deqp-watchdog=enable", 2405 APP_DIR + CASE_LIST_FILE_NAME); 2406 2407 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, 2408 commandLine, output); 2409 2410 mockListener.testRunStarted(getTestId(deqpTest), 1); 2411 EasyMock.expectLastCall().once(); 2412 2413 EasyMock 2414 .expect(mockDevice.executeShellCommand(EasyMock.eq( 2415 "settings delete global angle_gl_driver_selection_pkgs"))) 2416 .andReturn("") 2417 .once(); 2418 EasyMock 2419 .expect(mockDevice.executeShellCommand(EasyMock.eq( 2420 "settings delete global angle_gl_driver_selection_values"))) 2421 .andReturn("") 2422 .once(); 2423 2424 mockListener.testStarted(EasyMock.eq(testId)); 2425 EasyMock.expectLastCall().once(); 2426 2427 mockListener.testFailed(EasyMock.eq(testId), 2428 EasyMock.<String>notNull()); 2429 EasyMock.expectLastCall().andThrow(new RunInterruptedException( 2430 "message", InfraErrorIdentifier.TRADEFED_SHUTTING_DOWN)); 2431 2432 mockListener.testRunEnded( 2433 EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject()); 2434 EasyMock.expectLastCall().once(); 2435 EasyMock.replay(mockDevice, mockIDevice); 2436 EasyMock.replay(mockListener); 2437 EasyMock.replay(mockRunUtil); 2438 try { 2439 deqpTest.run(mockListener); 2440 fail("expected RunInterruptedException"); 2441 } catch (RunInterruptedException ex) { 2442 // expected 2443 } 2444 EasyMock.verify(mockRunUtil); 2445 EasyMock.verify(mockListener); 2446 EasyMock.verify(mockDevice, mockIDevice); 2447 } 2448 testRuntimeHint_optionSet()2449 public void testRuntimeHint_optionSet() throws Exception { 2450 /* MultiLineReceiver expects "\r\n" line ending. */ 2451 final String output = 2452 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 2453 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 2454 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 2455 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2456 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 2457 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 2458 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 2459 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2460 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 2461 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 2462 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 2463 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2464 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 2465 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2466 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2467 + 2468 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 2469 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2470 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2471 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2472 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2473 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2474 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2475 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2476 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2477 + 2478 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 2479 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2480 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2481 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2482 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2483 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2484 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2485 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2486 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2487 + 2488 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 2489 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2490 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2491 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2492 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2493 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2494 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2495 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2496 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2497 + 2498 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 2499 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2500 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2501 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2502 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2503 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2504 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2505 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2506 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2507 + 2508 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 2509 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2510 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2511 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2512 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2513 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2514 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2515 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2516 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2517 + 2518 "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 2519 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2520 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2521 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2522 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2523 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2524 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2525 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2526 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 2527 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2528 + "INSTRUMENTATION_CODE: 0\r\n"; 2529 2530 final TestDescription[] testIds = { 2531 new TestDescription("dEQP-GLES3.info", "vendor"), 2532 new TestDescription("dEQP-GLES3.info", "renderer"), 2533 new TestDescription("dEQP-GLES3.info", "version"), 2534 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 2535 new TestDescription("dEQP-GLES3.info", "extensions"), 2536 new TestDescription("dEQP-GLES3.info", "render_target")}; 2537 2538 final String[] testPaths = {"dEQP-GLES3.info.vendor", 2539 "dEQP-GLES3.info.renderer", 2540 "dEQP-GLES3.info.version", 2541 "dEQP-GLES3.info.shading_language_version", 2542 "dEQP-GLES3.info.extensions", 2543 "dEQP-GLES3.info.render_target"}; 2544 2545 final String testTrie = 2546 "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 2547 2548 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2549 ITestInvocationListener mockListener = 2550 EasyMock.createStrictMock(ITestInvocationListener.class); 2551 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 2552 2553 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2554 2555 for (TestDescription id : testIds) { 2556 tests.add(id); 2557 } 2558 2559 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2560 OptionSetter setter = new OptionSetter(deqpTest); 2561 final long runtimeMs = 123456; 2562 setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs)); 2563 assertEquals("Wrong expected runtime - option not passed cleanly", 2564 runtimeMs, deqpTest.getRuntimeHint()); 2565 2566 // Try running the tests as well. The unit tests do not set the hint be 2567 // default, so that case is covered. 2568 2569 int version = 3 << 16; 2570 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 2571 .andReturn(Integer.toString(version)) 2572 .atLeastOnce(); 2573 2574 expectRenderConfigQuery(mockDevice, 3, 0); 2575 2576 String commandLine = String.format( 2577 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2578 + "--deqp-screen-rotation=unspecified " 2579 + "--deqp-surface-type=window " 2580 + "--deqp-log-images=disable " 2581 + "--deqp-watchdog=enable", 2582 APP_DIR + CASE_LIST_FILE_NAME); 2583 2584 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, 2585 commandLine, output); 2586 2587 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 2588 EasyMock.expectLastCall().once(); 2589 2590 // Expect the calls twice: setupTestEnvironment() and 2591 // teardownTestEnvironment() 2592 EasyMock 2593 .expect(mockDevice.executeShellCommand(EasyMock.eq( 2594 "settings delete global angle_gl_driver_selection_pkgs"))) 2595 .andReturn("") 2596 .once(); 2597 EasyMock 2598 .expect(mockDevice.executeShellCommand(EasyMock.eq( 2599 "settings delete global angle_gl_driver_selection_values"))) 2600 .andReturn("") 2601 .once(); 2602 EasyMock 2603 .expect(mockDevice.executeShellCommand(EasyMock.eq( 2604 "settings delete global angle_gl_driver_selection_pkgs"))) 2605 .andReturn("") 2606 .once(); 2607 EasyMock 2608 .expect(mockDevice.executeShellCommand(EasyMock.eq( 2609 "settings delete global angle_gl_driver_selection_values"))) 2610 .andReturn("") 2611 .once(); 2612 2613 for (int i = 0; i < testPaths.length; i++) { 2614 mockListener.testStarted(EasyMock.eq(testIds[i])); 2615 EasyMock.expectLastCall().once(); 2616 2617 mockListener.testEnded(EasyMock.eq(testIds[i]), 2618 EasyMock.<HashMap<String, Metric>>notNull()); 2619 2620 EasyMock.expectLastCall().once(); 2621 } 2622 2623 mockListener.testRunEnded(EasyMock.anyLong(), 2624 EasyMock.<HashMap<String, Metric>>notNull()); 2625 EasyMock.expectLastCall().once(); 2626 2627 EasyMock.replay(mockDevice, mockIDevice); 2628 EasyMock.replay(mockListener); 2629 2630 deqpTest.setDevice(mockDevice); 2631 deqpTest.run(mockListener); 2632 2633 EasyMock.verify(mockListener); 2634 EasyMock.verify(mockDevice, mockIDevice); 2635 } 2636 testRuntimeHint_optionSetSharded()2637 public void testRuntimeHint_optionSetSharded() throws Exception { 2638 final int TEST_COUNT = 1237; 2639 final int SHARD_SIZE = 1000; 2640 2641 ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT); 2642 for (int i = 0; i < TEST_COUNT; i++) { 2643 testIds.add(new TestDescription("dEQP-GLES3.funny.group", 2644 String.valueOf(i))); 2645 } 2646 2647 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir); 2648 OptionSetter setter = new OptionSetter(deqpTest); 2649 final long fullRuntimeMs = testIds.size() * 100; 2650 setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs)); 2651 2652 ArrayList<IRemoteTest> shards = 2653 (ArrayList<IRemoteTest>)deqpTest.split(); 2654 assertEquals("First shard's time not proportional to test count", 2655 (fullRuntimeMs * SHARD_SIZE) / TEST_COUNT, 2656 ((IRuntimeHintProvider)shards.get(0)).getRuntimeHint()); 2657 assertEquals("Second shard's time not proportional to test count", 2658 (fullRuntimeMs * (TEST_COUNT - SHARD_SIZE)) / TEST_COUNT, 2659 ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint()); 2660 } 2661 testRuntimeHint_optionNotSet()2662 public void testRuntimeHint_optionNotSet() throws Exception { 2663 final TestDescription[] testIds = { 2664 new TestDescription("dEQP-GLES3.info", "vendor"), 2665 new TestDescription("dEQP-GLES3.info", "renderer"), 2666 new TestDescription("dEQP-GLES3.info", "version"), 2667 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 2668 new TestDescription("dEQP-GLES3.info", "extensions"), 2669 new TestDescription("dEQP-GLES3.info", "render_target")}; 2670 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2671 2672 for (TestDescription id : testIds) { 2673 tests.add(id); 2674 } 2675 2676 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2677 2678 long runtime = deqpTest.getRuntimeHint(); 2679 assertTrue("Runtime for tests must be positive", runtime > 0); 2680 assertTrue("Runtime for tests must be reasonable", 2681 runtime < (1000 * 10)); // Must be done in 10s 2682 } 2683 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String output)2684 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, 2685 IDevice mockIDevice, 2686 final String output) 2687 throws Exception { 2688 String cmd = String.format( 2689 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2690 + "--deqp-screen-rotation=unspecified " 2691 + "--deqp-surface-type=window " 2692 + "--deqp-log-images=disable " 2693 + "--deqp-watchdog=enable", 2694 APP_DIR + CASE_LIST_FILE_NAME); 2695 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, cmd, 2696 output); 2697 } 2698 2699 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String testTrie, final String cmd, final String output)2700 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 2701 final String testTrie, final String cmd, 2702 final String output) throws Exception { 2703 EasyMock 2704 .expect(mockDevice.executeShellCommand( 2705 EasyMock.eq("rm " + APP_DIR + CASE_LIST_FILE_NAME))) 2706 .andReturn("") 2707 .once(); 2708 2709 EasyMock 2710 .expect(mockDevice.executeShellCommand( 2711 EasyMock.eq("rm " + APP_DIR + LOG_FILE_NAME))) 2712 .andReturn("") 2713 .once(); 2714 2715 if (testTrie == null) { 2716 mockDevice.pushString((String)EasyMock.anyObject(), 2717 EasyMock.eq(APP_DIR + CASE_LIST_FILE_NAME)); 2718 } else { 2719 mockDevice.pushString(testTrie + "\n", 2720 APP_DIR + CASE_LIST_FILE_NAME); 2721 } 2722 EasyMock.expectLastCall().andReturn(true).once(); 2723 2724 String command = String.format( 2725 "am instrument %s -w -e deqpLogFilename \"%s\" -e deqpCmdLine \"%s\" " 2726 + "-e deqpLogData \"%s\" %s", 2727 AbiUtils.createAbiFlag(ABI.getName()), APP_DIR + LOG_FILE_NAME, cmd, 2728 false, INSTRUMENTATION_NAME); 2729 2730 EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice); 2731 mockIDevice.executeShellCommand( 2732 EasyMock.eq(command), EasyMock.<IShellOutputReceiver>notNull(), 2733 EasyMock.anyLong(), EasyMock.isA(TimeUnit.class)); 2734 2735 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 2736 @Override 2737 public Object answer() { 2738 IShellOutputReceiver receiver = 2739 (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 2740 2741 receiver.addOutput(output.getBytes(), 0, output.length()); 2742 receiver.flush(); 2743 2744 return null; 2745 } 2746 }); 2747 } 2748 writeStringsToFile(File target, Set<String> strings)2749 static private void writeStringsToFile(File target, Set<String> strings) 2750 throws IOException { 2751 try (PrintWriter out = new PrintWriter(new FileWriter(target))) { 2752 out.print(String.join(System.lineSeparator(), strings)); 2753 out.println(); 2754 } 2755 } 2756 addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option)2757 private void addFilterFileForOption(DeqpTestRunner test, 2758 Set<String> filters, String option) 2759 throws IOException, ConfigurationException { 2760 String filterFile = option + ".txt"; 2761 writeStringsToFile(new File(mTestsDir, filterFile), filters); 2762 OptionSetter setter = new OptionSetter(test); 2763 setter.setOptionValue(option, filterFile); 2764 } 2765 testIncludeFilterFile()2766 public void testIncludeFilterFile() throws Exception { 2767 final TestDescription[] testIds = { 2768 new TestDescription("dEQP-GLES3.missing", "no"), 2769 new TestDescription("dEQP-GLES3.missing", "nope"), 2770 new TestDescription("dEQP-GLES3.missing", "donotwant"), 2771 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2772 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2773 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2774 }; 2775 2776 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2777 for (TestDescription id : testIds) { 2778 allTests.add(id); 2779 } 2780 2781 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2782 activeTests.add(testIds[3]); 2783 activeTests.add(testIds[4]); 2784 activeTests.add(testIds[5]); 2785 2786 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2787 2788 DeqpTestRunner deqpTest = 2789 buildGlesTestRunner(3, 0, allTests, mTestsDir); 2790 Set<String> includes = new HashSet<>(); 2791 includes.add("dEQP-GLES3.pick_me#*"); 2792 addFilterFileForOption(deqpTest, includes, "include-filter-file"); 2793 testFiltering(deqpTest, expectedTrie, activeTests); 2794 } 2795 testMissingIncludeFilterFile()2796 public void testMissingIncludeFilterFile() throws Exception { 2797 final TestDescription[] testIds = { 2798 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2799 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2800 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2801 }; 2802 2803 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2804 for (TestDescription id : testIds) { 2805 allTests.add(id); 2806 } 2807 2808 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2809 2810 DeqpTestRunner deqpTest = 2811 buildGlesTestRunner(3, 0, allTests, mTestsDir); 2812 OptionSetter setter = new OptionSetter(deqpTest); 2813 setter.setOptionValue("include-filter-file", "not-a-file.txt"); 2814 try { 2815 testFiltering(deqpTest, expectedTrie, allTests); 2816 fail("Test execution should have aborted with exception."); 2817 } catch (RuntimeException e) { 2818 } 2819 } 2820 testExcludeFilterFile()2821 public void testExcludeFilterFile() throws Exception { 2822 final TestDescription[] testIds = { 2823 new TestDescription("dEQP-GLES3.missing", "no"), 2824 new TestDescription("dEQP-GLES3.missing", "nope"), 2825 new TestDescription("dEQP-GLES3.missing", "donotwant"), 2826 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2827 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2828 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2829 }; 2830 2831 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2832 for (TestDescription id : testIds) { 2833 allTests.add(id); 2834 } 2835 2836 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2837 activeTests.add(testIds[3]); 2838 activeTests.add(testIds[4]); 2839 activeTests.add(testIds[5]); 2840 2841 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2842 2843 DeqpTestRunner deqpTest = 2844 buildGlesTestRunner(3, 0, allTests, mTestsDir); 2845 Set<String> excludes = new HashSet<>(); 2846 excludes.add("dEQP-GLES3.missing#*"); 2847 addFilterFileForOption(deqpTest, excludes, "exclude-filter-file"); 2848 testFiltering(deqpTest, expectedTrie, activeTests); 2849 } 2850 testFilterComboWithFiles()2851 public void testFilterComboWithFiles() throws Exception { 2852 final TestDescription[] testIds = { 2853 new TestDescription("dEQP-GLES3.group1", "footah"), 2854 new TestDescription("dEQP-GLES3.group1", "foo"), 2855 new TestDescription("dEQP-GLES3.group1", "nope"), 2856 new TestDescription("dEQP-GLES3.group1", "nonotwant"), 2857 new TestDescription("dEQP-GLES3.group2", "foo"), 2858 new TestDescription("dEQP-GLES3.group2", "yes"), 2859 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"), 2860 }; 2861 2862 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2863 for (TestDescription id : testIds) { 2864 allTests.add(id); 2865 } 2866 2867 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2868 activeTests.add(testIds[0]); 2869 activeTests.add(testIds[5]); 2870 2871 String expectedTrie = "{dEQP-GLES3{group1{footah}group2{yes}}}"; 2872 2873 DeqpTestRunner deqpTest = 2874 buildGlesTestRunner(3, 0, allTests, mTestsDir); 2875 2876 Set<String> includes = new HashSet<>(); 2877 includes.add("dEQP-GLES3.group2#*"); 2878 deqpTest.addAllIncludeFilters(includes); 2879 2880 Set<String> fileIncludes = new HashSet<>(); 2881 fileIncludes.add("dEQP-GLES3.group1#no*"); 2882 fileIncludes.add("dEQP-GLES3.group1#foo*"); 2883 addFilterFileForOption(deqpTest, fileIncludes, "include-filter-file"); 2884 2885 Set<String> fileExcludes = new HashSet<>(); 2886 fileExcludes.add("*foo"); 2887 fileExcludes.add("*thoushallnotpass"); 2888 addFilterFileForOption(deqpTest, fileExcludes, "exclude-filter-file"); 2889 2890 deqpTest.addExcludeFilter("dEQP-GLES3.group1#no*"); 2891 2892 testFiltering(deqpTest, expectedTrie, activeTests); 2893 } 2894 testDotToHashConversionInFilters()2895 public void testDotToHashConversionInFilters() throws Exception { 2896 final TestDescription[] testIds = { 2897 new TestDescription("dEQP-GLES3.missing", "no"), 2898 new TestDescription("dEQP-GLES3.pick_me", "donotwant"), 2899 new TestDescription("dEQP-GLES3.pick_me", "yes")}; 2900 2901 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2902 for (TestDescription id : testIds) { 2903 allTests.add(id); 2904 } 2905 2906 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2907 activeTests.add(testIds[2]); 2908 2909 String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}"; 2910 2911 DeqpTestRunner deqpTest = 2912 buildGlesTestRunner(3, 0, allTests, mTestsDir); 2913 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me.yes"); 2914 testFiltering(deqpTest, expectedTrie, activeTests); 2915 } 2916 } 2917