1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.drawelements.deqp.runner; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.compatibility.common.tradefed.targetprep.IncrementalDeqpPreparer; 20 import com.android.ddmlib.IDevice; 21 import com.android.ddmlib.IShellOutputReceiver; 22 import com.android.tradefed.build.IBuildInfo; 23 import com.android.tradefed.build.IFolderBuildInfo; 24 import com.android.tradefed.config.ConfigurationException; 25 import com.android.tradefed.config.OptionSetter; 26 import com.android.tradefed.device.DeviceNotAvailableException; 27 import com.android.tradefed.device.IManagedTestDevice; 28 import com.android.tradefed.device.ITestDevice; 29 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 30 import com.android.tradefed.result.ITestInvocationListener; 31 import com.android.tradefed.result.TestDescription; 32 import com.android.tradefed.result.error.InfraErrorIdentifier; 33 import com.android.tradefed.testtype.Abi; 34 import com.android.tradefed.testtype.IAbi; 35 import com.android.tradefed.testtype.IRemoteTest; 36 import com.android.tradefed.testtype.IRuntimeHintProvider; 37 import com.android.tradefed.util.AbiUtils; 38 import com.android.tradefed.util.FileUtil; 39 import com.android.tradefed.util.IRunUtil; 40 import com.android.tradefed.util.RunInterruptedException; 41 42 import junit.framework.TestCase; 43 44 import org.easymock.EasyMock; 45 import org.easymock.IAnswer; 46 import org.easymock.IMocksControl; 47 48 import java.io.File; 49 import java.io.FileNotFoundException; 50 import java.io.FileWriter; 51 import java.io.IOException; 52 import java.io.PrintWriter; 53 import java.io.StringReader; 54 import java.io.StringWriter; 55 import java.util.ArrayList; 56 import java.util.Arrays; 57 import java.util.Collection; 58 import java.util.HashMap; 59 import java.util.HashSet; 60 import java.util.List; 61 import java.util.Map; 62 import java.util.Set; 63 import java.util.concurrent.TimeUnit; 64 65 /** 66 * Unit tests for {@link DeqpTestRunner}. 67 */ 68 public class DeqpTestRunnerTest extends TestCase { 69 private static final String NAME = "dEQP-GLES3"; 70 private static final IAbi ABI = new Abi("armeabi-v7a", "32"); 71 private static final String APP_DIR = "/sdcard/"; 72 private static final String CASE_LIST_FILE_NAME = "dEQP-TestCaseList.txt"; 73 private static final String LOG_FILE_NAME = "TestLog.qpa"; 74 private static final String INSTRUMENTATION_NAME = 75 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation"; 76 private static final String QUERY_INSTRUMENTATION_NAME = 77 "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation"; 78 private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk"; 79 private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp"; 80 private static final String ONLY_LANDSCAPE_FEATURES = 81 "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE; 82 private static final String ALL_FEATURES = 83 ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT; 84 private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS; 85 86 static { 87 DEFAULT_INSTANCE_ARGS = new ArrayList<>(1); DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>())88 DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>()); 89 DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8"); 90 DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified"); 91 DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window"); 92 } 93 94 private File mTestsDir = null; 95 96 public static class BuildHelperMock extends CompatibilityBuildHelper { 97 private File mTestsDir = null; BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir)98 public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) { 99 super(buildInfo); 100 mTestsDir = testsDir; 101 } 102 @Override getTestsDir()103 public File getTestsDir() throws FileNotFoundException { 104 return mTestsDir; 105 } 106 } 107 108 109 /** 110 * {@inheritDoc} 111 */ 112 @Override setUp()113 protected void setUp() throws Exception { 114 super.setUp(); 115 mTestsDir = FileUtil.createTempDir("deqp-test-cases"); 116 } 117 118 /** 119 * {@inheritDoc} 120 */ 121 @Override tearDown()122 protected void tearDown() throws Exception { 123 FileUtil.recursiveDelete(mTestsDir); 124 super.tearDown(); 125 } 126 buildGlesTestRunner(int majorVersion, int minorVersion, Collection<TestDescription> tests, File testsDir)127 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, 128 int minorVersion, 129 Collection<TestDescription> tests, 130 File testsDir) 131 throws ConfigurationException, IOException { 132 133 StringWriter testlist = new StringWriter(); 134 for (TestDescription test : tests) { 135 testlist.write(test.getClassName() + "." + test.getTestName() + "\n"); 136 } 137 return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString(), testsDir); 138 } 139 getMockBuildHelper(File testsDir)140 private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) { 141 IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class); 142 EasyMock.expect(mockIFolderBuildInfo.getBuildAttributes()).andReturn(new HashMap<>()).anyTimes(); 143 EasyMock.replay(mockIFolderBuildInfo); 144 return new BuildHelperMock(mockIFolderBuildInfo, testsDir); 145 } 146 buildGlesTestRunner(int majorVersion, int minorVersion, String testlist, File testsDir)147 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, 148 int minorVersion, 149 String testlist, 150 File testsDir) 151 throws ConfigurationException, IOException { 152 153 DeqpTestRunner runner = new DeqpTestRunner(); 154 OptionSetter setter = new OptionSetter(runner); 155 156 String deqpPackage = "dEQP-GLES" + Integer.toString(majorVersion) 157 + (minorVersion > 0 ? Integer.toString(minorVersion) : ""); 158 159 final File caselistsFile = new File(testsDir, "gles3-caselist.txt"); 160 FileUtil.writeToFile(testlist, caselistsFile); 161 162 setter.setOptionValue("deqp-package", deqpPackage); 163 setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8"); 164 setter.setOptionValue("deqp-caselist-file", caselistsFile.getName()); 165 setter.setOptionValue("deqp-screen-rotation", "unspecified"); 166 setter.setOptionValue("deqp-surface-type", "window"); 167 runner.setAbi(ABI); 168 runner.setBuildHelper(getMockBuildHelper(testsDir)); 169 170 return runner; 171 } 172 getTestId(DeqpTestRunner runner)173 private static String getTestId(DeqpTestRunner runner) { 174 return AbiUtils.createId(ABI.getName(), runner.getPackageName()); 175 } 176 177 /** 178 * Test version of OpenGL ES. 179 */ testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion)180 private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception { 181 final TestDescription testId = new TestDescription("dEQP-GLES" 182 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 183 + ".info", "version"); 184 185 final String testPath = "dEQP-GLES" 186 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 187 +".info.version"; 188 189 final String testTrie = "{dEQP-GLES" 190 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 191 + "{info{version}}}"; 192 193 final String resultCode = "Pass"; 194 195 /* MultiLineReceiver expects "\r\n" line ending. */ 196 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 197 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 198 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 199 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 200 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 201 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 202 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 203 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 204 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 205 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 206 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 207 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 208 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 209 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 210 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 211 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 212 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 213 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 214 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 215 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 216 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 217 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 218 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 219 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 220 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 221 + "INSTRUMENTATION_CODE: 0\r\n"; 222 223 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 224 ITestInvocationListener mockListener 225 = EasyMock.createStrictMock(ITestInvocationListener.class); 226 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 227 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 228 tests.add(testId); 229 230 DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests, mTestsDir); 231 232 int version = (majorVersion << 16) | minorVersion; 233 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 234 .andReturn(Integer.toString(version)).atLeastOnce(); 235 236 if (majorVersion > requiredMajorVersion 237 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) { 238 239 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 240 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 241 andReturn("").once(); 242 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 243 andReturn("").once(); 244 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 245 andReturn("").once(); 246 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 247 andReturn("").once(); 248 249 expectRenderConfigQuery(mockDevice, requiredMajorVersion, 250 requiredMinorVersion); 251 252 String commandLine = String.format( 253 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 254 + "--deqp-screen-rotation=unspecified " 255 + "--deqp-surface-type=window " 256 + "--deqp-log-images=disable " 257 + "--deqp-watchdog=enable", 258 APP_DIR + CASE_LIST_FILE_NAME); 259 260 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 261 output); 262 } 263 264 mockListener.testRunStarted(getTestId(deqpTest), 1); 265 EasyMock.expectLastCall().once(); 266 267 mockListener.testStarted(EasyMock.eq(testId)); 268 EasyMock.expectLastCall().once(); 269 270 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 271 EasyMock.expectLastCall().once(); 272 273 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 274 EasyMock.expectLastCall().once(); 275 276 EasyMock.replay(mockDevice, mockIDevice); 277 EasyMock.replay(mockListener); 278 279 deqpTest.setDevice(mockDevice); 280 deqpTest.run(mockListener); 281 282 EasyMock.verify(mockListener); 283 EasyMock.verify(mockDevice, mockIDevice); 284 } 285 expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, int minorVersion)286 private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, 287 int minorVersion) throws Exception { 288 expectRenderConfigQuery(mockDevice, 289 String.format("--deqp-gl-config-name=rgba8888d24s8 " 290 + "--deqp-screen-rotation=unspecified " 291 + "--deqp-surface-type=window " 292 + "--deqp-gl-major-version=%d " 293 + "--deqp-gl-minor-version=%d", majorVersion, minorVersion)); 294 } 295 expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)296 private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine) 297 throws Exception { 298 expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes"); 299 } 300 expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, String output)301 private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, 302 String output) throws Exception { 303 final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n" 304 + "INSTRUMENTATION_CODE: 0\r\n"; 305 final String command = String.format( 306 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine " 307 + "\"%s\" %s", 308 AbiUtils.createAbiFlag(ABI.getName()), commandLine, 309 QUERY_INSTRUMENTATION_NAME); 310 311 mockDevice.executeShellCommand(EasyMock.eq(command), 312 EasyMock.<IShellOutputReceiver>notNull()); 313 314 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 315 @Override 316 public Object answer() { 317 IShellOutputReceiver receiver 318 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 319 320 receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length()); 321 receiver.flush(); 322 323 return null; 324 } 325 }); 326 } 327 328 /** 329 * Test that result code produces correctly pass or fail. 330 */ testResultCode(final String resultCode, boolean pass)331 private void testResultCode(final String resultCode, boolean pass) throws Exception { 332 final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version"); 333 final String testPath = "dEQP-GLES3.info.version"; 334 final String testTrie = "{dEQP-GLES3{info{version}}}"; 335 336 /* MultiLineReceiver expects "\r\n" line ending. */ 337 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 338 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 339 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 340 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 341 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 342 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 343 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 344 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 345 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 346 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 347 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 348 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 349 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 350 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 351 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 352 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 353 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 354 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 355 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 356 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 357 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 358 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 359 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 360 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 361 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 362 + "INSTRUMENTATION_CODE: 0\r\n"; 363 364 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 365 ITestInvocationListener mockListener 366 = EasyMock.createStrictMock(ITestInvocationListener.class); 367 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 368 369 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 370 tests.add(testId); 371 372 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 373 374 int version = 3 << 16; 375 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 376 .andReturn(Integer.toString(version)).atLeastOnce(); 377 378 expectRenderConfigQuery(mockDevice, 3, 0); 379 380 String commandLine = String.format( 381 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 382 + "--deqp-screen-rotation=unspecified " 383 + "--deqp-surface-type=window " 384 + "--deqp-log-images=disable " 385 + "--deqp-watchdog=enable", 386 APP_DIR + CASE_LIST_FILE_NAME); 387 388 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 389 390 mockListener.testRunStarted(getTestId(deqpTest), 1); 391 EasyMock.expectLastCall().once(); 392 393 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 394 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 395 andReturn("").once(); 396 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 397 andReturn("").once(); 398 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 399 andReturn("").once(); 400 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 401 andReturn("").once(); 402 403 mockListener.testStarted(EasyMock.eq(testId)); 404 EasyMock.expectLastCall().once(); 405 406 if (!pass) { 407 mockListener.testFailed(testId, 408 "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n" 409 + resultCode + ": Detail" + resultCode); 410 411 EasyMock.expectLastCall().once(); 412 } 413 414 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 415 EasyMock.expectLastCall().once(); 416 417 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 418 EasyMock.expectLastCall().once(); 419 420 EasyMock.replay(mockDevice, mockIDevice); 421 EasyMock.replay(mockListener); 422 423 deqpTest.setDevice(mockDevice); 424 deqpTest.run(mockListener); 425 426 EasyMock.verify(mockListener); 427 EasyMock.verify(mockDevice, mockIDevice); 428 } 429 430 /** 431 * Test running multiple test cases. 432 */ testRun_multipleTests()433 public void testRun_multipleTests() throws Exception { 434 /* MultiLineReceiver expects "\r\n" line ending. */ 435 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 436 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 437 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 438 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 439 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 440 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 441 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 442 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 443 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 444 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 445 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 446 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 447 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 448 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 449 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 450 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 451 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 452 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 453 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 454 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 455 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 456 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 457 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 458 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 459 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 460 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 461 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 462 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 463 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 464 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 465 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 466 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 467 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 468 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 469 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 470 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 471 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 472 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 473 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 474 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 475 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 476 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 477 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 478 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 479 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 480 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 481 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 482 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 483 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 484 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 485 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 486 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 487 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 488 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 489 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 490 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 491 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 492 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 493 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 494 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 495 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 496 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 497 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 498 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 499 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 500 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 501 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 502 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 503 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 504 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 505 + "INSTRUMENTATION_CODE: 0\r\n"; 506 507 final TestDescription[] testIds = { 508 new TestDescription("dEQP-GLES3.info", "vendor"), 509 new TestDescription("dEQP-GLES3.info", "renderer"), 510 new TestDescription("dEQP-GLES3.info", "version"), 511 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 512 new TestDescription("dEQP-GLES3.info", "extensions"), 513 new TestDescription("dEQP-GLES3.info", "render_target") 514 }; 515 516 final String[] testPaths = { 517 "dEQP-GLES3.info.vendor", 518 "dEQP-GLES3.info.renderer", 519 "dEQP-GLES3.info.version", 520 "dEQP-GLES3.info.shading_language_version", 521 "dEQP-GLES3.info.extensions", 522 "dEQP-GLES3.info.render_target" 523 }; 524 525 final String testTrie 526 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 527 528 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 529 ITestInvocationListener mockListener 530 = EasyMock.createStrictMock(ITestInvocationListener.class); 531 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 532 533 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 534 535 for (TestDescription id : testIds) { 536 tests.add(id); 537 } 538 539 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 540 541 int version = 3 << 16; 542 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 543 .andReturn(Integer.toString(version)).atLeastOnce(); 544 545 expectRenderConfigQuery(mockDevice, 3, 0); 546 547 String commandLine = String.format( 548 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 549 + "--deqp-screen-rotation=unspecified " 550 + "--deqp-surface-type=window " 551 + "--deqp-log-images=disable " 552 + "--deqp-watchdog=enable", 553 APP_DIR + CASE_LIST_FILE_NAME); 554 555 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 556 557 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 558 EasyMock.expectLastCall().once(); 559 560 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 561 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 562 andReturn("").once(); 563 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 564 andReturn("").once(); 565 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 566 andReturn("").once(); 567 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 568 andReturn("").once(); 569 570 for (int i = 0; i < testPaths.length; i++) { 571 mockListener.testStarted(EasyMock.eq(testIds[i])); 572 EasyMock.expectLastCall().once(); 573 574 mockListener.testEnded(EasyMock.eq(testIds[i]), 575 EasyMock.<HashMap<String, Metric>>notNull()); 576 577 EasyMock.expectLastCall().once(); 578 } 579 580 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 581 EasyMock.expectLastCall().once(); 582 583 EasyMock.replay(mockDevice, mockIDevice); 584 EasyMock.replay(mockListener); 585 586 deqpTest.setDevice(mockDevice); 587 deqpTest.run(mockListener); 588 589 EasyMock.verify(mockListener); 590 EasyMock.verify(mockDevice, mockIDevice); 591 } 592 buildTestProcessOutput(List<TestDescription> tests)593 static private String buildTestProcessOutput(List<TestDescription> tests) { 594 /* MultiLineReceiver expects "\r\n" line ending. */ 595 final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 596 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 597 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 598 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 599 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 600 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 601 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 602 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 603 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 604 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 605 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 606 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 607 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 608 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; 609 610 final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 611 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 612 + "INSTRUMENTATION_CODE: 0\r\n"; 613 614 StringWriter output = new StringWriter(); 615 output.write(outputHeader); 616 for (TestDescription test : tests) { 617 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"); 618 output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath="); 619 output.write(test.getClassName()); 620 output.write("."); 621 output.write(test.getTestName()); 622 output.write("\r\n"); 623 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 624 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"); 625 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"); 626 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"); 627 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 628 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"); 629 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 630 } 631 output.write(outputEnd); 632 return output.toString(); 633 } 634 testFiltering(DeqpTestRunner deqpTest, String expectedTrie, List<TestDescription> expectedTests)635 private void testFiltering(DeqpTestRunner deqpTest, 636 String expectedTrie, 637 List<TestDescription> expectedTests) throws Exception { 638 int version = 3 << 16; 639 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 640 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 641 .andReturn(Integer.toString(version)).atLeastOnce(); 642 643 boolean thereAreTests = !expectedTests.isEmpty(); 644 645 ITestInvocationListener mockListener 646 = EasyMock.createStrictMock(ITestInvocationListener.class); 647 mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size()); 648 EasyMock.expectLastCall().once(); 649 650 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 651 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 652 andReturn("").once(); 653 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 654 andReturn("").once(); 655 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 656 andReturn("").once(); 657 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 658 andReturn("").once(); 659 660 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 661 if (thereAreTests) 662 { 663 expectRenderConfigQuery(mockDevice, 3, 0); 664 665 String testOut = buildTestProcessOutput(expectedTests); 666 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); 667 668 for (int i = 0; i < expectedTests.size(); i++) { 669 mockListener.testStarted(EasyMock.eq(expectedTests.get(i))); 670 EasyMock.expectLastCall().once(); 671 672 mockListener.testEnded(EasyMock.eq(expectedTests.get(i)), 673 EasyMock.<HashMap<String, Metric>>notNull()); 674 675 EasyMock.expectLastCall().once(); 676 } 677 } 678 679 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 680 EasyMock.expectLastCall().once(); 681 682 EasyMock.replay(mockDevice, mockIDevice); 683 EasyMock.replay(mockListener); 684 685 deqpTest.setDevice(mockDevice); 686 deqpTest.run(mockListener); 687 688 EasyMock.verify(mockListener); 689 EasyMock.verify(mockDevice, mockIDevice); 690 } 691 testRun_trivialIncludeFilter()692 public void testRun_trivialIncludeFilter() throws Exception { 693 final TestDescription[] testIds = { 694 new TestDescription("dEQP-GLES3.missing", "no"), 695 new TestDescription("dEQP-GLES3.missing", "nope"), 696 new TestDescription("dEQP-GLES3.missing", "donotwant"), 697 new TestDescription("dEQP-GLES3.pick_me", "yes"), 698 new TestDescription("dEQP-GLES3.pick_me", "ok"), 699 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 700 }; 701 702 List<TestDescription> allTests = new ArrayList<TestDescription>(); 703 for (TestDescription id : testIds) { 704 allTests.add(id); 705 } 706 707 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 708 activeTests.add(testIds[3]); 709 activeTests.add(testIds[4]); 710 activeTests.add(testIds[5]); 711 712 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 713 714 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 715 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me#*"); 716 testFiltering(deqpTest, expectedTrie, activeTests); 717 } 718 testRun_trivialExcludeFilter()719 public void testRun_trivialExcludeFilter() throws Exception { 720 final TestDescription[] testIds = { 721 new TestDescription("dEQP-GLES3.missing", "no"), 722 new TestDescription("dEQP-GLES3.missing", "nope"), 723 new TestDescription("dEQP-GLES3.missing", "donotwant"), 724 new TestDescription("dEQP-GLES3.pick_me", "yes"), 725 new TestDescription("dEQP-GLES3.pick_me", "ok"), 726 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 727 }; 728 729 List<TestDescription> allTests = new ArrayList<TestDescription>(); 730 for (TestDescription id : testIds) { 731 allTests.add(id); 732 } 733 734 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 735 activeTests.add(testIds[3]); 736 activeTests.add(testIds[4]); 737 activeTests.add(testIds[5]); 738 739 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 740 741 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 742 deqpTest.addExcludeFilter("dEQP-GLES3.missing#*"); 743 testFiltering(deqpTest, expectedTrie, activeTests); 744 } 745 testRun_includeAndExcludeFilter()746 public void testRun_includeAndExcludeFilter() throws Exception { 747 final TestDescription[] testIds = { 748 new TestDescription("dEQP-GLES3.group1", "foo"), 749 new TestDescription("dEQP-GLES3.group1", "nope"), 750 new TestDescription("dEQP-GLES3.group1", "donotwant"), 751 new TestDescription("dEQP-GLES3.group2", "foo"), 752 new TestDescription("dEQP-GLES3.group2", "yes"), 753 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"), 754 }; 755 756 List<TestDescription> allTests = new ArrayList<TestDescription>(); 757 for (TestDescription id : testIds) { 758 allTests.add(id); 759 } 760 761 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 762 activeTests.add(testIds[4]); 763 764 String expectedTrie = "{dEQP-GLES3{group2{yes}}}"; 765 766 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 767 768 Set<String> includes = new HashSet<>(); 769 includes.add("dEQP-GLES3.group2#*"); 770 deqpTest.addAllIncludeFilters(includes); 771 772 Set<String> excludes = new HashSet<>(); 773 excludes.add("*foo"); 774 excludes.add("*thoushallnotpass"); 775 deqpTest.addAllExcludeFilters(excludes); 776 testFiltering(deqpTest, expectedTrie, activeTests); 777 } 778 testRun_includeAll()779 public void testRun_includeAll() throws Exception { 780 final TestDescription[] testIds = { 781 new TestDescription("dEQP-GLES3.group1", "mememe"), 782 new TestDescription("dEQP-GLES3.group1", "yeah"), 783 new TestDescription("dEQP-GLES3.group1", "takeitall"), 784 new TestDescription("dEQP-GLES3.group2", "jeba"), 785 new TestDescription("dEQP-GLES3.group2", "yes"), 786 new TestDescription("dEQP-GLES3.group2", "granted"), 787 }; 788 789 List<TestDescription> allTests = new ArrayList<TestDescription>(); 790 for (TestDescription id : testIds) { 791 allTests.add(id); 792 } 793 794 String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}"; 795 796 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 797 deqpTest.addIncludeFilter("*"); 798 testFiltering(deqpTest, expectedTrie, allTests); 799 } 800 testRun_excludeAll()801 public void testRun_excludeAll() throws Exception { 802 final TestDescription[] testIds = { 803 new TestDescription("dEQP-GLES3.group1", "no"), 804 new TestDescription("dEQP-GLES3.group1", "nope"), 805 new TestDescription("dEQP-GLES3.group1", "nottoday"), 806 new TestDescription("dEQP-GLES3.group2", "banned"), 807 new TestDescription("dEQP-GLES3.group2", "notrecognized"), 808 new TestDescription("dEQP-GLES3.group2", "-2"), 809 }; 810 811 List<TestDescription> allTests = new ArrayList<TestDescription>(); 812 for (TestDescription id : testIds) { 813 allTests.add(id); 814 } 815 816 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 817 deqpTest.addExcludeFilter("*"); 818 ITestInvocationListener mockListener 819 = EasyMock.createStrictMock(ITestInvocationListener.class); 820 mockListener.testRunStarted(getTestId(deqpTest), 0); 821 EasyMock.expectLastCall().once(); 822 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 823 EasyMock.expectLastCall().once(); 824 825 EasyMock.replay(mockListener); 826 deqpTest.run(mockListener); 827 EasyMock.verify(mockListener); 828 } 829 testRun_incrementalDeqp()830 public void testRun_incrementalDeqp() throws Exception { 831 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 832 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 833 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 834 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 835 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 836 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 837 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 838 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 839 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 840 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 841 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 842 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 843 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 844 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 845 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 846 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.incremental-deqp.should-run-1\r\n" 847 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 848 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 849 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 850 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 851 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 852 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 853 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 854 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 855 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.incremental-deqp.should-run-2\r\n" 856 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 857 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 858 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 859 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 860 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 861 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 862 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 863 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 864 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.incremental-deqp.should-run-3\r\n" 865 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 866 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 867 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 868 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 869 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 870 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 871 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 872 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 873 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 874 + "INSTRUMENTATION_CODE: 0\r\n"; 875 final TestDescription[] testIds = { 876 new TestDescription("dEQP-GLES3.non-incremental-deqp", "should-skip-1"), 877 new TestDescription("dEQP-GLES3.non-incremental-deqp", "should-skip-2"), 878 new TestDescription("dEQP-GLES3.non-incremental-deqp", "should-skip-3"), 879 new TestDescription("dEQP-GLES3.incremental-deqp", "should-run-1"), 880 new TestDescription("dEQP-GLES3.incremental-deqp", "should-run-2"), 881 new TestDescription("dEQP-GLES3.incremental-deqp", "should-run-3"), 882 }; 883 884 List<TestDescription> allTests = new ArrayList<TestDescription>(); 885 for (TestDescription id : testIds) { 886 allTests.add(id); 887 } 888 889 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 890 activeTests.add(testIds[3]); 891 activeTests.add(testIds[4]); 892 activeTests.add(testIds[5]); 893 894 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 895 896 HashMap attributes = new HashMap<>(); 897 attributes.put(IncrementalDeqpPreparer.INCREMENTAL_DEQP_ATTRIBUTE_NAME, ""); 898 IFolderBuildInfo mockBuildInfo = EasyMock.createMock(IFolderBuildInfo.class); 899 EasyMock.expect(mockBuildInfo.getBuildAttributes()).andReturn(attributes).atLeastOnce(); 900 CompatibilityBuildHelper helper = new BuildHelperMock(mockBuildInfo, mTestsDir); 901 deqpTest.setBuildHelper(helper); 902 deqpTest.addIncrementalDeqpIncludeTests(Arrays.asList("dEQP-GLES3.incremental-deqp.should-run-1","dEQP-GLES3.incremental-deqp.should-run-2","dEQP-GLES3.incremental-deqp.should-run-3")); 903 EasyMock.replay(mockBuildInfo); 904 905 String testTrie = "{dEQP-GLES3{incremental-deqp{should-run-1,should-run-2,should-run-3}}}"; 906 907 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 908 ITestInvocationListener mockListener 909 = EasyMock.createStrictMock(ITestInvocationListener.class); 910 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 911 int version = 3 << 16; 912 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 913 .andReturn(Integer.toString(version)).atLeastOnce(); 914 915 expectRenderConfigQuery(mockDevice, 3, 0); 916 917 String commandLine = String.format( 918 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 919 + "--deqp-screen-rotation=unspecified " 920 + "--deqp-surface-type=window " 921 + "--deqp-log-images=disable " 922 + "--deqp-watchdog=enable", 923 APP_DIR + CASE_LIST_FILE_NAME); 924 925 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 926 927 mockListener.testRunStarted(getTestId(deqpTest), testIds.length); 928 EasyMock.expectLastCall().once(); 929 930 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 931 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 932 andReturn("").once(); 933 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 934 andReturn("").once(); 935 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 936 andReturn("").once(); 937 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 938 andReturn("").once(); 939 940 for (int i = 0; i < testIds.length; i++) { 941 mockListener.testStarted(EasyMock.eq(testIds[i])); 942 EasyMock.expectLastCall().once(); 943 944 mockListener.testEnded(EasyMock.eq(testIds[i]), 945 EasyMock.<HashMap<String, Metric>>notNull()); 946 947 EasyMock.expectLastCall().once(); 948 } 949 950 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 951 EasyMock.expectLastCall().once(); 952 953 EasyMock.replay(mockDevice, mockIDevice); 954 EasyMock.replay(mockListener); 955 956 deqpTest.setDevice(mockDevice); 957 deqpTest.run(mockListener); 958 959 EasyMock.verify(mockListener); 960 EasyMock.verify(mockDevice, mockIDevice); 961 } 962 963 /** 964 * Test running a unexecutable test. 965 */ testRun_unexecutableTests()966 public void testRun_unexecutableTests() throws Exception { 967 final String instrumentationAnswerNoExecs = 968 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 969 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 970 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 971 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 972 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 973 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 974 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 975 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 976 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 977 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 978 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 979 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 980 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 981 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 982 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 983 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 984 + "INSTRUMENTATION_CODE: 0\r\n"; 985 986 final TestDescription[] testIds = { 987 new TestDescription("dEQP-GLES3.missing", "no"), 988 new TestDescription("dEQP-GLES3.missing", "nope"), 989 new TestDescription("dEQP-GLES3.missing", "donotwant"), 990 }; 991 992 final String[] testPaths = { 993 "dEQP-GLES3.missing.no", 994 "dEQP-GLES3.missing.nope", 995 "dEQP-GLES3.missing.donotwant", 996 }; 997 998 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 999 ITestInvocationListener mockListener 1000 = EasyMock.createStrictMock(ITestInvocationListener.class); 1001 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1002 1003 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1004 1005 for (TestDescription id : testIds) { 1006 tests.add(id); 1007 } 1008 1009 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1010 1011 int version = 3 << 16; 1012 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1013 .andReturn(Integer.toString(version)).atLeastOnce(); 1014 1015 expectRenderConfigQuery(mockDevice, 3, 0); 1016 1017 String commandLine = String.format( 1018 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1019 + "--deqp-screen-rotation=unspecified " 1020 + "--deqp-surface-type=window " 1021 + "--deqp-log-images=disable " 1022 + "--deqp-watchdog=enable", 1023 APP_DIR + CASE_LIST_FILE_NAME); 1024 1025 // first try 1026 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1027 "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs); 1028 1029 // splitting begins 1030 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1031 "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs); 1032 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1033 "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs); 1034 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1035 "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs); 1036 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1037 "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs); 1038 1039 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 1040 EasyMock.expectLastCall().once(); 1041 1042 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 1043 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1044 andReturn("").once(); 1045 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1046 andReturn("").once(); 1047 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1048 andReturn("").once(); 1049 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1050 andReturn("").once(); 1051 1052 for (int i = 0; i < testPaths.length; i++) { 1053 mockListener.testStarted(EasyMock.eq(testIds[i])); 1054 EasyMock.expectLastCall().once(); 1055 1056 mockListener.testFailed(EasyMock.eq(testIds[i]), 1057 EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n" 1058 + "Abort: Test cannot be executed")); 1059 EasyMock.expectLastCall().once(); 1060 1061 mockListener.testEnded(EasyMock.eq(testIds[i]), 1062 EasyMock.<HashMap<String, Metric>>notNull()); 1063 EasyMock.expectLastCall().once(); 1064 } 1065 1066 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1067 EasyMock.expectLastCall().once(); 1068 1069 EasyMock.replay(mockDevice, mockIDevice); 1070 EasyMock.replay(mockListener); 1071 1072 deqpTest.setDevice(mockDevice); 1073 deqpTest.run(mockListener); 1074 1075 EasyMock.verify(mockListener); 1076 EasyMock.verify(mockDevice, mockIDevice); 1077 } 1078 1079 /** 1080 * Test that test are left unexecuted if pm list query fails 1081 */ testRun_queryPmListFailure()1082 public void testRun_queryPmListFailure() 1083 throws Exception { 1084 final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test"); 1085 1086 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1087 ITestInvocationListener mockListener 1088 = EasyMock.createStrictMock(ITestInvocationListener.class); 1089 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1090 tests.add(testId); 1091 1092 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1093 OptionSetter setter = new OptionSetter(deqpTest); 1094 // Note: If the rotation is the default unspecified, features are not queried at all 1095 setter.setOptionValue("deqp-screen-rotation", "90"); 1096 1097 deqpTest.setDevice(mockDevice); 1098 1099 int version = 3 << 16; 1100 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1101 .andReturn(Integer.toString(version)).atLeastOnce(); 1102 1103 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 1104 .andReturn("not a valid format"); 1105 1106 mockListener.testRunStarted(getTestId(deqpTest), 1); 1107 EasyMock.expectLastCall().once(); 1108 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1109 andReturn("").once(); 1110 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1111 andReturn("").once(); 1112 1113 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1114 EasyMock.expectLastCall().once(); 1115 1116 EasyMock.replay(mockDevice); 1117 EasyMock.replay(mockListener); 1118 deqpTest.run(mockListener); 1119 EasyMock.verify(mockListener); 1120 EasyMock.verify(mockDevice); 1121 } 1122 1123 /** 1124 * Test that test are left unexecuted if renderablity query fails 1125 */ testRun_queryRenderabilityFailure()1126 public void testRun_queryRenderabilityFailure() 1127 throws Exception { 1128 final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test"); 1129 1130 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1131 ITestInvocationListener mockListener 1132 = EasyMock.createStrictMock(ITestInvocationListener.class); 1133 1134 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1135 tests.add(testId); 1136 1137 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1138 1139 deqpTest.setDevice(mockDevice); 1140 1141 int version = 3 << 16; 1142 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1143 .andReturn(Integer.toString(version)).atLeastOnce(); 1144 1145 expectRenderConfigQueryAndReturn(mockDevice, 1146 "--deqp-gl-config-name=rgba8888d24s8 " 1147 + "--deqp-screen-rotation=unspecified " 1148 + "--deqp-surface-type=window " 1149 + "--deqp-gl-major-version=3 " 1150 + "--deqp-gl-minor-version=0", "Maybe?"); 1151 1152 mockListener.testRunStarted(getTestId(deqpTest), 1); 1153 EasyMock.expectLastCall().once(); 1154 1155 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1156 andReturn("").once(); 1157 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1158 andReturn("").once(); 1159 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1160 EasyMock.expectLastCall().once(); 1161 1162 EasyMock.replay(mockDevice); 1163 EasyMock.replay(mockListener); 1164 deqpTest.run(mockListener); 1165 EasyMock.verify(mockListener); 1166 EasyMock.verify(mockDevice); 1167 } 1168 1169 /** 1170 * Test that orientation is supplied to runner correctly 1171 */ testOrientation(final String rotation, final String featureString)1172 private void testOrientation(final String rotation, final String featureString) 1173 throws Exception { 1174 final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test"); 1175 final String testPath = "dEQP-GLES3.orientation.test"; 1176 final String testTrie = "{dEQP-GLES3{orientation{test}}}"; 1177 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1178 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1179 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1180 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1181 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1182 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1183 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1184 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1185 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1186 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1187 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1188 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1189 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1190 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1191 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1192 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 1193 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1194 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1195 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1196 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1197 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1198 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1199 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1200 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1201 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1202 + "INSTRUMENTATION_CODE: 0\r\n"; 1203 1204 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1205 ITestInvocationListener mockListener 1206 = EasyMock.createStrictMock(ITestInvocationListener.class); 1207 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1208 1209 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1210 tests.add(testId); 1211 1212 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1213 OptionSetter setter = new OptionSetter(deqpTest); 1214 setter.setOptionValue("deqp-screen-rotation", rotation); 1215 1216 deqpTest.setDevice(mockDevice); 1217 1218 int version = 3 << 16; 1219 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1220 .andReturn(Integer.toString(version)).atLeastOnce(); 1221 1222 if (!rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED)) { 1223 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 1224 .andReturn(featureString); 1225 } 1226 1227 final boolean isPortraitOrientation = 1228 rotation.equals(BatchRunConfiguration.ROTATION_PORTRAIT) || 1229 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT); 1230 final boolean isLandscapeOrientation = 1231 rotation.equals(BatchRunConfiguration.ROTATION_LANDSCAPE) || 1232 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE); 1233 final boolean executable = 1234 rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED) || 1235 (isPortraitOrientation && 1236 featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) || 1237 (isLandscapeOrientation && 1238 featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE)); 1239 1240 if (executable) { 1241 expectRenderConfigQuery(mockDevice, String.format( 1242 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s " 1243 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1244 + "--deqp-gl-minor-version=0", rotation)); 1245 1246 String commandLine = String.format( 1247 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1248 + "--deqp-screen-rotation=%s " 1249 + "--deqp-surface-type=window " 1250 + "--deqp-log-images=disable " 1251 + "--deqp-watchdog=enable", 1252 APP_DIR + CASE_LIST_FILE_NAME, rotation); 1253 1254 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 1255 output); 1256 } 1257 1258 mockListener.testRunStarted(getTestId(deqpTest), 1); 1259 EasyMock.expectLastCall().once(); 1260 1261 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 1262 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1263 andReturn("").once(); 1264 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1265 andReturn("").once(); 1266 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1267 andReturn("").once(); 1268 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1269 andReturn("").once(); 1270 1271 mockListener.testStarted(EasyMock.eq(testId)); 1272 EasyMock.expectLastCall().once(); 1273 1274 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 1275 EasyMock.expectLastCall().once(); 1276 1277 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1278 EasyMock.expectLastCall().once(); 1279 1280 EasyMock.replay(mockDevice, mockIDevice); 1281 EasyMock.replay(mockListener); 1282 deqpTest.run(mockListener); 1283 EasyMock.verify(mockListener); 1284 EasyMock.verify(mockDevice, mockIDevice); 1285 } 1286 1287 /** 1288 * Test OpeGL ES3 tests on device with OpenGL ES2. 1289 */ testRun_require30DeviceVersion20()1290 public void testRun_require30DeviceVersion20() throws Exception { 1291 testGlesVersion(3, 0, 2, 0); 1292 } 1293 1294 /** 1295 * Test OpeGL ES3.1 tests on device with OpenGL ES2. 1296 */ testRun_require31DeviceVersion20()1297 public void testRun_require31DeviceVersion20() throws Exception { 1298 testGlesVersion(3, 1, 2, 0); 1299 } 1300 1301 /** 1302 * Test OpeGL ES3 tests on device with OpenGL ES3. 1303 */ testRun_require30DeviceVersion30()1304 public void testRun_require30DeviceVersion30() throws Exception { 1305 testGlesVersion(3, 0, 3, 0); 1306 } 1307 1308 /** 1309 * Test OpeGL ES3.1 tests on device with OpenGL ES3. 1310 */ testRun_require31DeviceVersion30()1311 public void testRun_require31DeviceVersion30() throws Exception { 1312 testGlesVersion(3, 1, 3, 0); 1313 } 1314 1315 /** 1316 * Test OpeGL ES3 tests on device with OpenGL ES3.1. 1317 */ testRun_require30DeviceVersion31()1318 public void testRun_require30DeviceVersion31() throws Exception { 1319 testGlesVersion(3, 0, 3, 1); 1320 } 1321 1322 /** 1323 * Test OpeGL ES3.1 tests on device with OpenGL ES3.1. 1324 */ testRun_require31DeviceVersion31()1325 public void testRun_require31DeviceVersion31() throws Exception { 1326 testGlesVersion(3, 1, 3, 1); 1327 } 1328 1329 /** 1330 * Test dEQP Pass result code. 1331 */ testRun_resultPass()1332 public void testRun_resultPass() throws Exception { 1333 testResultCode("Pass", true); 1334 } 1335 1336 /** 1337 * Test dEQP Fail result code. 1338 */ testRun_resultFail()1339 public void testRun_resultFail() throws Exception { 1340 testResultCode("Fail", false); 1341 } 1342 1343 /** 1344 * Test dEQP NotSupported result code. 1345 */ testRun_resultNotSupported()1346 public void testRun_resultNotSupported() throws Exception { 1347 testResultCode("NotSupported", true); 1348 } 1349 1350 /** 1351 * Test dEQP QualityWarning result code. 1352 */ testRun_resultQualityWarning()1353 public void testRun_resultQualityWarning() throws Exception { 1354 testResultCode("QualityWarning", true); 1355 } 1356 1357 /** 1358 * Test dEQP CompatibilityWarning result code. 1359 */ testRun_resultCompatibilityWarning()1360 public void testRun_resultCompatibilityWarning() throws Exception { 1361 testResultCode("CompatibilityWarning", true); 1362 } 1363 1364 /** 1365 * Test dEQP ResourceError result code. 1366 */ testRun_resultResourceError()1367 public void testRun_resultResourceError() throws Exception { 1368 testResultCode("ResourceError", false); 1369 } 1370 1371 /** 1372 * Test dEQP InternalError result code. 1373 */ testRun_resultInternalError()1374 public void testRun_resultInternalError() throws Exception { 1375 testResultCode("InternalError", false); 1376 } 1377 1378 /** 1379 * Test dEQP Crash result code. 1380 */ testRun_resultCrash()1381 public void testRun_resultCrash() throws Exception { 1382 testResultCode("Crash", false); 1383 } 1384 1385 /** 1386 * Test dEQP Timeout result code. 1387 */ testRun_resultTimeout()1388 public void testRun_resultTimeout() throws Exception { 1389 testResultCode("Timeout", false); 1390 } 1391 /** 1392 * Test dEQP Orientation 1393 */ testRun_orientationLandscape()1394 public void testRun_orientationLandscape() throws Exception { 1395 testOrientation("90", ALL_FEATURES); 1396 } 1397 1398 /** 1399 * Test dEQP Orientation 1400 */ testRun_orientationPortrait()1401 public void testRun_orientationPortrait() throws Exception { 1402 testOrientation("0", ALL_FEATURES); 1403 } 1404 1405 /** 1406 * Test dEQP Orientation 1407 */ testRun_orientationReverseLandscape()1408 public void testRun_orientationReverseLandscape() throws Exception { 1409 testOrientation("270", ALL_FEATURES); 1410 } 1411 1412 /** 1413 * Test dEQP Orientation 1414 */ testRun_orientationReversePortrait()1415 public void testRun_orientationReversePortrait() throws Exception { 1416 testOrientation("180", ALL_FEATURES); 1417 } 1418 1419 /** 1420 * Test dEQP Orientation 1421 */ testRun_orientationUnspecified()1422 public void testRun_orientationUnspecified() throws Exception { 1423 testOrientation("unspecified", ALL_FEATURES); 1424 } 1425 1426 /** 1427 * Test dEQP Orientation with limited features 1428 */ testRun_orientationUnspecifiedLimitedFeatures()1429 public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception { 1430 testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES); 1431 } 1432 1433 /** 1434 * Test dEQP Orientation with limited features 1435 */ testRun_orientationLandscapeLimitedFeatures()1436 public void testRun_orientationLandscapeLimitedFeatures() throws Exception { 1437 testOrientation("90", ONLY_LANDSCAPE_FEATURES); 1438 } 1439 1440 /** 1441 * Test dEQP Orientation with limited features 1442 */ testRun_orientationPortraitLimitedFeatures()1443 public void testRun_orientationPortraitLimitedFeatures() throws Exception { 1444 testOrientation("0", ONLY_LANDSCAPE_FEATURES); 1445 } 1446 1447 /** 1448 * Test dEQP unsupported pixel format 1449 */ testRun_unsupportedPixelFormat()1450 public void testRun_unsupportedPixelFormat() throws Exception { 1451 final String pixelFormat = "rgba5658d16m4"; 1452 final TestDescription testId = new TestDescription("dEQP-GLES3.pixelformat", "test"); 1453 1454 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1455 ITestInvocationListener mockListener 1456 = EasyMock.createStrictMock(ITestInvocationListener.class); 1457 1458 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1459 tests.add(testId); 1460 1461 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1462 OptionSetter setter = new OptionSetter(deqpTest); 1463 setter.setOptionValue("deqp-gl-config-name", pixelFormat); 1464 1465 deqpTest.setDevice(mockDevice); 1466 1467 int version = 3 << 16; 1468 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1469 .andReturn(Integer.toString(version)).atLeastOnce(); 1470 1471 expectRenderConfigQueryAndReturn(mockDevice, String.format( 1472 "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified " 1473 + "--deqp-surface-type=window " 1474 + "--deqp-gl-major-version=3 " 1475 + "--deqp-gl-minor-version=0", pixelFormat), "No"); 1476 1477 mockListener.testRunStarted(getTestId(deqpTest), 1); 1478 EasyMock.expectLastCall().once(); 1479 1480 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 1481 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1482 andReturn("").once(); 1483 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1484 andReturn("").once(); 1485 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1486 andReturn("").once(); 1487 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1488 andReturn("").once(); 1489 1490 mockListener.testStarted(EasyMock.eq(testId)); 1491 EasyMock.expectLastCall().once(); 1492 1493 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 1494 EasyMock.expectLastCall().once(); 1495 1496 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1497 EasyMock.expectLastCall().once(); 1498 1499 EasyMock.replay(mockDevice); 1500 EasyMock.replay(mockListener); 1501 deqpTest.run(mockListener); 1502 EasyMock.verify(mockListener); 1503 EasyMock.verify(mockDevice); 1504 } 1505 1506 /** 1507 * Test interface to mock Tradefed device types. 1508 */ 1509 public static interface RecoverableTestDevice extends ITestDevice, IManagedTestDevice { 1510 } 1511 1512 private static enum RecoveryEvent { 1513 PROGRESS, 1514 FAIL_CONNECTION_REFUSED, 1515 FAIL_LINK_KILLED, 1516 } 1517 runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)1518 private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events) 1519 throws DeviceNotAvailableException { 1520 for (RecoveryEvent event : events) { 1521 switch (event) { 1522 case PROGRESS: 1523 recovery.onExecutionProgressed(); 1524 break; 1525 case FAIL_CONNECTION_REFUSED: 1526 recovery.recoverConnectionRefused(); 1527 break; 1528 case FAIL_LINK_KILLED: 1529 recovery.recoverComLinkKilled(); 1530 break; 1531 } 1532 } 1533 } 1534 setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider)1535 private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) { 1536 mockSleepProvider.sleep(EasyMock.gt(0)); 1537 EasyMock.expectLastCall().once(); 1538 } 1539 setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider)1540 private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, 1541 DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException { 1542 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1543 andReturn("root 1234 com.drawelement.deqp").once(); 1544 1545 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1546 andReturn("").once(); 1547 1548 // Recovery checks if kill failed 1549 mockSleepProvider.sleep(EasyMock.gt(0)); 1550 EasyMock.expectLastCall().once(); 1551 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1552 andReturn("").once(); 1553 } 1554 setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)1555 private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice) 1556 throws DeviceNotAvailableException { 1557 EasyMock.expect(mockDevice.recoverDevice()).andReturn(true).once(); 1558 } 1559 setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)1560 private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice) 1561 throws DeviceNotAvailableException { 1562 mockDevice.reboot(); 1563 EasyMock.expectLastCall().once(); 1564 } 1565 setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)1566 private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, 1567 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 1568 throws DeviceNotAvailableException { 1569 switch (numConsecutiveErrors) { 1570 case 0: 1571 case 1: 1572 setRecoveryExpectationRecovery(mockDevice); 1573 return 2; 1574 case 2: 1575 setRecoveryExpectationReboot(mockDevice); 1576 return 3; 1577 default: 1578 return 4; 1579 } 1580 } 1581 setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)1582 private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, 1583 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 1584 throws DeviceNotAvailableException { 1585 switch (numConsecutiveErrors) { 1586 case 0: 1587 setRecoveryExpectationWait(mockSleepProvider); 1588 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1589 return 1; 1590 case 1: 1591 setRecoveryExpectationRecovery(mockDevice); 1592 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1593 return 2; 1594 case 2: 1595 setRecoveryExpectationReboot(mockDevice); 1596 return 3; 1597 default: 1598 return 4; 1599 } 1600 } 1601 setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)1602 private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, 1603 DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events) 1604 throws DeviceNotAvailableException { 1605 int numConsecutiveErrors = 0; 1606 for (RecoveryEvent event : events) { 1607 switch (event) { 1608 case PROGRESS: 1609 numConsecutiveErrors = 0; 1610 break; 1611 case FAIL_CONNECTION_REFUSED: 1612 numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice, 1613 mockSleepProvider, numConsecutiveErrors); 1614 break; 1615 case FAIL_LINK_KILLED: 1616 numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice, 1617 mockSleepProvider, numConsecutiveErrors); 1618 break; 1619 } 1620 } 1621 } 1622 1623 /** 1624 * Test dEQP runner recovery state machine. 1625 */ testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)1626 private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern) 1627 throws Exception { 1628 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 1629 IMocksControl orderedControl = EasyMock.createStrictControl(); 1630 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 1631 EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL"); 1632 DeqpTestRunner.ISleepProvider mockSleepProvider = 1633 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 1634 1635 setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern); 1636 1637 orderedControl.replay(); 1638 1639 recovery.setDevice(mockDevice); 1640 recovery.setSleepProvider(mockSleepProvider); 1641 try { 1642 runRecoveryWithPattern(recovery, pattern); 1643 if (!expectSuccess) { 1644 fail("Expected DeviceNotAvailableException"); 1645 } 1646 } catch (DeviceNotAvailableException ex) { 1647 if (expectSuccess) { 1648 fail("Did not expect DeviceNotAvailableException"); 1649 } 1650 } 1651 1652 orderedControl.verify(); 1653 } 1654 1655 // basic patterns 1656 testRecovery_NoEvents()1657 public void testRecovery_NoEvents() throws Exception { 1658 testRecoveryWithPattern(true); 1659 } 1660 testRecovery_AllOk()1661 public void testRecovery_AllOk() throws Exception { 1662 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS); 1663 } 1664 1665 // conn fail patterns 1666 testRecovery_OneConnectionFailureBegin()1667 public void testRecovery_OneConnectionFailureBegin() throws Exception { 1668 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1669 RecoveryEvent.PROGRESS); 1670 } 1671 testRecovery_TwoConnectionFailuresBegin()1672 public void testRecovery_TwoConnectionFailuresBegin() throws Exception { 1673 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1674 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1675 } 1676 testRecovery_ThreeConnectionFailuresBegin()1677 public void testRecovery_ThreeConnectionFailuresBegin() throws Exception { 1678 testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1679 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED); 1680 } 1681 testRecovery_OneConnectionFailureMid()1682 public void testRecovery_OneConnectionFailureMid() throws Exception { 1683 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1684 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1685 } 1686 testRecovery_TwoConnectionFailuresMid()1687 public void testRecovery_TwoConnectionFailuresMid() throws Exception { 1688 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1689 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1690 RecoveryEvent.PROGRESS); 1691 } 1692 testRecovery_ThreeConnectionFailuresMid()1693 public void testRecovery_ThreeConnectionFailuresMid() throws Exception { 1694 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, 1695 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1696 RecoveryEvent.FAIL_CONNECTION_REFUSED); 1697 } 1698 1699 // link fail patterns 1700 testRecovery_OneLinkFailureBegin()1701 public void testRecovery_OneLinkFailureBegin() throws Exception { 1702 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1703 RecoveryEvent.PROGRESS); 1704 } 1705 testRecovery_TwoLinkFailuresBegin()1706 public void testRecovery_TwoLinkFailuresBegin() throws Exception { 1707 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1708 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1709 } 1710 testRecovery_ThreeLinkFailuresBegin()1711 public void testRecovery_ThreeLinkFailuresBegin() throws Exception { 1712 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1713 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1714 RecoveryEvent.PROGRESS); 1715 } 1716 testRecovery_FourLinkFailuresBegin()1717 public void testRecovery_FourLinkFailuresBegin() throws Exception { 1718 testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED, 1719 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1720 RecoveryEvent.FAIL_LINK_KILLED); 1721 } 1722 testRecovery_OneLinkFailureMid()1723 public void testRecovery_OneLinkFailureMid() throws Exception { 1724 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1725 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1726 } 1727 testRecovery_TwoLinkFailuresMid()1728 public void testRecovery_TwoLinkFailuresMid() throws Exception { 1729 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1730 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1731 RecoveryEvent.PROGRESS); 1732 } 1733 testRecovery_ThreeLinkFailuresMid()1734 public void testRecovery_ThreeLinkFailuresMid() throws Exception { 1735 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1736 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1737 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1738 } 1739 testRecovery_FourLinkFailuresMid()1740 public void testRecovery_FourLinkFailuresMid() throws Exception { 1741 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1742 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1743 RecoveryEvent.FAIL_LINK_KILLED); 1744 } 1745 1746 // mixed patterns 1747 testRecovery_MixedFailuresProgressBetween()1748 public void testRecovery_MixedFailuresProgressBetween() throws Exception { 1749 testRecoveryWithPattern(true, 1750 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1751 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1752 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1753 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1754 RecoveryEvent.PROGRESS); 1755 } 1756 testRecovery_MixedFailuresNoProgressBetween()1757 public void testRecovery_MixedFailuresNoProgressBetween() throws Exception { 1758 testRecoveryWithPattern(true, 1759 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1760 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED, 1761 RecoveryEvent.PROGRESS); 1762 } 1763 1764 /** 1765 * Test recovery if process cannot be killed 1766 */ testRecovery_unkillableProcess()1767 public void testRecovery_unkillableProcess () throws Exception { 1768 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 1769 IMocksControl orderedControl = EasyMock.createStrictControl(); 1770 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 1771 DeqpTestRunner.ISleepProvider mockSleepProvider = 1772 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 1773 1774 // recovery attempts to kill the process after a timeout 1775 mockSleepProvider.sleep(EasyMock.gt(0)); 1776 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1777 andReturn("root 1234 com.drawelement.deqp").once(); 1778 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1779 andReturn("").once(); 1780 1781 // Recovery checks if kill failed 1782 mockSleepProvider.sleep(EasyMock.gt(0)); 1783 EasyMock.expectLastCall().once(); 1784 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1785 andReturn("root 1234 com.drawelement.deqp").once(); 1786 1787 // Recovery resets the connection 1788 EasyMock.expect(mockDevice.recoverDevice()).andReturn(true); 1789 1790 // and attempts to kill the process again 1791 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1792 andReturn("root 1234 com.drawelement.deqp").once(); 1793 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1794 andReturn("").once(); 1795 1796 // Recovery checks if kill failed 1797 mockSleepProvider.sleep(EasyMock.gt(0)); 1798 EasyMock.expectLastCall().once(); 1799 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1800 andReturn("root 1234 com.drawelement.deqp").once(); 1801 1802 // recovery reboots the device 1803 mockDevice.reboot(); 1804 EasyMock.expectLastCall().once(); 1805 1806 orderedControl.replay(); 1807 recovery.setDevice(mockDevice); 1808 recovery.setSleepProvider(mockSleepProvider); 1809 recovery.recoverComLinkKilled(); 1810 orderedControl.verify(); 1811 } 1812 1813 /** 1814 * Test external interruption before batch run. 1815 */ testInterrupt_killBeforeBatch()1816 public void testInterrupt_killBeforeBatch() throws Exception { 1817 final TestDescription testId = new TestDescription("dEQP-GLES3.interrupt", "test"); 1818 1819 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1820 tests.add(testId); 1821 1822 ITestInvocationListener mockListener 1823 = EasyMock.createStrictMock(ITestInvocationListener.class); 1824 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1825 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1826 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 1827 1828 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1829 1830 deqpTest.setDevice(mockDevice); 1831 deqpTest.setRunUtil(mockRunUtil); 1832 1833 int version = 3 << 16; 1834 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1835 .andReturn(Integer.toString(version)).atLeastOnce(); 1836 1837 expectRenderConfigQuery(mockDevice, 1838 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 1839 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1840 + "--deqp-gl-minor-version=0"); 1841 1842 mockRunUtil.sleep(0); 1843 EasyMock.expectLastCall().andThrow(new RunInterruptedException( 1844 "message", InfraErrorIdentifier.TRADEFED_SHUTTING_DOWN)); 1845 1846 mockListener.testRunStarted(getTestId(deqpTest), 1); 1847 EasyMock.expectLastCall().once(); 1848 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1849 andReturn("").once(); 1850 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1851 andReturn("").once(); 1852 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject()); 1853 EasyMock.expectLastCall().once(); 1854 1855 EasyMock.replay(mockDevice, mockIDevice); 1856 EasyMock.replay(mockListener); 1857 EasyMock.replay(mockRunUtil); 1858 try { 1859 deqpTest.run(mockListener); 1860 fail("expected RunInterruptedException"); 1861 } catch (RunInterruptedException ex) { 1862 // expected 1863 } 1864 EasyMock.verify(mockRunUtil); 1865 EasyMock.verify(mockListener); 1866 EasyMock.verify(mockDevice, mockIDevice); 1867 } 1868 runShardedTest(TestDescription[] testIds, ArrayList<ArrayList<TestDescription>> testsForShard)1869 private void runShardedTest(TestDescription[] testIds, 1870 ArrayList<ArrayList<TestDescription>> testsForShard) throws Exception { 1871 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1872 for (TestDescription id : testIds) tests.add(id); 1873 1874 DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests, mTestsDir); 1875 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split(); 1876 1877 for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) { 1878 DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex); 1879 shard.setBuildHelper(getMockBuildHelper(mTestsDir)); 1880 1881 ArrayList<TestDescription> shardTests = testsForShard.get(shardIndex); 1882 1883 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1884 ITestInvocationListener mockListener 1885 = EasyMock.createStrictMock(ITestInvocationListener.class); 1886 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1887 int version = 3 << 16; 1888 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1889 .andReturn(Integer.toString(version)).atLeastOnce(); 1890 1891 mockListener.testRunStarted(getTestId(shard), shardTests.size()); 1892 EasyMock.expectLastCall().once(); 1893 1894 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 1895 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1896 andReturn("").once(); 1897 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1898 andReturn("").once(); 1899 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1900 andReturn("").once(); 1901 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1902 andReturn("").once(); 1903 1904 expectRenderConfigQuery(mockDevice, 3, 0); 1905 1906 String testOut = buildTestProcessOutput(shardTests); 1907 // NOTE: This assumes that there won't be multiple batches per shard! 1908 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); 1909 1910 for (int i = 0; i < shardTests.size(); i++) { 1911 mockListener.testStarted(EasyMock.eq(shardTests.get(i))); 1912 EasyMock.expectLastCall().once(); 1913 1914 mockListener.testEnded(EasyMock.eq(shardTests.get(i)), 1915 EasyMock.<HashMap<String, Metric>>notNull()); 1916 1917 EasyMock.expectLastCall().once(); 1918 } 1919 1920 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1921 EasyMock.expectLastCall().once(); 1922 1923 EasyMock.replay(mockDevice, mockIDevice); 1924 EasyMock.replay(mockListener); 1925 1926 shard.setDevice(mockDevice); 1927 shard.run(mockListener); 1928 1929 EasyMock.verify(mockListener); 1930 EasyMock.verify(mockDevice, mockIDevice); 1931 } 1932 } 1933 testSharding_smallTrivial()1934 public void testSharding_smallTrivial() throws Exception { 1935 final TestDescription[] testIds = { 1936 new TestDescription("dEQP-GLES3.info", "vendor"), 1937 new TestDescription("dEQP-GLES3.info", "renderer"), 1938 new TestDescription("dEQP-GLES3.info", "version"), 1939 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 1940 new TestDescription("dEQP-GLES3.info", "extensions"), 1941 new TestDescription("dEQP-GLES3.info", "render_target") 1942 }; 1943 ArrayList<ArrayList<TestDescription>> shardedTests = new ArrayList<>(); 1944 ArrayList<TestDescription> shardOne = new ArrayList<>(); 1945 for (int i = 0; i < testIds.length; i++) { 1946 shardOne.add(testIds[i]); 1947 } 1948 shardedTests.add(shardOne); 1949 runShardedTest(testIds, shardedTests); 1950 } 1951 testSharding_twoShards()1952 public void testSharding_twoShards() throws Exception { 1953 final int TEST_COUNT = 1237; 1954 final int SHARD_SIZE = 1000; 1955 1956 ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT); 1957 for (int i = 0; i < TEST_COUNT; i++) { 1958 testIds.add(new TestDescription("dEQP-GLES3.funny.group", String.valueOf(i))); 1959 } 1960 1961 ArrayList<ArrayList<TestDescription>> shardedTests = new ArrayList<>(); 1962 ArrayList<TestDescription> shard = new ArrayList<>(); 1963 for (int i = 0; i < testIds.size(); i++) { 1964 if (i == SHARD_SIZE) { 1965 shardedTests.add(shard); 1966 shard = new ArrayList<>(); 1967 } 1968 shard.add(testIds.get(i)); 1969 } 1970 shardedTests.add(shard); 1971 runShardedTest(testIds.toArray(new TestDescription[testIds.size()]), shardedTests); 1972 } 1973 testSharding_empty()1974 public void testSharding_empty() throws Exception { 1975 DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList<TestDescription>(), mTestsDir); 1976 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split(); 1977 // Returns null when cannot be sharded. 1978 assertNull(shards); 1979 } 1980 1981 /** 1982 * Test external interruption in testFailed(). 1983 */ testInterrupt_killReportTestFailed()1984 public void testInterrupt_killReportTestFailed() throws Exception { 1985 final TestDescription testId = new TestDescription("dEQP-GLES3.interrupt", "test"); 1986 final String testPath = "dEQP-GLES3.interrupt.test"; 1987 final String testTrie = "{dEQP-GLES3{interrupt{test}}}"; 1988 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1989 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1990 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1991 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1992 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1993 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1994 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1995 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1996 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1997 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1998 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1999 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2000 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 2001 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2002 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2003 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 2004 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2005 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n" 2006 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n" 2007 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2008 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2009 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2010 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2011 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 2012 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2013 + "INSTRUMENTATION_CODE: 0\r\n"; 2014 2015 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2016 tests.add(testId); 2017 2018 ITestInvocationListener mockListener 2019 = EasyMock.createStrictMock(ITestInvocationListener.class); 2020 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2021 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 2022 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 2023 2024 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2025 2026 deqpTest.setDevice(mockDevice); 2027 deqpTest.setRunUtil(mockRunUtil); 2028 2029 int version = 3 << 16; 2030 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 2031 .andReturn(Integer.toString(version)).atLeastOnce(); 2032 2033 expectRenderConfigQuery(mockDevice, 2034 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 2035 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 2036 + "--deqp-gl-minor-version=0"); 2037 2038 mockRunUtil.sleep(0); 2039 EasyMock.expectLastCall().once(); 2040 2041 String commandLine = String.format( 2042 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2043 + "--deqp-screen-rotation=unspecified " 2044 + "--deqp-surface-type=window " 2045 + "--deqp-log-images=disable " 2046 + "--deqp-watchdog=enable", 2047 APP_DIR + CASE_LIST_FILE_NAME); 2048 2049 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 2050 output); 2051 2052 mockListener.testRunStarted(getTestId(deqpTest), 1); 2053 EasyMock.expectLastCall().once(); 2054 2055 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 2056 andReturn("").once(); 2057 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 2058 andReturn("").once(); 2059 2060 mockListener.testStarted(EasyMock.eq(testId)); 2061 EasyMock.expectLastCall().once(); 2062 2063 mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull()); 2064 EasyMock.expectLastCall().andThrow(new RunInterruptedException( 2065 "message", InfraErrorIdentifier.TRADEFED_SHUTTING_DOWN)); 2066 2067 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject()); 2068 EasyMock.expectLastCall().once(); 2069 EasyMock.replay(mockDevice, mockIDevice); 2070 EasyMock.replay(mockListener); 2071 EasyMock.replay(mockRunUtil); 2072 try { 2073 deqpTest.run(mockListener); 2074 fail("expected RunInterruptedException"); 2075 } catch (RunInterruptedException ex) { 2076 // expected 2077 } 2078 EasyMock.verify(mockRunUtil); 2079 EasyMock.verify(mockListener); 2080 EasyMock.verify(mockDevice, mockIDevice); 2081 } 2082 testRuntimeHint_optionSet()2083 public void testRuntimeHint_optionSet() throws Exception { 2084 /* MultiLineReceiver expects "\r\n" line ending. */ 2085 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 2086 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 2087 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 2088 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2089 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 2090 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 2091 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 2092 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2093 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 2094 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 2095 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 2096 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2097 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 2098 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2099 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2100 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 2101 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2102 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2103 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2104 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2105 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2106 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2107 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2108 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2109 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 2110 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2111 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2112 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2113 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2114 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2115 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2116 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2117 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2118 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 2119 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2120 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2121 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2122 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2123 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2124 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2125 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2126 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2127 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 2128 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2129 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2130 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2131 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2132 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2133 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2134 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2135 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2136 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 2137 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2138 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2139 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2140 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2141 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2142 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2143 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2144 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2145 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 2146 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2147 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2148 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2149 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2150 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2151 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2152 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2153 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 2154 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2155 + "INSTRUMENTATION_CODE: 0\r\n"; 2156 2157 final TestDescription[] testIds = { 2158 new TestDescription("dEQP-GLES3.info", "vendor"), 2159 new TestDescription("dEQP-GLES3.info", "renderer"), 2160 new TestDescription("dEQP-GLES3.info", "version"), 2161 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 2162 new TestDescription("dEQP-GLES3.info", "extensions"), 2163 new TestDescription("dEQP-GLES3.info", "render_target") 2164 }; 2165 2166 final String[] testPaths = { 2167 "dEQP-GLES3.info.vendor", 2168 "dEQP-GLES3.info.renderer", 2169 "dEQP-GLES3.info.version", 2170 "dEQP-GLES3.info.shading_language_version", 2171 "dEQP-GLES3.info.extensions", 2172 "dEQP-GLES3.info.render_target" 2173 }; 2174 2175 final String testTrie 2176 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 2177 2178 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2179 ITestInvocationListener mockListener 2180 = EasyMock.createStrictMock(ITestInvocationListener.class); 2181 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 2182 2183 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2184 2185 for (TestDescription id : testIds) { 2186 tests.add(id); 2187 } 2188 2189 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2190 OptionSetter setter = new OptionSetter(deqpTest); 2191 final long runtimeMs = 123456; 2192 setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs)); 2193 assertEquals("Wrong expected runtime - option not passed cleanly", runtimeMs, deqpTest.getRuntimeHint()); 2194 2195 // Try running the tests as well. The unit tests do not set the hint be default, 2196 // so that case is covered. 2197 2198 int version = 3 << 16; 2199 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 2200 .andReturn(Integer.toString(version)).atLeastOnce(); 2201 2202 expectRenderConfigQuery(mockDevice, 3, 0); 2203 2204 String commandLine = String.format( 2205 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2206 + "--deqp-screen-rotation=unspecified " 2207 + "--deqp-surface-type=window " 2208 + "--deqp-log-images=disable " 2209 + "--deqp-watchdog=enable", 2210 APP_DIR + CASE_LIST_FILE_NAME); 2211 2212 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 2213 2214 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 2215 EasyMock.expectLastCall().once(); 2216 2217 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 2218 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 2219 andReturn("").once(); 2220 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 2221 andReturn("").once(); 2222 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 2223 andReturn("").once(); 2224 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 2225 andReturn("").once(); 2226 2227 for (int i = 0; i < testPaths.length; i++) { 2228 mockListener.testStarted(EasyMock.eq(testIds[i])); 2229 EasyMock.expectLastCall().once(); 2230 2231 mockListener.testEnded(EasyMock.eq(testIds[i]), 2232 EasyMock.<HashMap<String, Metric>>notNull()); 2233 2234 EasyMock.expectLastCall().once(); 2235 } 2236 2237 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 2238 EasyMock.expectLastCall().once(); 2239 2240 EasyMock.replay(mockDevice, mockIDevice); 2241 EasyMock.replay(mockListener); 2242 2243 deqpTest.setDevice(mockDevice); 2244 deqpTest.run(mockListener); 2245 2246 EasyMock.verify(mockListener); 2247 EasyMock.verify(mockDevice, mockIDevice); 2248 } 2249 testRuntimeHint_optionSetSharded()2250 public void testRuntimeHint_optionSetSharded() throws Exception { 2251 final int TEST_COUNT = 1237; 2252 final int SHARD_SIZE = 1000; 2253 2254 ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT); 2255 for (int i = 0; i < TEST_COUNT; i++) { 2256 testIds.add(new TestDescription("dEQP-GLES3.funny.group", String.valueOf(i))); 2257 } 2258 2259 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir); 2260 OptionSetter setter = new OptionSetter(deqpTest); 2261 final long fullRuntimeMs = testIds.size()*100; 2262 setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs)); 2263 2264 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)deqpTest.split(); 2265 assertEquals("First shard's time not proportional to test count", 2266 (fullRuntimeMs*SHARD_SIZE)/TEST_COUNT, 2267 ((IRuntimeHintProvider)shards.get(0)).getRuntimeHint()); 2268 assertEquals("Second shard's time not proportional to test count", 2269 (fullRuntimeMs*(TEST_COUNT-SHARD_SIZE))/TEST_COUNT, 2270 ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint()); 2271 } 2272 testRuntimeHint_optionNotSet()2273 public void testRuntimeHint_optionNotSet() throws Exception { 2274 final TestDescription[] testIds = { 2275 new TestDescription("dEQP-GLES3.info", "vendor"), 2276 new TestDescription("dEQP-GLES3.info", "renderer"), 2277 new TestDescription("dEQP-GLES3.info", "version"), 2278 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 2279 new TestDescription("dEQP-GLES3.info", "extensions"), 2280 new TestDescription("dEQP-GLES3.info", "render_target") 2281 }; 2282 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2283 2284 for (TestDescription id : testIds) { 2285 tests.add(id); 2286 } 2287 2288 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2289 2290 long runtime = deqpTest.getRuntimeHint(); 2291 assertTrue("Runtime for tests must be positive", runtime > 0); 2292 assertTrue("Runtime for tests must be reasonable", runtime < (1000 * 10)); // Must be done in 10s 2293 } 2294 2295 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String output)2296 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 2297 final String output) throws Exception { 2298 String cmd = String.format( 2299 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2300 + "--deqp-screen-rotation=unspecified " 2301 + "--deqp-surface-type=window " 2302 + "--deqp-log-images=disable " 2303 + "--deqp-watchdog=enable", 2304 APP_DIR + CASE_LIST_FILE_NAME); 2305 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, cmd, output); 2306 } 2307 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String testTrie, final String cmd, final String output)2308 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 2309 final String testTrie, final String cmd, final String output) throws Exception { 2310 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + CASE_LIST_FILE_NAME))) 2311 .andReturn("").once(); 2312 2313 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + LOG_FILE_NAME))) 2314 .andReturn("").once(); 2315 2316 if (testTrie == null) { 2317 mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(APP_DIR + CASE_LIST_FILE_NAME)); 2318 } 2319 else { 2320 mockDevice.pushString(testTrie + "\n", APP_DIR + CASE_LIST_FILE_NAME); 2321 } 2322 EasyMock.expectLastCall().andReturn(true).once(); 2323 2324 String command = String.format( 2325 "am instrument %s -w -e deqpLogFilename \"%s\" -e deqpCmdLine \"%s\" " 2326 + "-e deqpLogData \"%s\" %s", 2327 AbiUtils.createAbiFlag(ABI.getName()), APP_DIR + LOG_FILE_NAME, cmd, false, 2328 INSTRUMENTATION_NAME); 2329 2330 EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice); 2331 mockIDevice.executeShellCommand(EasyMock.eq(command), 2332 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(), 2333 EasyMock.isA(TimeUnit.class)); 2334 2335 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 2336 @Override 2337 public Object answer() { 2338 IShellOutputReceiver receiver 2339 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 2340 2341 receiver.addOutput(output.getBytes(), 0, output.length()); 2342 receiver.flush(); 2343 2344 return null; 2345 } 2346 }); 2347 } 2348 writeStringsToFile(File target, Set<String> strings)2349 static private void writeStringsToFile(File target, Set<String> strings) throws IOException { 2350 try (PrintWriter out = new PrintWriter(new FileWriter(target))) { 2351 out.print(String.join(System.lineSeparator(), strings)); 2352 out.println(); 2353 } 2354 } 2355 addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option)2356 private void addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option) 2357 throws IOException, ConfigurationException { 2358 String filterFile = option + ".txt"; 2359 writeStringsToFile(new File(mTestsDir, filterFile), filters); 2360 OptionSetter setter = new OptionSetter(test); 2361 setter.setOptionValue(option, filterFile); 2362 } 2363 testIncludeFilterFile()2364 public void testIncludeFilterFile() throws Exception { 2365 final TestDescription[] testIds = { 2366 new TestDescription("dEQP-GLES3.missing", "no"), 2367 new TestDescription("dEQP-GLES3.missing", "nope"), 2368 new TestDescription("dEQP-GLES3.missing", "donotwant"), 2369 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2370 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2371 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2372 }; 2373 2374 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2375 for (TestDescription id : testIds) { 2376 allTests.add(id); 2377 } 2378 2379 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2380 activeTests.add(testIds[3]); 2381 activeTests.add(testIds[4]); 2382 activeTests.add(testIds[5]); 2383 2384 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2385 2386 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2387 Set<String> includes = new HashSet<>(); 2388 includes.add("dEQP-GLES3.pick_me#*"); 2389 addFilterFileForOption(deqpTest, includes, "include-filter-file"); 2390 testFiltering(deqpTest, expectedTrie, activeTests); 2391 } 2392 testMissingIncludeFilterFile()2393 public void testMissingIncludeFilterFile() throws Exception { 2394 final TestDescription[] testIds = { 2395 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2396 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2397 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2398 }; 2399 2400 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2401 for (TestDescription id : testIds) { 2402 allTests.add(id); 2403 } 2404 2405 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2406 2407 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2408 OptionSetter setter = new OptionSetter(deqpTest); 2409 setter.setOptionValue("include-filter-file", "not-a-file.txt"); 2410 try { 2411 testFiltering(deqpTest, expectedTrie, allTests); 2412 fail("Test execution should have aborted with exception."); 2413 } catch (RuntimeException e) { 2414 } 2415 } 2416 testExcludeFilterFile()2417 public void testExcludeFilterFile() throws Exception { 2418 final TestDescription[] testIds = { 2419 new TestDescription("dEQP-GLES3.missing", "no"), 2420 new TestDescription("dEQP-GLES3.missing", "nope"), 2421 new TestDescription("dEQP-GLES3.missing", "donotwant"), 2422 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2423 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2424 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2425 }; 2426 2427 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2428 for (TestDescription id : testIds) { 2429 allTests.add(id); 2430 } 2431 2432 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2433 activeTests.add(testIds[3]); 2434 activeTests.add(testIds[4]); 2435 activeTests.add(testIds[5]); 2436 2437 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2438 2439 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2440 Set<String> excludes = new HashSet<>(); 2441 excludes.add("dEQP-GLES3.missing#*"); 2442 addFilterFileForOption(deqpTest, excludes, "exclude-filter-file"); 2443 testFiltering(deqpTest, expectedTrie, activeTests); 2444 } 2445 testFilterComboWithFiles()2446 public void testFilterComboWithFiles() throws Exception { 2447 final TestDescription[] testIds = { 2448 new TestDescription("dEQP-GLES3.group1", "footah"), 2449 new TestDescription("dEQP-GLES3.group1", "foo"), 2450 new TestDescription("dEQP-GLES3.group1", "nope"), 2451 new TestDescription("dEQP-GLES3.group1", "nonotwant"), 2452 new TestDescription("dEQP-GLES3.group2", "foo"), 2453 new TestDescription("dEQP-GLES3.group2", "yes"), 2454 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"), 2455 }; 2456 2457 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2458 for (TestDescription id : testIds) { 2459 allTests.add(id); 2460 } 2461 2462 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2463 activeTests.add(testIds[0]); 2464 activeTests.add(testIds[5]); 2465 2466 String expectedTrie = "{dEQP-GLES3{group1{footah}group2{yes}}}"; 2467 2468 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2469 2470 Set<String> includes = new HashSet<>(); 2471 includes.add("dEQP-GLES3.group2#*"); 2472 deqpTest.addAllIncludeFilters(includes); 2473 2474 Set<String> fileIncludes = new HashSet<>(); 2475 fileIncludes.add("dEQP-GLES3.group1#no*"); 2476 fileIncludes.add("dEQP-GLES3.group1#foo*"); 2477 addFilterFileForOption(deqpTest, fileIncludes, "include-filter-file"); 2478 2479 Set<String> fileExcludes = new HashSet<>(); 2480 fileExcludes.add("*foo"); 2481 fileExcludes.add("*thoushallnotpass"); 2482 addFilterFileForOption(deqpTest, fileExcludes, "exclude-filter-file"); 2483 2484 deqpTest.addExcludeFilter("dEQP-GLES3.group1#no*"); 2485 2486 testFiltering(deqpTest, expectedTrie, activeTests); 2487 } 2488 testDotToHashConversionInFilters()2489 public void testDotToHashConversionInFilters() throws Exception { 2490 final TestDescription[] testIds = { 2491 new TestDescription("dEQP-GLES3.missing", "no"), 2492 new TestDescription("dEQP-GLES3.pick_me", "donotwant"), 2493 new TestDescription("dEQP-GLES3.pick_me", "yes") 2494 }; 2495 2496 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2497 for (TestDescription id : testIds) { 2498 allTests.add(id); 2499 } 2500 2501 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2502 activeTests.add(testIds[2]); 2503 2504 String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}"; 2505 2506 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2507 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me.yes"); 2508 testFiltering(deqpTest, expectedTrie, activeTests); 2509 } 2510 } 2511