1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.drawelements.deqp.runner; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.ddmlib.IDevice; 20 import com.android.ddmlib.IShellOutputReceiver; 21 import com.android.tradefed.build.IFolderBuildInfo; 22 import com.android.tradefed.config.ConfigurationException; 23 import com.android.tradefed.config.OptionSetter; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.IManagedTestDevice; 26 import com.android.tradefed.device.ITestDevice; 27 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 28 import com.android.tradefed.result.ITestInvocationListener; 29 import com.android.tradefed.result.TestDescription; 30 import com.android.tradefed.result.error.InfraErrorIdentifier; 31 import com.android.tradefed.testtype.Abi; 32 import com.android.tradefed.testtype.IAbi; 33 import com.android.tradefed.testtype.IRemoteTest; 34 import com.android.tradefed.testtype.IRuntimeHintProvider; 35 import com.android.tradefed.util.AbiUtils; 36 import com.android.tradefed.util.FileUtil; 37 import com.android.tradefed.util.IRunUtil; 38 import com.android.tradefed.util.RunInterruptedException; 39 40 import junit.framework.TestCase; 41 42 import org.easymock.EasyMock; 43 import org.easymock.IAnswer; 44 import org.easymock.IMocksControl; 45 46 import java.io.File; 47 import java.io.FileNotFoundException; 48 import java.io.FileWriter; 49 import java.io.IOException; 50 import java.io.PrintWriter; 51 import java.io.StringReader; 52 import java.io.StringWriter; 53 import java.util.ArrayList; 54 import java.util.Collection; 55 import java.util.HashMap; 56 import java.util.HashSet; 57 import java.util.List; 58 import java.util.Map; 59 import java.util.Set; 60 import java.util.concurrent.TimeUnit; 61 62 /** 63 * Unit tests for {@link DeqpTestRunner}. 64 */ 65 public class DeqpTestRunnerTest extends TestCase { 66 private static final String NAME = "dEQP-GLES3"; 67 private static final IAbi ABI = new Abi("armeabi-v7a", "32"); 68 private static final String APP_DIR = "/sdcard/"; 69 private static final String CASE_LIST_FILE_NAME = "dEQP-TestCaseList.txt"; 70 private static final String LOG_FILE_NAME = "TestLog.qpa"; 71 private static final String INSTRUMENTATION_NAME = 72 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation"; 73 private static final String QUERY_INSTRUMENTATION_NAME = 74 "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation"; 75 private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk"; 76 private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp"; 77 private static final String ONLY_LANDSCAPE_FEATURES = 78 "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE; 79 private static final String ALL_FEATURES = 80 ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT; 81 private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS; 82 83 static { 84 DEFAULT_INSTANCE_ARGS = new ArrayList<>(1); DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>())85 DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>()); 86 DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8"); 87 DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified"); 88 DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window"); 89 } 90 91 private File mTestsDir = null; 92 93 public static class BuildHelperMock extends CompatibilityBuildHelper { 94 private File mTestsDir = null; BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir)95 public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) { 96 super(buildInfo); 97 mTestsDir = testsDir; 98 } 99 @Override getTestsDir()100 public File getTestsDir() throws FileNotFoundException { 101 return mTestsDir; 102 } 103 } 104 105 106 /** 107 * {@inheritDoc} 108 */ 109 @Override setUp()110 protected void setUp() throws Exception { 111 super.setUp(); 112 mTestsDir = FileUtil.createTempDir("deqp-test-cases"); 113 } 114 115 /** 116 * {@inheritDoc} 117 */ 118 @Override tearDown()119 protected void tearDown() throws Exception { 120 FileUtil.recursiveDelete(mTestsDir); 121 super.tearDown(); 122 } 123 buildGlesTestRunner(int majorVersion, int minorVersion, Collection<TestDescription> tests, File testsDir)124 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, 125 int minorVersion, 126 Collection<TestDescription> tests, 127 File testsDir) throws ConfigurationException { 128 StringWriter testlist = new StringWriter(); 129 for (TestDescription test : tests) { 130 testlist.write(test.getClassName() + "." + test.getTestName() + "\n"); 131 } 132 return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString(), testsDir); 133 } 134 getMockBuildHelper(File testsDir)135 private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) { 136 IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class); 137 EasyMock.replay(mockIFolderBuildInfo); 138 return new BuildHelperMock(mockIFolderBuildInfo, testsDir); 139 } 140 buildGlesTestRunner(int majorVersion, int minorVersion, String testlist, File testsDir)141 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, 142 int minorVersion, 143 String testlist, 144 File testsDir) throws ConfigurationException { 145 DeqpTestRunner runner = new DeqpTestRunner(); 146 OptionSetter setter = new OptionSetter(runner); 147 148 String deqpPackage = "dEQP-GLES" + Integer.toString(majorVersion) 149 + (minorVersion > 0 ? Integer.toString(minorVersion) : ""); 150 151 setter.setOptionValue("deqp-package", deqpPackage); 152 setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8"); 153 setter.setOptionValue("deqp-caselist-file", "dummyfile.txt"); 154 setter.setOptionValue("deqp-screen-rotation", "unspecified"); 155 setter.setOptionValue("deqp-surface-type", "window"); 156 157 runner.setCaselistReader(new StringReader(testlist)); 158 runner.setAbi(ABI); 159 runner.setBuildHelper(getMockBuildHelper(testsDir)); 160 161 return runner; 162 } 163 getTestId(DeqpTestRunner runner)164 private static String getTestId(DeqpTestRunner runner) { 165 return AbiUtils.createId(ABI.getName(), runner.getPackageName()); 166 } 167 168 /** 169 * Test version of OpenGL ES. 170 */ testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion)171 private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception { 172 final TestDescription testId = new TestDescription("dEQP-GLES" 173 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 174 + ".info", "version"); 175 176 final String testPath = "dEQP-GLES" 177 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 178 +".info.version"; 179 180 final String testTrie = "{dEQP-GLES" 181 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 182 + "{info{version}}}"; 183 184 final String resultCode = "Pass"; 185 186 /* MultiLineReceiver expects "\r\n" line ending. */ 187 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 188 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 189 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 190 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 191 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 192 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 193 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 194 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 195 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 196 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 197 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 198 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 199 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 200 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 201 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 202 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 203 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 204 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 205 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 206 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 207 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 208 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 209 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 210 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 211 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 212 + "INSTRUMENTATION_CODE: 0\r\n"; 213 214 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 215 ITestInvocationListener mockListener 216 = EasyMock.createStrictMock(ITestInvocationListener.class); 217 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 218 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 219 tests.add(testId); 220 221 DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests, mTestsDir); 222 223 int version = (majorVersion << 16) | minorVersion; 224 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 225 .andReturn(Integer.toString(version)).atLeastOnce(); 226 227 if (majorVersion > requiredMajorVersion 228 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) { 229 230 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 231 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 232 andReturn("").once(); 233 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 234 andReturn("").once(); 235 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 236 andReturn("").once(); 237 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 238 andReturn("").once(); 239 240 expectRenderConfigQuery(mockDevice, requiredMajorVersion, 241 requiredMinorVersion); 242 243 String commandLine = String.format( 244 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 245 + "--deqp-screen-rotation=unspecified " 246 + "--deqp-surface-type=window " 247 + "--deqp-log-images=disable " 248 + "--deqp-watchdog=enable", 249 APP_DIR + CASE_LIST_FILE_NAME); 250 251 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 252 output); 253 } 254 255 mockListener.testRunStarted(getTestId(deqpTest), 1); 256 EasyMock.expectLastCall().once(); 257 258 mockListener.testStarted(EasyMock.eq(testId)); 259 EasyMock.expectLastCall().once(); 260 261 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 262 EasyMock.expectLastCall().once(); 263 264 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 265 EasyMock.expectLastCall().once(); 266 267 EasyMock.replay(mockDevice, mockIDevice); 268 EasyMock.replay(mockListener); 269 270 deqpTest.setDevice(mockDevice); 271 deqpTest.run(mockListener); 272 273 EasyMock.verify(mockListener); 274 EasyMock.verify(mockDevice, mockIDevice); 275 } 276 expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, int minorVersion)277 private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, 278 int minorVersion) throws Exception { 279 expectRenderConfigQuery(mockDevice, 280 String.format("--deqp-gl-config-name=rgba8888d24s8 " 281 + "--deqp-screen-rotation=unspecified " 282 + "--deqp-surface-type=window " 283 + "--deqp-gl-major-version=%d " 284 + "--deqp-gl-minor-version=%d", majorVersion, minorVersion)); 285 } 286 expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)287 private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine) 288 throws Exception { 289 expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes"); 290 } 291 expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, String output)292 private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, 293 String output) throws Exception { 294 final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n" 295 + "INSTRUMENTATION_CODE: 0\r\n"; 296 final String command = String.format( 297 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine " 298 + "\"%s\" %s", 299 AbiUtils.createAbiFlag(ABI.getName()), commandLine, 300 QUERY_INSTRUMENTATION_NAME); 301 302 mockDevice.executeShellCommand(EasyMock.eq(command), 303 EasyMock.<IShellOutputReceiver>notNull()); 304 305 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 306 @Override 307 public Object answer() { 308 IShellOutputReceiver receiver 309 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 310 311 receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length()); 312 receiver.flush(); 313 314 return null; 315 } 316 }); 317 } 318 319 /** 320 * Test that result code produces correctly pass or fail. 321 */ testResultCode(final String resultCode, boolean pass)322 private void testResultCode(final String resultCode, boolean pass) throws Exception { 323 final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version"); 324 final String testPath = "dEQP-GLES3.info.version"; 325 final String testTrie = "{dEQP-GLES3{info{version}}}"; 326 327 /* MultiLineReceiver expects "\r\n" line ending. */ 328 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 329 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 330 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 331 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 332 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 333 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 334 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 335 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 336 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 337 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 338 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 339 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 340 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 341 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 342 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 343 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 344 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 345 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 346 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 347 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 348 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 349 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 350 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 351 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 352 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 353 + "INSTRUMENTATION_CODE: 0\r\n"; 354 355 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 356 ITestInvocationListener mockListener 357 = EasyMock.createStrictMock(ITestInvocationListener.class); 358 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 359 360 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 361 tests.add(testId); 362 363 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 364 365 int version = 3 << 16; 366 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 367 .andReturn(Integer.toString(version)).atLeastOnce(); 368 369 expectRenderConfigQuery(mockDevice, 3, 0); 370 371 String commandLine = String.format( 372 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 373 + "--deqp-screen-rotation=unspecified " 374 + "--deqp-surface-type=window " 375 + "--deqp-log-images=disable " 376 + "--deqp-watchdog=enable", 377 APP_DIR + CASE_LIST_FILE_NAME); 378 379 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 380 381 mockListener.testRunStarted(getTestId(deqpTest), 1); 382 EasyMock.expectLastCall().once(); 383 384 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 385 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 386 andReturn("").once(); 387 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 388 andReturn("").once(); 389 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 390 andReturn("").once(); 391 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 392 andReturn("").once(); 393 394 mockListener.testStarted(EasyMock.eq(testId)); 395 EasyMock.expectLastCall().once(); 396 397 if (!pass) { 398 mockListener.testFailed(testId, 399 "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n" 400 + resultCode + ": Detail" + resultCode); 401 402 EasyMock.expectLastCall().once(); 403 } 404 405 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 406 EasyMock.expectLastCall().once(); 407 408 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 409 EasyMock.expectLastCall().once(); 410 411 EasyMock.replay(mockDevice, mockIDevice); 412 EasyMock.replay(mockListener); 413 414 deqpTest.setDevice(mockDevice); 415 deqpTest.run(mockListener); 416 417 EasyMock.verify(mockListener); 418 EasyMock.verify(mockDevice, mockIDevice); 419 } 420 421 /** 422 * Test running multiple test cases. 423 */ testRun_multipleTests()424 public void testRun_multipleTests() throws Exception { 425 /* MultiLineReceiver expects "\r\n" line ending. */ 426 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 427 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 428 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 429 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 430 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 431 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 432 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 433 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 434 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 435 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 436 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 437 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 438 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 439 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 440 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 441 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 442 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 443 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 444 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 445 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 446 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 447 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 448 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 449 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 450 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 451 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 452 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 453 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 454 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 455 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 456 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 457 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 458 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 459 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 460 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 461 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 462 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 463 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 464 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 465 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 466 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 467 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 468 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 469 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 470 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 471 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 472 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 473 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 474 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 475 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 476 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 477 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 478 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 479 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 480 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 481 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 482 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 483 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 484 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 485 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 486 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 487 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 488 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 489 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 490 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 491 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 492 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 493 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 494 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 495 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 496 + "INSTRUMENTATION_CODE: 0\r\n"; 497 498 final TestDescription[] testIds = { 499 new TestDescription("dEQP-GLES3.info", "vendor"), 500 new TestDescription("dEQP-GLES3.info", "renderer"), 501 new TestDescription("dEQP-GLES3.info", "version"), 502 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 503 new TestDescription("dEQP-GLES3.info", "extensions"), 504 new TestDescription("dEQP-GLES3.info", "render_target") 505 }; 506 507 final String[] testPaths = { 508 "dEQP-GLES3.info.vendor", 509 "dEQP-GLES3.info.renderer", 510 "dEQP-GLES3.info.version", 511 "dEQP-GLES3.info.shading_language_version", 512 "dEQP-GLES3.info.extensions", 513 "dEQP-GLES3.info.render_target" 514 }; 515 516 final String testTrie 517 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 518 519 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 520 ITestInvocationListener mockListener 521 = EasyMock.createStrictMock(ITestInvocationListener.class); 522 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 523 524 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 525 526 for (TestDescription id : testIds) { 527 tests.add(id); 528 } 529 530 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 531 532 int version = 3 << 16; 533 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 534 .andReturn(Integer.toString(version)).atLeastOnce(); 535 536 expectRenderConfigQuery(mockDevice, 3, 0); 537 538 String commandLine = String.format( 539 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 540 + "--deqp-screen-rotation=unspecified " 541 + "--deqp-surface-type=window " 542 + "--deqp-log-images=disable " 543 + "--deqp-watchdog=enable", 544 APP_DIR + CASE_LIST_FILE_NAME); 545 546 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 547 548 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 549 EasyMock.expectLastCall().once(); 550 551 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 552 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 553 andReturn("").once(); 554 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 555 andReturn("").once(); 556 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 557 andReturn("").once(); 558 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 559 andReturn("").once(); 560 561 for (int i = 0; i < testPaths.length; i++) { 562 mockListener.testStarted(EasyMock.eq(testIds[i])); 563 EasyMock.expectLastCall().once(); 564 565 mockListener.testEnded(EasyMock.eq(testIds[i]), 566 EasyMock.<HashMap<String, Metric>>notNull()); 567 568 EasyMock.expectLastCall().once(); 569 } 570 571 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 572 EasyMock.expectLastCall().once(); 573 574 EasyMock.replay(mockDevice, mockIDevice); 575 EasyMock.replay(mockListener); 576 577 deqpTest.setDevice(mockDevice); 578 deqpTest.run(mockListener); 579 580 EasyMock.verify(mockListener); 581 EasyMock.verify(mockDevice, mockIDevice); 582 } 583 buildTestProcessOutput(List<TestDescription> tests)584 static private String buildTestProcessOutput(List<TestDescription> tests) { 585 /* MultiLineReceiver expects "\r\n" line ending. */ 586 final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 587 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 588 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 589 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 590 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 591 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 592 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 593 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 594 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 595 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 596 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 597 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 598 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 599 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; 600 601 final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 602 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 603 + "INSTRUMENTATION_CODE: 0\r\n"; 604 605 StringWriter output = new StringWriter(); 606 output.write(outputHeader); 607 for (TestDescription test : tests) { 608 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"); 609 output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath="); 610 output.write(test.getClassName()); 611 output.write("."); 612 output.write(test.getTestName()); 613 output.write("\r\n"); 614 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 615 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"); 616 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"); 617 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"); 618 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 619 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"); 620 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 621 } 622 output.write(outputEnd); 623 return output.toString(); 624 } 625 testFiltering(DeqpTestRunner deqpTest, String expectedTrie, List<TestDescription> expectedTests)626 private void testFiltering(DeqpTestRunner deqpTest, 627 String expectedTrie, 628 List<TestDescription> expectedTests) throws Exception { 629 int version = 3 << 16; 630 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 631 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 632 .andReturn(Integer.toString(version)).atLeastOnce(); 633 634 boolean thereAreTests = !expectedTests.isEmpty(); 635 636 ITestInvocationListener mockListener 637 = EasyMock.createStrictMock(ITestInvocationListener.class); 638 mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size()); 639 EasyMock.expectLastCall().once(); 640 641 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 642 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 643 andReturn("").once(); 644 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 645 andReturn("").once(); 646 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 647 andReturn("").once(); 648 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 649 andReturn("").once(); 650 651 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 652 if (thereAreTests) 653 { 654 expectRenderConfigQuery(mockDevice, 3, 0); 655 656 String testOut = buildTestProcessOutput(expectedTests); 657 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); 658 659 for (int i = 0; i < expectedTests.size(); i++) { 660 mockListener.testStarted(EasyMock.eq(expectedTests.get(i))); 661 EasyMock.expectLastCall().once(); 662 663 mockListener.testEnded(EasyMock.eq(expectedTests.get(i)), 664 EasyMock.<HashMap<String, Metric>>notNull()); 665 666 EasyMock.expectLastCall().once(); 667 } 668 } 669 670 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 671 EasyMock.expectLastCall().once(); 672 673 EasyMock.replay(mockDevice, mockIDevice); 674 EasyMock.replay(mockListener); 675 676 deqpTest.setDevice(mockDevice); 677 deqpTest.run(mockListener); 678 679 EasyMock.verify(mockListener); 680 EasyMock.verify(mockDevice, mockIDevice); 681 } 682 testRun_trivialIncludeFilter()683 public void testRun_trivialIncludeFilter() throws Exception { 684 final TestDescription[] testIds = { 685 new TestDescription("dEQP-GLES3.missing", "no"), 686 new TestDescription("dEQP-GLES3.missing", "nope"), 687 new TestDescription("dEQP-GLES3.missing", "donotwant"), 688 new TestDescription("dEQP-GLES3.pick_me", "yes"), 689 new TestDescription("dEQP-GLES3.pick_me", "ok"), 690 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 691 }; 692 693 List<TestDescription> allTests = new ArrayList<TestDescription>(); 694 for (TestDescription id : testIds) { 695 allTests.add(id); 696 } 697 698 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 699 activeTests.add(testIds[3]); 700 activeTests.add(testIds[4]); 701 activeTests.add(testIds[5]); 702 703 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 704 705 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 706 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me#*"); 707 testFiltering(deqpTest, expectedTrie, activeTests); 708 } 709 testRun_trivialExcludeFilter()710 public void testRun_trivialExcludeFilter() throws Exception { 711 final TestDescription[] testIds = { 712 new TestDescription("dEQP-GLES3.missing", "no"), 713 new TestDescription("dEQP-GLES3.missing", "nope"), 714 new TestDescription("dEQP-GLES3.missing", "donotwant"), 715 new TestDescription("dEQP-GLES3.pick_me", "yes"), 716 new TestDescription("dEQP-GLES3.pick_me", "ok"), 717 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 718 }; 719 720 List<TestDescription> allTests = new ArrayList<TestDescription>(); 721 for (TestDescription id : testIds) { 722 allTests.add(id); 723 } 724 725 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 726 activeTests.add(testIds[3]); 727 activeTests.add(testIds[4]); 728 activeTests.add(testIds[5]); 729 730 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 731 732 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 733 deqpTest.addExcludeFilter("dEQP-GLES3.missing#*"); 734 testFiltering(deqpTest, expectedTrie, activeTests); 735 } 736 testRun_includeAndExcludeFilter()737 public void testRun_includeAndExcludeFilter() throws Exception { 738 final TestDescription[] testIds = { 739 new TestDescription("dEQP-GLES3.group1", "foo"), 740 new TestDescription("dEQP-GLES3.group1", "nope"), 741 new TestDescription("dEQP-GLES3.group1", "donotwant"), 742 new TestDescription("dEQP-GLES3.group2", "foo"), 743 new TestDescription("dEQP-GLES3.group2", "yes"), 744 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"), 745 }; 746 747 List<TestDescription> allTests = new ArrayList<TestDescription>(); 748 for (TestDescription id : testIds) { 749 allTests.add(id); 750 } 751 752 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 753 activeTests.add(testIds[4]); 754 755 String expectedTrie = "{dEQP-GLES3{group2{yes}}}"; 756 757 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 758 759 Set<String> includes = new HashSet<>(); 760 includes.add("dEQP-GLES3.group2#*"); 761 deqpTest.addAllIncludeFilters(includes); 762 763 Set<String> excludes = new HashSet<>(); 764 excludes.add("*foo"); 765 excludes.add("*thoushallnotpass"); 766 deqpTest.addAllExcludeFilters(excludes); 767 testFiltering(deqpTest, expectedTrie, activeTests); 768 } 769 testRun_includeAll()770 public void testRun_includeAll() throws Exception { 771 final TestDescription[] testIds = { 772 new TestDescription("dEQP-GLES3.group1", "mememe"), 773 new TestDescription("dEQP-GLES3.group1", "yeah"), 774 new TestDescription("dEQP-GLES3.group1", "takeitall"), 775 new TestDescription("dEQP-GLES3.group2", "jeba"), 776 new TestDescription("dEQP-GLES3.group2", "yes"), 777 new TestDescription("dEQP-GLES3.group2", "granted"), 778 }; 779 780 List<TestDescription> allTests = new ArrayList<TestDescription>(); 781 for (TestDescription id : testIds) { 782 allTests.add(id); 783 } 784 785 String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}"; 786 787 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 788 deqpTest.addIncludeFilter("*"); 789 testFiltering(deqpTest, expectedTrie, allTests); 790 } 791 testRun_excludeAll()792 public void testRun_excludeAll() throws Exception { 793 final TestDescription[] testIds = { 794 new TestDescription("dEQP-GLES3.group1", "no"), 795 new TestDescription("dEQP-GLES3.group1", "nope"), 796 new TestDescription("dEQP-GLES3.group1", "nottoday"), 797 new TestDescription("dEQP-GLES3.group2", "banned"), 798 new TestDescription("dEQP-GLES3.group2", "notrecognized"), 799 new TestDescription("dEQP-GLES3.group2", "-2"), 800 }; 801 802 List<TestDescription> allTests = new ArrayList<TestDescription>(); 803 for (TestDescription id : testIds) { 804 allTests.add(id); 805 } 806 807 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 808 deqpTest.addExcludeFilter("*"); 809 ITestInvocationListener mockListener 810 = EasyMock.createStrictMock(ITestInvocationListener.class); 811 mockListener.testRunStarted(getTestId(deqpTest), 0); 812 EasyMock.expectLastCall().once(); 813 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 814 EasyMock.expectLastCall().once(); 815 816 EasyMock.replay(mockListener); 817 deqpTest.run(mockListener); 818 EasyMock.verify(mockListener); 819 } 820 821 /** 822 * Test running a unexecutable test. 823 */ testRun_unexecutableTests()824 public void testRun_unexecutableTests() throws Exception { 825 final String instrumentationAnswerNoExecs = 826 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 827 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 828 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 829 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 830 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 831 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 832 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 833 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 834 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 835 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 836 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 837 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 838 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 839 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 840 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 841 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 842 + "INSTRUMENTATION_CODE: 0\r\n"; 843 844 final TestDescription[] testIds = { 845 new TestDescription("dEQP-GLES3.missing", "no"), 846 new TestDescription("dEQP-GLES3.missing", "nope"), 847 new TestDescription("dEQP-GLES3.missing", "donotwant"), 848 }; 849 850 final String[] testPaths = { 851 "dEQP-GLES3.missing.no", 852 "dEQP-GLES3.missing.nope", 853 "dEQP-GLES3.missing.donotwant", 854 }; 855 856 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 857 ITestInvocationListener mockListener 858 = EasyMock.createStrictMock(ITestInvocationListener.class); 859 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 860 861 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 862 863 for (TestDescription id : testIds) { 864 tests.add(id); 865 } 866 867 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 868 869 int version = 3 << 16; 870 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 871 .andReturn(Integer.toString(version)).atLeastOnce(); 872 873 expectRenderConfigQuery(mockDevice, 3, 0); 874 875 String commandLine = String.format( 876 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 877 + "--deqp-screen-rotation=unspecified " 878 + "--deqp-surface-type=window " 879 + "--deqp-log-images=disable " 880 + "--deqp-watchdog=enable", 881 APP_DIR + CASE_LIST_FILE_NAME); 882 883 // first try 884 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 885 "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs); 886 887 // splitting begins 888 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 889 "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs); 890 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 891 "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs); 892 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 893 "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs); 894 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 895 "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs); 896 897 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 898 EasyMock.expectLastCall().once(); 899 900 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 901 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 902 andReturn("").once(); 903 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 904 andReturn("").once(); 905 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 906 andReturn("").once(); 907 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 908 andReturn("").once(); 909 910 for (int i = 0; i < testPaths.length; i++) { 911 mockListener.testStarted(EasyMock.eq(testIds[i])); 912 EasyMock.expectLastCall().once(); 913 914 mockListener.testFailed(EasyMock.eq(testIds[i]), 915 EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n" 916 + "Abort: Test cannot be executed")); 917 EasyMock.expectLastCall().once(); 918 919 mockListener.testEnded(EasyMock.eq(testIds[i]), 920 EasyMock.<HashMap<String, Metric>>notNull()); 921 EasyMock.expectLastCall().once(); 922 } 923 924 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 925 EasyMock.expectLastCall().once(); 926 927 EasyMock.replay(mockDevice, mockIDevice); 928 EasyMock.replay(mockListener); 929 930 deqpTest.setDevice(mockDevice); 931 deqpTest.run(mockListener); 932 933 EasyMock.verify(mockListener); 934 EasyMock.verify(mockDevice, mockIDevice); 935 } 936 937 /** 938 * Test that test are left unexecuted if pm list query fails 939 */ testRun_queryPmListFailure()940 public void testRun_queryPmListFailure() 941 throws Exception { 942 final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test"); 943 944 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 945 ITestInvocationListener mockListener 946 = EasyMock.createStrictMock(ITestInvocationListener.class); 947 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 948 tests.add(testId); 949 950 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 951 OptionSetter setter = new OptionSetter(deqpTest); 952 // Note: If the rotation is the default unspecified, features are not queried at all 953 setter.setOptionValue("deqp-screen-rotation", "90"); 954 955 deqpTest.setDevice(mockDevice); 956 957 int version = 3 << 16; 958 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 959 .andReturn(Integer.toString(version)).atLeastOnce(); 960 961 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 962 .andReturn("not a valid format"); 963 964 mockListener.testRunStarted(getTestId(deqpTest), 1); 965 EasyMock.expectLastCall().once(); 966 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 967 andReturn("").once(); 968 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 969 andReturn("").once(); 970 971 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 972 EasyMock.expectLastCall().once(); 973 974 EasyMock.replay(mockDevice); 975 EasyMock.replay(mockListener); 976 deqpTest.run(mockListener); 977 EasyMock.verify(mockListener); 978 EasyMock.verify(mockDevice); 979 } 980 981 /** 982 * Test that test are left unexecuted if renderablity query fails 983 */ testRun_queryRenderabilityFailure()984 public void testRun_queryRenderabilityFailure() 985 throws Exception { 986 final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test"); 987 988 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 989 ITestInvocationListener mockListener 990 = EasyMock.createStrictMock(ITestInvocationListener.class); 991 992 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 993 tests.add(testId); 994 995 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 996 997 deqpTest.setDevice(mockDevice); 998 999 int version = 3 << 16; 1000 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1001 .andReturn(Integer.toString(version)).atLeastOnce(); 1002 1003 expectRenderConfigQueryAndReturn(mockDevice, 1004 "--deqp-gl-config-name=rgba8888d24s8 " 1005 + "--deqp-screen-rotation=unspecified " 1006 + "--deqp-surface-type=window " 1007 + "--deqp-gl-major-version=3 " 1008 + "--deqp-gl-minor-version=0", "Maybe?"); 1009 1010 mockListener.testRunStarted(getTestId(deqpTest), 1); 1011 EasyMock.expectLastCall().once(); 1012 1013 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1014 andReturn("").once(); 1015 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1016 andReturn("").once(); 1017 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1018 EasyMock.expectLastCall().once(); 1019 1020 EasyMock.replay(mockDevice); 1021 EasyMock.replay(mockListener); 1022 deqpTest.run(mockListener); 1023 EasyMock.verify(mockListener); 1024 EasyMock.verify(mockDevice); 1025 } 1026 1027 /** 1028 * Test that orientation is supplied to runner correctly 1029 */ testOrientation(final String rotation, final String featureString)1030 private void testOrientation(final String rotation, final String featureString) 1031 throws Exception { 1032 final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test"); 1033 final String testPath = "dEQP-GLES3.orientation.test"; 1034 final String testTrie = "{dEQP-GLES3{orientation{test}}}"; 1035 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1036 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1037 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1038 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1039 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1040 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1041 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1042 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1043 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1044 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1045 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1046 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1047 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1048 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1049 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1050 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 1051 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1052 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1053 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1054 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1055 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1056 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1057 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1058 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1059 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1060 + "INSTRUMENTATION_CODE: 0\r\n"; 1061 1062 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1063 ITestInvocationListener mockListener 1064 = EasyMock.createStrictMock(ITestInvocationListener.class); 1065 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1066 1067 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1068 tests.add(testId); 1069 1070 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1071 OptionSetter setter = new OptionSetter(deqpTest); 1072 setter.setOptionValue("deqp-screen-rotation", rotation); 1073 1074 deqpTest.setDevice(mockDevice); 1075 1076 int version = 3 << 16; 1077 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1078 .andReturn(Integer.toString(version)).atLeastOnce(); 1079 1080 if (!rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED)) { 1081 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 1082 .andReturn(featureString); 1083 } 1084 1085 final boolean isPortraitOrientation = 1086 rotation.equals(BatchRunConfiguration.ROTATION_PORTRAIT) || 1087 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT); 1088 final boolean isLandscapeOrientation = 1089 rotation.equals(BatchRunConfiguration.ROTATION_LANDSCAPE) || 1090 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE); 1091 final boolean executable = 1092 rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED) || 1093 (isPortraitOrientation && 1094 featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) || 1095 (isLandscapeOrientation && 1096 featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE)); 1097 1098 if (executable) { 1099 expectRenderConfigQuery(mockDevice, String.format( 1100 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s " 1101 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1102 + "--deqp-gl-minor-version=0", rotation)); 1103 1104 String commandLine = String.format( 1105 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1106 + "--deqp-screen-rotation=%s " 1107 + "--deqp-surface-type=window " 1108 + "--deqp-log-images=disable " 1109 + "--deqp-watchdog=enable", 1110 APP_DIR + CASE_LIST_FILE_NAME, rotation); 1111 1112 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 1113 output); 1114 } 1115 1116 mockListener.testRunStarted(getTestId(deqpTest), 1); 1117 EasyMock.expectLastCall().once(); 1118 1119 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 1120 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1121 andReturn("").once(); 1122 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1123 andReturn("").once(); 1124 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1125 andReturn("").once(); 1126 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1127 andReturn("").once(); 1128 1129 mockListener.testStarted(EasyMock.eq(testId)); 1130 EasyMock.expectLastCall().once(); 1131 1132 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 1133 EasyMock.expectLastCall().once(); 1134 1135 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1136 EasyMock.expectLastCall().once(); 1137 1138 EasyMock.replay(mockDevice, mockIDevice); 1139 EasyMock.replay(mockListener); 1140 deqpTest.run(mockListener); 1141 EasyMock.verify(mockListener); 1142 EasyMock.verify(mockDevice, mockIDevice); 1143 } 1144 1145 /** 1146 * Test OpeGL ES3 tests on device with OpenGL ES2. 1147 */ testRun_require30DeviceVersion20()1148 public void testRun_require30DeviceVersion20() throws Exception { 1149 testGlesVersion(3, 0, 2, 0); 1150 } 1151 1152 /** 1153 * Test OpeGL ES3.1 tests on device with OpenGL ES2. 1154 */ testRun_require31DeviceVersion20()1155 public void testRun_require31DeviceVersion20() throws Exception { 1156 testGlesVersion(3, 1, 2, 0); 1157 } 1158 1159 /** 1160 * Test OpeGL ES3 tests on device with OpenGL ES3. 1161 */ testRun_require30DeviceVersion30()1162 public void testRun_require30DeviceVersion30() throws Exception { 1163 testGlesVersion(3, 0, 3, 0); 1164 } 1165 1166 /** 1167 * Test OpeGL ES3.1 tests on device with OpenGL ES3. 1168 */ testRun_require31DeviceVersion30()1169 public void testRun_require31DeviceVersion30() throws Exception { 1170 testGlesVersion(3, 1, 3, 0); 1171 } 1172 1173 /** 1174 * Test OpeGL ES3 tests on device with OpenGL ES3.1. 1175 */ testRun_require30DeviceVersion31()1176 public void testRun_require30DeviceVersion31() throws Exception { 1177 testGlesVersion(3, 0, 3, 1); 1178 } 1179 1180 /** 1181 * Test OpeGL ES3.1 tests on device with OpenGL ES3.1. 1182 */ testRun_require31DeviceVersion31()1183 public void testRun_require31DeviceVersion31() throws Exception { 1184 testGlesVersion(3, 1, 3, 1); 1185 } 1186 1187 /** 1188 * Test dEQP Pass result code. 1189 */ testRun_resultPass()1190 public void testRun_resultPass() throws Exception { 1191 testResultCode("Pass", true); 1192 } 1193 1194 /** 1195 * Test dEQP Fail result code. 1196 */ testRun_resultFail()1197 public void testRun_resultFail() throws Exception { 1198 testResultCode("Fail", false); 1199 } 1200 1201 /** 1202 * Test dEQP NotSupported result code. 1203 */ testRun_resultNotSupported()1204 public void testRun_resultNotSupported() throws Exception { 1205 testResultCode("NotSupported", true); 1206 } 1207 1208 /** 1209 * Test dEQP QualityWarning result code. 1210 */ testRun_resultQualityWarning()1211 public void testRun_resultQualityWarning() throws Exception { 1212 testResultCode("QualityWarning", true); 1213 } 1214 1215 /** 1216 * Test dEQP CompatibilityWarning result code. 1217 */ testRun_resultCompatibilityWarning()1218 public void testRun_resultCompatibilityWarning() throws Exception { 1219 testResultCode("CompatibilityWarning", true); 1220 } 1221 1222 /** 1223 * Test dEQP ResourceError result code. 1224 */ testRun_resultResourceError()1225 public void testRun_resultResourceError() throws Exception { 1226 testResultCode("ResourceError", false); 1227 } 1228 1229 /** 1230 * Test dEQP InternalError result code. 1231 */ testRun_resultInternalError()1232 public void testRun_resultInternalError() throws Exception { 1233 testResultCode("InternalError", false); 1234 } 1235 1236 /** 1237 * Test dEQP Crash result code. 1238 */ testRun_resultCrash()1239 public void testRun_resultCrash() throws Exception { 1240 testResultCode("Crash", false); 1241 } 1242 1243 /** 1244 * Test dEQP Timeout result code. 1245 */ testRun_resultTimeout()1246 public void testRun_resultTimeout() throws Exception { 1247 testResultCode("Timeout", false); 1248 } 1249 /** 1250 * Test dEQP Orientation 1251 */ testRun_orientationLandscape()1252 public void testRun_orientationLandscape() throws Exception { 1253 testOrientation("90", ALL_FEATURES); 1254 } 1255 1256 /** 1257 * Test dEQP Orientation 1258 */ testRun_orientationPortrait()1259 public void testRun_orientationPortrait() throws Exception { 1260 testOrientation("0", ALL_FEATURES); 1261 } 1262 1263 /** 1264 * Test dEQP Orientation 1265 */ testRun_orientationReverseLandscape()1266 public void testRun_orientationReverseLandscape() throws Exception { 1267 testOrientation("270", ALL_FEATURES); 1268 } 1269 1270 /** 1271 * Test dEQP Orientation 1272 */ testRun_orientationReversePortrait()1273 public void testRun_orientationReversePortrait() throws Exception { 1274 testOrientation("180", ALL_FEATURES); 1275 } 1276 1277 /** 1278 * Test dEQP Orientation 1279 */ testRun_orientationUnspecified()1280 public void testRun_orientationUnspecified() throws Exception { 1281 testOrientation("unspecified", ALL_FEATURES); 1282 } 1283 1284 /** 1285 * Test dEQP Orientation with limited features 1286 */ testRun_orientationUnspecifiedLimitedFeatures()1287 public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception { 1288 testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES); 1289 } 1290 1291 /** 1292 * Test dEQP Orientation with limited features 1293 */ testRun_orientationLandscapeLimitedFeatures()1294 public void testRun_orientationLandscapeLimitedFeatures() throws Exception { 1295 testOrientation("90", ONLY_LANDSCAPE_FEATURES); 1296 } 1297 1298 /** 1299 * Test dEQP Orientation with limited features 1300 */ testRun_orientationPortraitLimitedFeatures()1301 public void testRun_orientationPortraitLimitedFeatures() throws Exception { 1302 testOrientation("0", ONLY_LANDSCAPE_FEATURES); 1303 } 1304 1305 /** 1306 * Test dEQP unsupported pixel format 1307 */ testRun_unsupportedPixelFormat()1308 public void testRun_unsupportedPixelFormat() throws Exception { 1309 final String pixelFormat = "rgba5658d16m4"; 1310 final TestDescription testId = new TestDescription("dEQP-GLES3.pixelformat", "test"); 1311 1312 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1313 ITestInvocationListener mockListener 1314 = EasyMock.createStrictMock(ITestInvocationListener.class); 1315 1316 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1317 tests.add(testId); 1318 1319 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1320 OptionSetter setter = new OptionSetter(deqpTest); 1321 setter.setOptionValue("deqp-gl-config-name", pixelFormat); 1322 1323 deqpTest.setDevice(mockDevice); 1324 1325 int version = 3 << 16; 1326 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1327 .andReturn(Integer.toString(version)).atLeastOnce(); 1328 1329 expectRenderConfigQueryAndReturn(mockDevice, String.format( 1330 "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified " 1331 + "--deqp-surface-type=window " 1332 + "--deqp-gl-major-version=3 " 1333 + "--deqp-gl-minor-version=0", pixelFormat), "No"); 1334 1335 mockListener.testRunStarted(getTestId(deqpTest), 1); 1336 EasyMock.expectLastCall().once(); 1337 1338 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 1339 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1340 andReturn("").once(); 1341 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1342 andReturn("").once(); 1343 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1344 andReturn("").once(); 1345 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1346 andReturn("").once(); 1347 1348 mockListener.testStarted(EasyMock.eq(testId)); 1349 EasyMock.expectLastCall().once(); 1350 1351 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 1352 EasyMock.expectLastCall().once(); 1353 1354 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1355 EasyMock.expectLastCall().once(); 1356 1357 EasyMock.replay(mockDevice); 1358 EasyMock.replay(mockListener); 1359 deqpTest.run(mockListener); 1360 EasyMock.verify(mockListener); 1361 EasyMock.verify(mockDevice); 1362 } 1363 1364 /** 1365 * Test interface to mock Tradefed device types. 1366 */ 1367 public static interface RecoverableTestDevice extends ITestDevice, IManagedTestDevice { 1368 } 1369 1370 private static enum RecoveryEvent { 1371 PROGRESS, 1372 FAIL_CONNECTION_REFUSED, 1373 FAIL_LINK_KILLED, 1374 } 1375 runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)1376 private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events) 1377 throws DeviceNotAvailableException { 1378 for (RecoveryEvent event : events) { 1379 switch (event) { 1380 case PROGRESS: 1381 recovery.onExecutionProgressed(); 1382 break; 1383 case FAIL_CONNECTION_REFUSED: 1384 recovery.recoverConnectionRefused(); 1385 break; 1386 case FAIL_LINK_KILLED: 1387 recovery.recoverComLinkKilled(); 1388 break; 1389 } 1390 } 1391 } 1392 setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider)1393 private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) { 1394 mockSleepProvider.sleep(EasyMock.gt(0)); 1395 EasyMock.expectLastCall().once(); 1396 } 1397 setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider)1398 private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, 1399 DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException { 1400 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1401 andReturn("root 1234 com.drawelement.deqp").once(); 1402 1403 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1404 andReturn("").once(); 1405 1406 // Recovery checks if kill failed 1407 mockSleepProvider.sleep(EasyMock.gt(0)); 1408 EasyMock.expectLastCall().once(); 1409 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1410 andReturn("").once(); 1411 } 1412 setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)1413 private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice) 1414 throws DeviceNotAvailableException { 1415 mockDevice.recoverDevice(); 1416 EasyMock.expectLastCall().once(); 1417 } 1418 setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)1419 private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice) 1420 throws DeviceNotAvailableException { 1421 mockDevice.reboot(); 1422 EasyMock.expectLastCall().once(); 1423 } 1424 setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)1425 private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, 1426 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 1427 throws DeviceNotAvailableException { 1428 switch (numConsecutiveErrors) { 1429 case 0: 1430 case 1: 1431 setRecoveryExpectationRecovery(mockDevice); 1432 return 2; 1433 case 2: 1434 setRecoveryExpectationReboot(mockDevice); 1435 return 3; 1436 default: 1437 return 4; 1438 } 1439 } 1440 setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)1441 private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, 1442 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 1443 throws DeviceNotAvailableException { 1444 switch (numConsecutiveErrors) { 1445 case 0: 1446 setRecoveryExpectationWait(mockSleepProvider); 1447 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1448 return 1; 1449 case 1: 1450 setRecoveryExpectationRecovery(mockDevice); 1451 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1452 return 2; 1453 case 2: 1454 setRecoveryExpectationReboot(mockDevice); 1455 return 3; 1456 default: 1457 return 4; 1458 } 1459 } 1460 setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)1461 private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, 1462 DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events) 1463 throws DeviceNotAvailableException { 1464 int numConsecutiveErrors = 0; 1465 for (RecoveryEvent event : events) { 1466 switch (event) { 1467 case PROGRESS: 1468 numConsecutiveErrors = 0; 1469 break; 1470 case FAIL_CONNECTION_REFUSED: 1471 numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice, 1472 mockSleepProvider, numConsecutiveErrors); 1473 break; 1474 case FAIL_LINK_KILLED: 1475 numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice, 1476 mockSleepProvider, numConsecutiveErrors); 1477 break; 1478 } 1479 } 1480 } 1481 1482 /** 1483 * Test dEQP runner recovery state machine. 1484 */ testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)1485 private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern) 1486 throws Exception { 1487 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 1488 IMocksControl orderedControl = EasyMock.createStrictControl(); 1489 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 1490 EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL"); 1491 DeqpTestRunner.ISleepProvider mockSleepProvider = 1492 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 1493 1494 setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern); 1495 1496 orderedControl.replay(); 1497 1498 recovery.setDevice(mockDevice); 1499 recovery.setSleepProvider(mockSleepProvider); 1500 try { 1501 runRecoveryWithPattern(recovery, pattern); 1502 if (!expectSuccess) { 1503 fail("Expected DeviceNotAvailableException"); 1504 } 1505 } catch (DeviceNotAvailableException ex) { 1506 if (expectSuccess) { 1507 fail("Did not expect DeviceNotAvailableException"); 1508 } 1509 } 1510 1511 orderedControl.verify(); 1512 } 1513 1514 // basic patterns 1515 testRecovery_NoEvents()1516 public void testRecovery_NoEvents() throws Exception { 1517 testRecoveryWithPattern(true); 1518 } 1519 testRecovery_AllOk()1520 public void testRecovery_AllOk() throws Exception { 1521 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS); 1522 } 1523 1524 // conn fail patterns 1525 testRecovery_OneConnectionFailureBegin()1526 public void testRecovery_OneConnectionFailureBegin() throws Exception { 1527 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1528 RecoveryEvent.PROGRESS); 1529 } 1530 testRecovery_TwoConnectionFailuresBegin()1531 public void testRecovery_TwoConnectionFailuresBegin() throws Exception { 1532 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1533 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1534 } 1535 testRecovery_ThreeConnectionFailuresBegin()1536 public void testRecovery_ThreeConnectionFailuresBegin() throws Exception { 1537 testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1538 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED); 1539 } 1540 testRecovery_OneConnectionFailureMid()1541 public void testRecovery_OneConnectionFailureMid() throws Exception { 1542 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1543 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1544 } 1545 testRecovery_TwoConnectionFailuresMid()1546 public void testRecovery_TwoConnectionFailuresMid() throws Exception { 1547 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1548 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1549 RecoveryEvent.PROGRESS); 1550 } 1551 testRecovery_ThreeConnectionFailuresMid()1552 public void testRecovery_ThreeConnectionFailuresMid() throws Exception { 1553 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, 1554 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1555 RecoveryEvent.FAIL_CONNECTION_REFUSED); 1556 } 1557 1558 // link fail patterns 1559 testRecovery_OneLinkFailureBegin()1560 public void testRecovery_OneLinkFailureBegin() throws Exception { 1561 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1562 RecoveryEvent.PROGRESS); 1563 } 1564 testRecovery_TwoLinkFailuresBegin()1565 public void testRecovery_TwoLinkFailuresBegin() throws Exception { 1566 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1567 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1568 } 1569 testRecovery_ThreeLinkFailuresBegin()1570 public void testRecovery_ThreeLinkFailuresBegin() throws Exception { 1571 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1572 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1573 RecoveryEvent.PROGRESS); 1574 } 1575 testRecovery_FourLinkFailuresBegin()1576 public void testRecovery_FourLinkFailuresBegin() throws Exception { 1577 testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED, 1578 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1579 RecoveryEvent.FAIL_LINK_KILLED); 1580 } 1581 testRecovery_OneLinkFailureMid()1582 public void testRecovery_OneLinkFailureMid() throws Exception { 1583 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1584 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1585 } 1586 testRecovery_TwoLinkFailuresMid()1587 public void testRecovery_TwoLinkFailuresMid() throws Exception { 1588 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1589 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1590 RecoveryEvent.PROGRESS); 1591 } 1592 testRecovery_ThreeLinkFailuresMid()1593 public void testRecovery_ThreeLinkFailuresMid() throws Exception { 1594 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1595 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1596 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1597 } 1598 testRecovery_FourLinkFailuresMid()1599 public void testRecovery_FourLinkFailuresMid() throws Exception { 1600 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1601 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1602 RecoveryEvent.FAIL_LINK_KILLED); 1603 } 1604 1605 // mixed patterns 1606 testRecovery_MixedFailuresProgressBetween()1607 public void testRecovery_MixedFailuresProgressBetween() throws Exception { 1608 testRecoveryWithPattern(true, 1609 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1610 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1611 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1612 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1613 RecoveryEvent.PROGRESS); 1614 } 1615 testRecovery_MixedFailuresNoProgressBetween()1616 public void testRecovery_MixedFailuresNoProgressBetween() throws Exception { 1617 testRecoveryWithPattern(true, 1618 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1619 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED, 1620 RecoveryEvent.PROGRESS); 1621 } 1622 1623 /** 1624 * Test recovery if process cannot be killed 1625 */ testRecovery_unkillableProcess()1626 public void testRecovery_unkillableProcess () throws Exception { 1627 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 1628 IMocksControl orderedControl = EasyMock.createStrictControl(); 1629 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 1630 DeqpTestRunner.ISleepProvider mockSleepProvider = 1631 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 1632 1633 // recovery attempts to kill the process after a timeout 1634 mockSleepProvider.sleep(EasyMock.gt(0)); 1635 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1636 andReturn("root 1234 com.drawelement.deqp").once(); 1637 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1638 andReturn("").once(); 1639 1640 // Recovery checks if kill failed 1641 mockSleepProvider.sleep(EasyMock.gt(0)); 1642 EasyMock.expectLastCall().once(); 1643 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1644 andReturn("root 1234 com.drawelement.deqp").once(); 1645 1646 // Recovery resets the connection 1647 mockDevice.recoverDevice(); 1648 EasyMock.expectLastCall().once(); 1649 1650 // and attempts to kill the process again 1651 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1652 andReturn("root 1234 com.drawelement.deqp").once(); 1653 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1654 andReturn("").once(); 1655 1656 // Recovery checks if kill failed 1657 mockSleepProvider.sleep(EasyMock.gt(0)); 1658 EasyMock.expectLastCall().once(); 1659 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1660 andReturn("root 1234 com.drawelement.deqp").once(); 1661 1662 // recovery reboots the device 1663 mockDevice.reboot(); 1664 EasyMock.expectLastCall().once(); 1665 1666 orderedControl.replay(); 1667 recovery.setDevice(mockDevice); 1668 recovery.setSleepProvider(mockSleepProvider); 1669 recovery.recoverComLinkKilled(); 1670 orderedControl.verify(); 1671 } 1672 1673 /** 1674 * Test external interruption before batch run. 1675 */ testInterrupt_killBeforeBatch()1676 public void testInterrupt_killBeforeBatch() throws Exception { 1677 final TestDescription testId = new TestDescription("dEQP-GLES3.interrupt", "test"); 1678 1679 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1680 tests.add(testId); 1681 1682 ITestInvocationListener mockListener 1683 = EasyMock.createStrictMock(ITestInvocationListener.class); 1684 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1685 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1686 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 1687 1688 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1689 1690 deqpTest.setDevice(mockDevice); 1691 deqpTest.setRunUtil(mockRunUtil); 1692 1693 int version = 3 << 16; 1694 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1695 .andReturn(Integer.toString(version)).atLeastOnce(); 1696 1697 expectRenderConfigQuery(mockDevice, 1698 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 1699 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1700 + "--deqp-gl-minor-version=0"); 1701 1702 mockRunUtil.sleep(0); 1703 EasyMock.expectLastCall().andThrow(new RunInterruptedException( 1704 "message", InfraErrorIdentifier.TRADEFED_SHUTTING_DOWN)); 1705 1706 mockListener.testRunStarted(getTestId(deqpTest), 1); 1707 EasyMock.expectLastCall().once(); 1708 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1709 andReturn("").once(); 1710 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1711 andReturn("").once(); 1712 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject()); 1713 EasyMock.expectLastCall().once(); 1714 1715 EasyMock.replay(mockDevice, mockIDevice); 1716 EasyMock.replay(mockListener); 1717 EasyMock.replay(mockRunUtil); 1718 try { 1719 deqpTest.run(mockListener); 1720 fail("expected RunInterruptedException"); 1721 } catch (RunInterruptedException ex) { 1722 // expected 1723 } 1724 EasyMock.verify(mockRunUtil); 1725 EasyMock.verify(mockListener); 1726 EasyMock.verify(mockDevice, mockIDevice); 1727 } 1728 runShardedTest(TestDescription[] testIds, ArrayList<ArrayList<TestDescription>> testsForShard)1729 private void runShardedTest(TestDescription[] testIds, 1730 ArrayList<ArrayList<TestDescription>> testsForShard) throws Exception { 1731 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1732 for (TestDescription id : testIds) tests.add(id); 1733 1734 DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests, mTestsDir); 1735 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split(); 1736 1737 for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) { 1738 DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex); 1739 shard.setBuildHelper(getMockBuildHelper(mTestsDir)); 1740 1741 ArrayList<TestDescription> shardTests = testsForShard.get(shardIndex); 1742 1743 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1744 ITestInvocationListener mockListener 1745 = EasyMock.createStrictMock(ITestInvocationListener.class); 1746 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1747 int version = 3 << 16; 1748 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1749 .andReturn(Integer.toString(version)).atLeastOnce(); 1750 1751 mockListener.testRunStarted(getTestId(shard), shardTests.size()); 1752 EasyMock.expectLastCall().once(); 1753 1754 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 1755 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1756 andReturn("").once(); 1757 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1758 andReturn("").once(); 1759 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1760 andReturn("").once(); 1761 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1762 andReturn("").once(); 1763 1764 expectRenderConfigQuery(mockDevice, 3, 0); 1765 1766 String testOut = buildTestProcessOutput(shardTests); 1767 // NOTE: This assumes that there won't be multiple batches per shard! 1768 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); 1769 1770 for (int i = 0; i < shardTests.size(); i++) { 1771 mockListener.testStarted(EasyMock.eq(shardTests.get(i))); 1772 EasyMock.expectLastCall().once(); 1773 1774 mockListener.testEnded(EasyMock.eq(shardTests.get(i)), 1775 EasyMock.<HashMap<String, Metric>>notNull()); 1776 1777 EasyMock.expectLastCall().once(); 1778 } 1779 1780 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1781 EasyMock.expectLastCall().once(); 1782 1783 EasyMock.replay(mockDevice, mockIDevice); 1784 EasyMock.replay(mockListener); 1785 1786 shard.setDevice(mockDevice); 1787 shard.run(mockListener); 1788 1789 EasyMock.verify(mockListener); 1790 EasyMock.verify(mockDevice, mockIDevice); 1791 } 1792 } 1793 testSharding_smallTrivial()1794 public void testSharding_smallTrivial() throws Exception { 1795 final TestDescription[] testIds = { 1796 new TestDescription("dEQP-GLES3.info", "vendor"), 1797 new TestDescription("dEQP-GLES3.info", "renderer"), 1798 new TestDescription("dEQP-GLES3.info", "version"), 1799 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 1800 new TestDescription("dEQP-GLES3.info", "extensions"), 1801 new TestDescription("dEQP-GLES3.info", "render_target") 1802 }; 1803 ArrayList<ArrayList<TestDescription>> shardedTests = new ArrayList<>(); 1804 ArrayList<TestDescription> shardOne = new ArrayList<>(); 1805 for (int i = 0; i < testIds.length; i++) { 1806 shardOne.add(testIds[i]); 1807 } 1808 shardedTests.add(shardOne); 1809 runShardedTest(testIds, shardedTests); 1810 } 1811 testSharding_twoShards()1812 public void testSharding_twoShards() throws Exception { 1813 final int TEST_COUNT = 1237; 1814 final int SHARD_SIZE = 1000; 1815 1816 ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT); 1817 for (int i = 0; i < TEST_COUNT; i++) { 1818 testIds.add(new TestDescription("dEQP-GLES3.funny.group", String.valueOf(i))); 1819 } 1820 1821 ArrayList<ArrayList<TestDescription>> shardedTests = new ArrayList<>(); 1822 ArrayList<TestDescription> shard = new ArrayList<>(); 1823 for (int i = 0; i < testIds.size(); i++) { 1824 if (i == SHARD_SIZE) { 1825 shardedTests.add(shard); 1826 shard = new ArrayList<>(); 1827 } 1828 shard.add(testIds.get(i)); 1829 } 1830 shardedTests.add(shard); 1831 runShardedTest(testIds.toArray(new TestDescription[testIds.size()]), shardedTests); 1832 } 1833 testSharding_empty()1834 public void testSharding_empty() throws Exception { 1835 DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList<TestDescription>(), mTestsDir); 1836 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split(); 1837 // Returns null when cannot be sharded. 1838 assertNull(shards); 1839 } 1840 1841 /** 1842 * Test external interruption in testFailed(). 1843 */ testInterrupt_killReportTestFailed()1844 public void testInterrupt_killReportTestFailed() throws Exception { 1845 final TestDescription testId = new TestDescription("dEQP-GLES3.interrupt", "test"); 1846 final String testPath = "dEQP-GLES3.interrupt.test"; 1847 final String testTrie = "{dEQP-GLES3{interrupt{test}}}"; 1848 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1849 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1850 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1851 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1852 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1853 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1854 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1855 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1856 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1857 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1858 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1859 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1860 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1861 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1862 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1863 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 1864 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1865 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n" 1866 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n" 1867 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1868 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1869 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1870 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1871 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1872 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1873 + "INSTRUMENTATION_CODE: 0\r\n"; 1874 1875 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1876 tests.add(testId); 1877 1878 ITestInvocationListener mockListener 1879 = EasyMock.createStrictMock(ITestInvocationListener.class); 1880 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1881 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1882 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 1883 1884 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1885 1886 deqpTest.setDevice(mockDevice); 1887 deqpTest.setRunUtil(mockRunUtil); 1888 1889 int version = 3 << 16; 1890 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1891 .andReturn(Integer.toString(version)).atLeastOnce(); 1892 1893 expectRenderConfigQuery(mockDevice, 1894 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 1895 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1896 + "--deqp-gl-minor-version=0"); 1897 1898 mockRunUtil.sleep(0); 1899 EasyMock.expectLastCall().once(); 1900 1901 String commandLine = String.format( 1902 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1903 + "--deqp-screen-rotation=unspecified " 1904 + "--deqp-surface-type=window " 1905 + "--deqp-log-images=disable " 1906 + "--deqp-watchdog=enable", 1907 APP_DIR + CASE_LIST_FILE_NAME); 1908 1909 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 1910 output); 1911 1912 mockListener.testRunStarted(getTestId(deqpTest), 1); 1913 EasyMock.expectLastCall().once(); 1914 1915 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 1916 andReturn("").once(); 1917 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 1918 andReturn("").once(); 1919 1920 mockListener.testStarted(EasyMock.eq(testId)); 1921 EasyMock.expectLastCall().once(); 1922 1923 mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull()); 1924 EasyMock.expectLastCall().andThrow(new RunInterruptedException( 1925 "message", InfraErrorIdentifier.TRADEFED_SHUTTING_DOWN)); 1926 1927 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject()); 1928 EasyMock.expectLastCall().once(); 1929 EasyMock.replay(mockDevice, mockIDevice); 1930 EasyMock.replay(mockListener); 1931 EasyMock.replay(mockRunUtil); 1932 try { 1933 deqpTest.run(mockListener); 1934 fail("expected RunInterruptedException"); 1935 } catch (RunInterruptedException ex) { 1936 // expected 1937 } 1938 EasyMock.verify(mockRunUtil); 1939 EasyMock.verify(mockListener); 1940 EasyMock.verify(mockDevice, mockIDevice); 1941 } 1942 testRuntimeHint_optionSet()1943 public void testRuntimeHint_optionSet() throws Exception { 1944 /* MultiLineReceiver expects "\r\n" line ending. */ 1945 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1946 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1947 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1948 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1949 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1950 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1951 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1952 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1953 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1954 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1955 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1956 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1957 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1958 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1959 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1960 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 1961 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1962 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1963 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1964 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1965 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1966 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1967 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1968 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1969 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 1970 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1971 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1972 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1973 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1974 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1975 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1976 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1977 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1978 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 1979 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1980 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1981 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1982 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1983 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1984 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1985 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1986 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1987 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 1988 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1989 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1990 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1991 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1992 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1993 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1994 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1995 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1996 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 1997 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1998 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1999 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2000 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2001 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2002 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2003 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2004 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2005 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 2006 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2007 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2008 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2009 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2010 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2011 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2012 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2013 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 2014 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2015 + "INSTRUMENTATION_CODE: 0\r\n"; 2016 2017 final TestDescription[] testIds = { 2018 new TestDescription("dEQP-GLES3.info", "vendor"), 2019 new TestDescription("dEQP-GLES3.info", "renderer"), 2020 new TestDescription("dEQP-GLES3.info", "version"), 2021 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 2022 new TestDescription("dEQP-GLES3.info", "extensions"), 2023 new TestDescription("dEQP-GLES3.info", "render_target") 2024 }; 2025 2026 final String[] testPaths = { 2027 "dEQP-GLES3.info.vendor", 2028 "dEQP-GLES3.info.renderer", 2029 "dEQP-GLES3.info.version", 2030 "dEQP-GLES3.info.shading_language_version", 2031 "dEQP-GLES3.info.extensions", 2032 "dEQP-GLES3.info.render_target" 2033 }; 2034 2035 final String testTrie 2036 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 2037 2038 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2039 ITestInvocationListener mockListener 2040 = EasyMock.createStrictMock(ITestInvocationListener.class); 2041 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 2042 2043 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2044 2045 for (TestDescription id : testIds) { 2046 tests.add(id); 2047 } 2048 2049 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2050 OptionSetter setter = new OptionSetter(deqpTest); 2051 final long runtimeMs = 123456; 2052 setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs)); 2053 assertEquals("Wrong expected runtime - option not passed cleanly", runtimeMs, deqpTest.getRuntimeHint()); 2054 2055 // Try running the tests as well. The unit tests do not set the hint be default, 2056 // so that case is covered. 2057 2058 int version = 3 << 16; 2059 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 2060 .andReturn(Integer.toString(version)).atLeastOnce(); 2061 2062 expectRenderConfigQuery(mockDevice, 3, 0); 2063 2064 String commandLine = String.format( 2065 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2066 + "--deqp-screen-rotation=unspecified " 2067 + "--deqp-surface-type=window " 2068 + "--deqp-log-images=disable " 2069 + "--deqp-watchdog=enable", 2070 APP_DIR + CASE_LIST_FILE_NAME); 2071 2072 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 2073 2074 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 2075 EasyMock.expectLastCall().once(); 2076 2077 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 2078 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 2079 andReturn("").once(); 2080 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 2081 andReturn("").once(); 2082 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 2083 andReturn("").once(); 2084 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 2085 andReturn("").once(); 2086 2087 for (int i = 0; i < testPaths.length; i++) { 2088 mockListener.testStarted(EasyMock.eq(testIds[i])); 2089 EasyMock.expectLastCall().once(); 2090 2091 mockListener.testEnded(EasyMock.eq(testIds[i]), 2092 EasyMock.<HashMap<String, Metric>>notNull()); 2093 2094 EasyMock.expectLastCall().once(); 2095 } 2096 2097 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 2098 EasyMock.expectLastCall().once(); 2099 2100 EasyMock.replay(mockDevice, mockIDevice); 2101 EasyMock.replay(mockListener); 2102 2103 deqpTest.setDevice(mockDevice); 2104 deqpTest.run(mockListener); 2105 2106 EasyMock.verify(mockListener); 2107 EasyMock.verify(mockDevice, mockIDevice); 2108 } 2109 testRuntimeHint_optionSetSharded()2110 public void testRuntimeHint_optionSetSharded() throws Exception { 2111 final int TEST_COUNT = 1237; 2112 final int SHARD_SIZE = 1000; 2113 2114 ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT); 2115 for (int i = 0; i < TEST_COUNT; i++) { 2116 testIds.add(new TestDescription("dEQP-GLES3.funny.group", String.valueOf(i))); 2117 } 2118 2119 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir); 2120 OptionSetter setter = new OptionSetter(deqpTest); 2121 final long fullRuntimeMs = testIds.size()*100; 2122 setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs)); 2123 2124 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)deqpTest.split(); 2125 assertEquals("First shard's time not proportional to test count", 2126 (fullRuntimeMs*SHARD_SIZE)/TEST_COUNT, 2127 ((IRuntimeHintProvider)shards.get(0)).getRuntimeHint()); 2128 assertEquals("Second shard's time not proportional to test count", 2129 (fullRuntimeMs*(TEST_COUNT-SHARD_SIZE))/TEST_COUNT, 2130 ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint()); 2131 } 2132 testRuntimeHint_optionNotSet()2133 public void testRuntimeHint_optionNotSet() throws Exception { 2134 final TestDescription[] testIds = { 2135 new TestDescription("dEQP-GLES3.info", "vendor"), 2136 new TestDescription("dEQP-GLES3.info", "renderer"), 2137 new TestDescription("dEQP-GLES3.info", "version"), 2138 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 2139 new TestDescription("dEQP-GLES3.info", "extensions"), 2140 new TestDescription("dEQP-GLES3.info", "render_target") 2141 }; 2142 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2143 2144 for (TestDescription id : testIds) { 2145 tests.add(id); 2146 } 2147 2148 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2149 2150 long runtime = deqpTest.getRuntimeHint(); 2151 assertTrue("Runtime for tests must be positive", runtime > 0); 2152 assertTrue("Runtime for tests must be reasonable", runtime < (1000 * 10)); // Must be done in 10s 2153 } 2154 2155 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String output)2156 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 2157 final String output) throws Exception { 2158 String cmd = String.format( 2159 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2160 + "--deqp-screen-rotation=unspecified " 2161 + "--deqp-surface-type=window " 2162 + "--deqp-log-images=disable " 2163 + "--deqp-watchdog=enable", 2164 APP_DIR + CASE_LIST_FILE_NAME); 2165 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, cmd, output); 2166 } 2167 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String testTrie, final String cmd, final String output)2168 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 2169 final String testTrie, final String cmd, final String output) throws Exception { 2170 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + CASE_LIST_FILE_NAME))) 2171 .andReturn("").once(); 2172 2173 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + LOG_FILE_NAME))) 2174 .andReturn("").once(); 2175 2176 if (testTrie == null) { 2177 mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(APP_DIR + CASE_LIST_FILE_NAME)); 2178 } 2179 else { 2180 mockDevice.pushString(testTrie + "\n", APP_DIR + CASE_LIST_FILE_NAME); 2181 } 2182 EasyMock.expectLastCall().andReturn(true).once(); 2183 2184 String command = String.format( 2185 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" " 2186 + "-e deqpLogData \"%s\" %s", 2187 AbiUtils.createAbiFlag(ABI.getName()), APP_DIR + LOG_FILE_NAME, cmd, false, 2188 INSTRUMENTATION_NAME); 2189 2190 EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice); 2191 mockIDevice.executeShellCommand(EasyMock.eq(command), 2192 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(), 2193 EasyMock.isA(TimeUnit.class)); 2194 2195 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 2196 @Override 2197 public Object answer() { 2198 IShellOutputReceiver receiver 2199 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 2200 2201 receiver.addOutput(output.getBytes(), 0, output.length()); 2202 receiver.flush(); 2203 2204 return null; 2205 } 2206 }); 2207 } 2208 writeStringsToFile(File target, Set<String> strings)2209 static private void writeStringsToFile(File target, Set<String> strings) throws IOException { 2210 try (PrintWriter out = new PrintWriter(new FileWriter(target))) { 2211 out.print(String.join(System.lineSeparator(), strings)); 2212 out.println(); 2213 } 2214 } 2215 addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option)2216 private void addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option) 2217 throws IOException, ConfigurationException { 2218 String filterFile = option + ".txt"; 2219 writeStringsToFile(new File(mTestsDir, filterFile), filters); 2220 OptionSetter setter = new OptionSetter(test); 2221 setter.setOptionValue(option, filterFile); 2222 } 2223 testIncludeFilterFile()2224 public void testIncludeFilterFile() throws Exception { 2225 final TestDescription[] testIds = { 2226 new TestDescription("dEQP-GLES3.missing", "no"), 2227 new TestDescription("dEQP-GLES3.missing", "nope"), 2228 new TestDescription("dEQP-GLES3.missing", "donotwant"), 2229 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2230 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2231 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2232 }; 2233 2234 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2235 for (TestDescription id : testIds) { 2236 allTests.add(id); 2237 } 2238 2239 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2240 activeTests.add(testIds[3]); 2241 activeTests.add(testIds[4]); 2242 activeTests.add(testIds[5]); 2243 2244 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2245 2246 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2247 Set<String> includes = new HashSet<>(); 2248 includes.add("dEQP-GLES3.pick_me#*"); 2249 addFilterFileForOption(deqpTest, includes, "include-filter-file"); 2250 testFiltering(deqpTest, expectedTrie, activeTests); 2251 } 2252 testMissingIncludeFilterFile()2253 public void testMissingIncludeFilterFile() throws Exception { 2254 final TestDescription[] testIds = { 2255 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2256 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2257 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2258 }; 2259 2260 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2261 for (TestDescription id : testIds) { 2262 allTests.add(id); 2263 } 2264 2265 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2266 2267 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2268 OptionSetter setter = new OptionSetter(deqpTest); 2269 setter.setOptionValue("include-filter-file", "not-a-file.txt"); 2270 try { 2271 testFiltering(deqpTest, expectedTrie, allTests); 2272 fail("Test execution should have aborted with exception."); 2273 } catch (RuntimeException e) { 2274 } 2275 } 2276 testExcludeFilterFile()2277 public void testExcludeFilterFile() throws Exception { 2278 final TestDescription[] testIds = { 2279 new TestDescription("dEQP-GLES3.missing", "no"), 2280 new TestDescription("dEQP-GLES3.missing", "nope"), 2281 new TestDescription("dEQP-GLES3.missing", "donotwant"), 2282 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2283 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2284 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2285 }; 2286 2287 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2288 for (TestDescription id : testIds) { 2289 allTests.add(id); 2290 } 2291 2292 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2293 activeTests.add(testIds[3]); 2294 activeTests.add(testIds[4]); 2295 activeTests.add(testIds[5]); 2296 2297 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2298 2299 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2300 Set<String> excludes = new HashSet<>(); 2301 excludes.add("dEQP-GLES3.missing#*"); 2302 addFilterFileForOption(deqpTest, excludes, "exclude-filter-file"); 2303 testFiltering(deqpTest, expectedTrie, activeTests); 2304 } 2305 testFilterComboWithFiles()2306 public void testFilterComboWithFiles() throws Exception { 2307 final TestDescription[] testIds = { 2308 new TestDescription("dEQP-GLES3.group1", "footah"), 2309 new TestDescription("dEQP-GLES3.group1", "foo"), 2310 new TestDescription("dEQP-GLES3.group1", "nope"), 2311 new TestDescription("dEQP-GLES3.group1", "nonotwant"), 2312 new TestDescription("dEQP-GLES3.group2", "foo"), 2313 new TestDescription("dEQP-GLES3.group2", "yes"), 2314 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"), 2315 }; 2316 2317 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2318 for (TestDescription id : testIds) { 2319 allTests.add(id); 2320 } 2321 2322 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2323 activeTests.add(testIds[0]); 2324 activeTests.add(testIds[5]); 2325 2326 String expectedTrie = "{dEQP-GLES3{group1{footah}group2{yes}}}"; 2327 2328 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2329 2330 Set<String> includes = new HashSet<>(); 2331 includes.add("dEQP-GLES3.group2#*"); 2332 deqpTest.addAllIncludeFilters(includes); 2333 2334 Set<String> fileIncludes = new HashSet<>(); 2335 fileIncludes.add("dEQP-GLES3.group1#no*"); 2336 fileIncludes.add("dEQP-GLES3.group1#foo*"); 2337 addFilterFileForOption(deqpTest, fileIncludes, "include-filter-file"); 2338 2339 Set<String> fileExcludes = new HashSet<>(); 2340 fileExcludes.add("*foo"); 2341 fileExcludes.add("*thoushallnotpass"); 2342 addFilterFileForOption(deqpTest, fileExcludes, "exclude-filter-file"); 2343 2344 deqpTest.addExcludeFilter("dEQP-GLES3.group1#no*"); 2345 2346 testFiltering(deqpTest, expectedTrie, activeTests); 2347 } 2348 testDotToHashConversionInFilters()2349 public void testDotToHashConversionInFilters() throws Exception { 2350 final TestDescription[] testIds = { 2351 new TestDescription("dEQP-GLES3.missing", "no"), 2352 new TestDescription("dEQP-GLES3.pick_me", "donotwant"), 2353 new TestDescription("dEQP-GLES3.pick_me", "yes") 2354 }; 2355 2356 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2357 for (TestDescription id : testIds) { 2358 allTests.add(id); 2359 } 2360 2361 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2362 activeTests.add(testIds[2]); 2363 2364 String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}"; 2365 2366 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2367 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me.yes"); 2368 testFiltering(deqpTest, expectedTrie, activeTests); 2369 } 2370 } 2371