1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.drawelements.deqp.runner; 17 18 import static org.junit.Assert.assertSame; 19 import static org.junit.Assert.assertTrue; 20 21 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 22 import com.android.ddmlib.IDevice; 23 import com.android.ddmlib.IShellOutputReceiver; 24 import com.android.tradefed.build.IFolderBuildInfo; 25 import com.android.tradefed.config.ConfigurationException; 26 import com.android.tradefed.config.OptionSetter; 27 import com.android.tradefed.device.DeviceNotAvailableException; 28 import com.android.tradefed.device.IManagedTestDevice; 29 import com.android.tradefed.device.ITestDevice; 30 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 31 import com.android.tradefed.result.ITestInvocationListener; 32 import com.android.tradefed.result.TestDescription; 33 import com.android.tradefed.testtype.Abi; 34 import com.android.tradefed.testtype.IAbi; 35 import com.android.tradefed.testtype.IRemoteTest; 36 import com.android.tradefed.testtype.IRuntimeHintProvider; 37 import com.android.tradefed.testtype.NativeCodeCoverageListener; 38 import com.android.tradefed.util.AbiUtils; 39 import com.android.tradefed.util.FileUtil; 40 import com.android.tradefed.util.IRunUtil; 41 import com.android.tradefed.util.RunInterruptedException; 42 43 import junit.framework.TestCase; 44 45 import org.easymock.EasyMock; 46 import org.easymock.IAnswer; 47 import org.easymock.IMocksControl; 48 49 import java.io.File; 50 import java.io.FileNotFoundException; 51 import java.io.FileWriter; 52 import java.io.IOException; 53 import java.io.PrintWriter; 54 import java.io.StringReader; 55 import java.io.StringWriter; 56 import java.util.ArrayList; 57 import java.util.Collection; 58 import java.util.HashMap; 59 import java.util.HashSet; 60 import java.util.List; 61 import java.util.Map; 62 import java.util.Set; 63 import java.util.concurrent.TimeUnit; 64 65 /** 66 * Unit tests for {@link DeqpTestRunner}. 67 */ 68 public class DeqpTestRunnerTest extends TestCase { 69 private static final String NAME = "dEQP-GLES3"; 70 private static final IAbi ABI = new Abi("armeabi-v7a", "32"); 71 private static final String APP_DIR = "/sdcard/"; 72 private static final String CASE_LIST_FILE_NAME = "dEQP-TestCaseList.txt"; 73 private static final String LOG_FILE_NAME = "TestLog.qpa"; 74 private static final String INSTRUMENTATION_NAME = 75 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation"; 76 private static final String QUERY_INSTRUMENTATION_NAME = 77 "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation"; 78 private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk"; 79 private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp"; 80 private static final String ONLY_LANDSCAPE_FEATURES = 81 "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE; 82 private static final String ALL_FEATURES = 83 ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT; 84 private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS; 85 86 static { 87 DEFAULT_INSTANCE_ARGS = new ArrayList<>(1); DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>())88 DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>()); 89 DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8"); 90 DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified"); 91 DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window"); 92 } 93 94 private File mTestsDir = null; 95 96 public static class BuildHelperMock extends CompatibilityBuildHelper { 97 private File mTestsDir = null; BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir)98 public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) { 99 super(buildInfo); 100 mTestsDir = testsDir; 101 } 102 @Override getTestsDir()103 public File getTestsDir() throws FileNotFoundException { 104 return mTestsDir; 105 } 106 } 107 108 109 /** 110 * {@inheritDoc} 111 */ 112 @Override setUp()113 protected void setUp() throws Exception { 114 super.setUp(); 115 mTestsDir = FileUtil.createTempDir("deqp-test-cases"); 116 } 117 118 /** 119 * {@inheritDoc} 120 */ 121 @Override tearDown()122 protected void tearDown() throws Exception { 123 FileUtil.recursiveDelete(mTestsDir); 124 super.tearDown(); 125 } 126 buildGlesTestRunner(int majorVersion, int minorVersion, Collection<TestDescription> tests, File testsDir)127 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, 128 int minorVersion, 129 Collection<TestDescription> tests, 130 File testsDir) throws ConfigurationException { 131 StringWriter testlist = new StringWriter(); 132 for (TestDescription test : tests) { 133 testlist.write(test.getClassName() + "." + test.getTestName() + "\n"); 134 } 135 return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString(), testsDir); 136 } 137 getMockBuildHelper(File testsDir)138 private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) { 139 IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class); 140 EasyMock.replay(mockIFolderBuildInfo); 141 return new BuildHelperMock(mockIFolderBuildInfo, testsDir); 142 } 143 buildGlesTestRunner(int majorVersion, int minorVersion, String testlist, File testsDir)144 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, 145 int minorVersion, 146 String testlist, 147 File testsDir) throws ConfigurationException { 148 DeqpTestRunner runner = new DeqpTestRunner(); 149 OptionSetter setter = new OptionSetter(runner); 150 151 String deqpPackage = "dEQP-GLES" + Integer.toString(majorVersion) 152 + (minorVersion > 0 ? Integer.toString(minorVersion) : ""); 153 154 setter.setOptionValue("deqp-package", deqpPackage); 155 setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8"); 156 setter.setOptionValue("deqp-caselist-file", "dummyfile.txt"); 157 setter.setOptionValue("deqp-screen-rotation", "unspecified"); 158 setter.setOptionValue("deqp-surface-type", "window"); 159 160 runner.setCaselistReader(new StringReader(testlist)); 161 runner.setAbi(ABI); 162 runner.setBuildHelper(getMockBuildHelper(testsDir)); 163 164 return runner; 165 } 166 getTestId(DeqpTestRunner runner)167 private static String getTestId(DeqpTestRunner runner) { 168 return AbiUtils.createId(ABI.getName(), runner.getPackageName()); 169 } 170 171 /** 172 * Test version of OpenGL ES. 173 */ testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion)174 private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception { 175 final TestDescription testId = new TestDescription("dEQP-GLES" 176 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 177 + ".info", "version"); 178 179 final String testPath = "dEQP-GLES" 180 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 181 +".info.version"; 182 183 final String testTrie = "{dEQP-GLES" 184 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 185 + "{info{version}}}"; 186 187 final String resultCode = "Pass"; 188 189 /* MultiLineReceiver expects "\r\n" line ending. */ 190 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 191 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 192 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 193 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 194 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 195 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 196 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 197 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 198 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 199 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 200 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 201 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 202 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 203 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 204 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 205 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 206 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 207 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 208 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 209 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 210 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 211 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 212 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 213 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 214 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 215 + "INSTRUMENTATION_CODE: 0\r\n"; 216 217 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 218 ITestInvocationListener mockListener 219 = EasyMock.createStrictMock(ITestInvocationListener.class); 220 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 221 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 222 tests.add(testId); 223 224 DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests, mTestsDir); 225 226 int version = (majorVersion << 16) | minorVersion; 227 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 228 .andReturn(Integer.toString(version)).atLeastOnce(); 229 230 if (majorVersion > requiredMajorVersion 231 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) { 232 233 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 234 andReturn("").once(); 235 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 236 andReturn("").once(); 237 238 expectRenderConfigQuery(mockDevice, requiredMajorVersion, 239 requiredMinorVersion); 240 241 String commandLine = String.format( 242 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 243 + "--deqp-screen-rotation=unspecified " 244 + "--deqp-surface-type=window " 245 + "--deqp-log-images=disable " 246 + "--deqp-watchdog=enable", 247 APP_DIR + CASE_LIST_FILE_NAME); 248 249 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 250 output); 251 } 252 253 mockListener.testRunStarted(getTestId(deqpTest), 1); 254 EasyMock.expectLastCall().once(); 255 256 mockListener.testStarted(EasyMock.eq(testId)); 257 EasyMock.expectLastCall().once(); 258 259 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 260 EasyMock.expectLastCall().once(); 261 262 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 263 EasyMock.expectLastCall().once(); 264 265 EasyMock.replay(mockDevice, mockIDevice); 266 EasyMock.replay(mockListener); 267 268 deqpTest.setDevice(mockDevice); 269 deqpTest.run(mockListener); 270 271 EasyMock.verify(mockListener); 272 EasyMock.verify(mockDevice, mockIDevice); 273 } 274 expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, int minorVersion)275 private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, 276 int minorVersion) throws Exception { 277 expectRenderConfigQuery(mockDevice, 278 String.format("--deqp-gl-config-name=rgba8888d24s8 " 279 + "--deqp-screen-rotation=unspecified " 280 + "--deqp-surface-type=window " 281 + "--deqp-gl-major-version=%d " 282 + "--deqp-gl-minor-version=%d", majorVersion, minorVersion)); 283 } 284 expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)285 private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine) 286 throws Exception { 287 expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes"); 288 } 289 expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, String output)290 private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, 291 String output) throws Exception { 292 final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n" 293 + "INSTRUMENTATION_CODE: 0\r\n"; 294 final String command = String.format( 295 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine " 296 + "\"%s\" %s", 297 AbiUtils.createAbiFlag(ABI.getName()), commandLine, 298 QUERY_INSTRUMENTATION_NAME); 299 300 mockDevice.executeShellCommand(EasyMock.eq(command), 301 EasyMock.<IShellOutputReceiver>notNull()); 302 303 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 304 @Override 305 public Object answer() { 306 IShellOutputReceiver receiver 307 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 308 309 receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length()); 310 receiver.flush(); 311 312 return null; 313 } 314 }); 315 } 316 317 /** 318 * Test that result code produces correctly pass or fail. 319 */ testResultCode(final String resultCode, boolean pass)320 private void testResultCode(final String resultCode, boolean pass) throws Exception { 321 final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version"); 322 final String testPath = "dEQP-GLES3.info.version"; 323 final String testTrie = "{dEQP-GLES3{info{version}}}"; 324 325 /* MultiLineReceiver expects "\r\n" line ending. */ 326 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 327 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 328 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 329 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 330 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 331 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 332 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 333 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 334 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 335 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 336 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 337 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 338 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 339 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 340 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 341 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 342 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 343 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 344 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 345 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 346 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 347 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 348 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 349 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 350 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 351 + "INSTRUMENTATION_CODE: 0\r\n"; 352 353 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 354 ITestInvocationListener mockListener 355 = EasyMock.createStrictMock(ITestInvocationListener.class); 356 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 357 358 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 359 tests.add(testId); 360 361 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 362 363 int version = 3 << 16; 364 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 365 .andReturn(Integer.toString(version)).atLeastOnce(); 366 367 expectRenderConfigQuery(mockDevice, 3, 0); 368 369 String commandLine = String.format( 370 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 371 + "--deqp-screen-rotation=unspecified " 372 + "--deqp-surface-type=window " 373 + "--deqp-log-images=disable " 374 + "--deqp-watchdog=enable", 375 APP_DIR + CASE_LIST_FILE_NAME); 376 377 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 378 379 mockListener.testRunStarted(getTestId(deqpTest), 1); 380 EasyMock.expectLastCall().once(); 381 382 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 383 andReturn("").once(); 384 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 385 andReturn("").once(); 386 387 mockListener.testStarted(EasyMock.eq(testId)); 388 EasyMock.expectLastCall().once(); 389 390 if (!pass) { 391 mockListener.testFailed(testId, 392 "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n" 393 + resultCode + ": Detail" + resultCode); 394 395 EasyMock.expectLastCall().once(); 396 } 397 398 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 399 EasyMock.expectLastCall().once(); 400 401 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 402 EasyMock.expectLastCall().once(); 403 404 EasyMock.replay(mockDevice, mockIDevice); 405 EasyMock.replay(mockListener); 406 407 deqpTest.setDevice(mockDevice); 408 deqpTest.run(mockListener); 409 410 EasyMock.verify(mockListener); 411 EasyMock.verify(mockDevice, mockIDevice); 412 } 413 414 /** 415 * Test running multiple test cases. 416 */ testRun_multipleTests()417 public void testRun_multipleTests() throws Exception { 418 /* MultiLineReceiver expects "\r\n" line ending. */ 419 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 420 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 421 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 422 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 423 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 424 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 425 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 426 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 427 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 428 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 429 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 430 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 431 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 432 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 433 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 434 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 435 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 436 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 437 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 438 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 439 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 440 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 441 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 442 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 443 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 444 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 445 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 446 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 447 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 448 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 449 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 450 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 451 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 452 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 453 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 454 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 455 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 456 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 457 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 458 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 459 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 460 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 461 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 462 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 463 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 464 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 465 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 466 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 467 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 468 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 469 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 470 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 471 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 472 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 473 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 474 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 475 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 476 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 477 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 478 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 479 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 480 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 481 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 482 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 483 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 484 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 485 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 486 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 487 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 488 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 489 + "INSTRUMENTATION_CODE: 0\r\n"; 490 491 final TestDescription[] testIds = { 492 new TestDescription("dEQP-GLES3.info", "vendor"), 493 new TestDescription("dEQP-GLES3.info", "renderer"), 494 new TestDescription("dEQP-GLES3.info", "version"), 495 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 496 new TestDescription("dEQP-GLES3.info", "extensions"), 497 new TestDescription("dEQP-GLES3.info", "render_target") 498 }; 499 500 final String[] testPaths = { 501 "dEQP-GLES3.info.vendor", 502 "dEQP-GLES3.info.renderer", 503 "dEQP-GLES3.info.version", 504 "dEQP-GLES3.info.shading_language_version", 505 "dEQP-GLES3.info.extensions", 506 "dEQP-GLES3.info.render_target" 507 }; 508 509 final String testTrie 510 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 511 512 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 513 ITestInvocationListener mockListener 514 = EasyMock.createStrictMock(ITestInvocationListener.class); 515 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 516 517 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 518 519 for (TestDescription id : testIds) { 520 tests.add(id); 521 } 522 523 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 524 525 int version = 3 << 16; 526 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 527 .andReturn(Integer.toString(version)).atLeastOnce(); 528 529 expectRenderConfigQuery(mockDevice, 3, 0); 530 531 String commandLine = String.format( 532 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 533 + "--deqp-screen-rotation=unspecified " 534 + "--deqp-surface-type=window " 535 + "--deqp-log-images=disable " 536 + "--deqp-watchdog=enable", 537 APP_DIR + CASE_LIST_FILE_NAME); 538 539 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 540 541 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 542 EasyMock.expectLastCall().once(); 543 544 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 545 andReturn("").once(); 546 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 547 andReturn("").once(); 548 549 for (int i = 0; i < testPaths.length; i++) { 550 mockListener.testStarted(EasyMock.eq(testIds[i])); 551 EasyMock.expectLastCall().once(); 552 553 mockListener.testEnded(EasyMock.eq(testIds[i]), 554 EasyMock.<HashMap<String, Metric>>notNull()); 555 556 EasyMock.expectLastCall().once(); 557 } 558 559 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 560 EasyMock.expectLastCall().once(); 561 562 EasyMock.replay(mockDevice, mockIDevice); 563 EasyMock.replay(mockListener); 564 565 deqpTest.setDevice(mockDevice); 566 deqpTest.run(mockListener); 567 568 EasyMock.verify(mockListener); 569 EasyMock.verify(mockDevice, mockIDevice); 570 } 571 buildTestProcessOutput(List<TestDescription> tests)572 static private String buildTestProcessOutput(List<TestDescription> tests) { 573 /* MultiLineReceiver expects "\r\n" line ending. */ 574 final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 575 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 576 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 577 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 578 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 579 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 580 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 581 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 582 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 583 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 584 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 585 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 586 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 587 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; 588 589 final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 590 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 591 + "INSTRUMENTATION_CODE: 0\r\n"; 592 593 StringWriter output = new StringWriter(); 594 output.write(outputHeader); 595 for (TestDescription test : tests) { 596 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"); 597 output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath="); 598 output.write(test.getClassName()); 599 output.write("."); 600 output.write(test.getTestName()); 601 output.write("\r\n"); 602 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 603 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"); 604 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"); 605 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"); 606 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 607 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"); 608 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 609 } 610 output.write(outputEnd); 611 return output.toString(); 612 } 613 testFiltering(DeqpTestRunner deqpTest, String expectedTrie, List<TestDescription> expectedTests)614 private void testFiltering(DeqpTestRunner deqpTest, 615 String expectedTrie, 616 List<TestDescription> expectedTests) throws Exception { 617 int version = 3 << 16; 618 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 619 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 620 .andReturn(Integer.toString(version)).atLeastOnce(); 621 622 boolean thereAreTests = !expectedTests.isEmpty(); 623 624 ITestInvocationListener mockListener 625 = EasyMock.createStrictMock(ITestInvocationListener.class); 626 mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size()); 627 EasyMock.expectLastCall().once(); 628 629 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 630 andReturn("").once(); 631 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 632 andReturn("").once(); 633 634 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 635 if (thereAreTests) 636 { 637 expectRenderConfigQuery(mockDevice, 3, 0); 638 639 String testOut = buildTestProcessOutput(expectedTests); 640 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); 641 642 for (int i = 0; i < expectedTests.size(); i++) { 643 mockListener.testStarted(EasyMock.eq(expectedTests.get(i))); 644 EasyMock.expectLastCall().once(); 645 646 mockListener.testEnded(EasyMock.eq(expectedTests.get(i)), 647 EasyMock.<HashMap<String, Metric>>notNull()); 648 649 EasyMock.expectLastCall().once(); 650 } 651 } 652 653 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 654 EasyMock.expectLastCall().once(); 655 656 EasyMock.replay(mockDevice, mockIDevice); 657 EasyMock.replay(mockListener); 658 659 deqpTest.setDevice(mockDevice); 660 deqpTest.run(mockListener); 661 662 EasyMock.verify(mockListener); 663 EasyMock.verify(mockDevice, mockIDevice); 664 } 665 testRun_trivialIncludeFilter()666 public void testRun_trivialIncludeFilter() throws Exception { 667 final TestDescription[] testIds = { 668 new TestDescription("dEQP-GLES3.missing", "no"), 669 new TestDescription("dEQP-GLES3.missing", "nope"), 670 new TestDescription("dEQP-GLES3.missing", "donotwant"), 671 new TestDescription("dEQP-GLES3.pick_me", "yes"), 672 new TestDescription("dEQP-GLES3.pick_me", "ok"), 673 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 674 }; 675 676 List<TestDescription> allTests = new ArrayList<TestDescription>(); 677 for (TestDescription id : testIds) { 678 allTests.add(id); 679 } 680 681 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 682 activeTests.add(testIds[3]); 683 activeTests.add(testIds[4]); 684 activeTests.add(testIds[5]); 685 686 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 687 688 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 689 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me#*"); 690 testFiltering(deqpTest, expectedTrie, activeTests); 691 } 692 testRun_trivialExcludeFilter()693 public void testRun_trivialExcludeFilter() throws Exception { 694 final TestDescription[] testIds = { 695 new TestDescription("dEQP-GLES3.missing", "no"), 696 new TestDescription("dEQP-GLES3.missing", "nope"), 697 new TestDescription("dEQP-GLES3.missing", "donotwant"), 698 new TestDescription("dEQP-GLES3.pick_me", "yes"), 699 new TestDescription("dEQP-GLES3.pick_me", "ok"), 700 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 701 }; 702 703 List<TestDescription> allTests = new ArrayList<TestDescription>(); 704 for (TestDescription id : testIds) { 705 allTests.add(id); 706 } 707 708 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 709 activeTests.add(testIds[3]); 710 activeTests.add(testIds[4]); 711 activeTests.add(testIds[5]); 712 713 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 714 715 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 716 deqpTest.addExcludeFilter("dEQP-GLES3.missing#*"); 717 testFiltering(deqpTest, expectedTrie, activeTests); 718 } 719 testRun_includeAndExcludeFilter()720 public void testRun_includeAndExcludeFilter() throws Exception { 721 final TestDescription[] testIds = { 722 new TestDescription("dEQP-GLES3.group1", "foo"), 723 new TestDescription("dEQP-GLES3.group1", "nope"), 724 new TestDescription("dEQP-GLES3.group1", "donotwant"), 725 new TestDescription("dEQP-GLES3.group2", "foo"), 726 new TestDescription("dEQP-GLES3.group2", "yes"), 727 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"), 728 }; 729 730 List<TestDescription> allTests = new ArrayList<TestDescription>(); 731 for (TestDescription id : testIds) { 732 allTests.add(id); 733 } 734 735 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 736 activeTests.add(testIds[4]); 737 738 String expectedTrie = "{dEQP-GLES3{group2{yes}}}"; 739 740 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 741 742 Set<String> includes = new HashSet<>(); 743 includes.add("dEQP-GLES3.group2#*"); 744 deqpTest.addAllIncludeFilters(includes); 745 746 Set<String> excludes = new HashSet<>(); 747 excludes.add("*foo"); 748 excludes.add("*thoushallnotpass"); 749 deqpTest.addAllExcludeFilters(excludes); 750 testFiltering(deqpTest, expectedTrie, activeTests); 751 } 752 testRun_includeAll()753 public void testRun_includeAll() throws Exception { 754 final TestDescription[] testIds = { 755 new TestDescription("dEQP-GLES3.group1", "mememe"), 756 new TestDescription("dEQP-GLES3.group1", "yeah"), 757 new TestDescription("dEQP-GLES3.group1", "takeitall"), 758 new TestDescription("dEQP-GLES3.group2", "jeba"), 759 new TestDescription("dEQP-GLES3.group2", "yes"), 760 new TestDescription("dEQP-GLES3.group2", "granted"), 761 }; 762 763 List<TestDescription> allTests = new ArrayList<TestDescription>(); 764 for (TestDescription id : testIds) { 765 allTests.add(id); 766 } 767 768 String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}"; 769 770 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 771 deqpTest.addIncludeFilter("*"); 772 testFiltering(deqpTest, expectedTrie, allTests); 773 } 774 testRun_excludeAll()775 public void testRun_excludeAll() throws Exception { 776 final TestDescription[] testIds = { 777 new TestDescription("dEQP-GLES3.group1", "no"), 778 new TestDescription("dEQP-GLES3.group1", "nope"), 779 new TestDescription("dEQP-GLES3.group1", "nottoday"), 780 new TestDescription("dEQP-GLES3.group2", "banned"), 781 new TestDescription("dEQP-GLES3.group2", "notrecognized"), 782 new TestDescription("dEQP-GLES3.group2", "-2"), 783 }; 784 785 List<TestDescription> allTests = new ArrayList<TestDescription>(); 786 for (TestDescription id : testIds) { 787 allTests.add(id); 788 } 789 790 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 791 deqpTest.addExcludeFilter("*"); 792 ITestInvocationListener mockListener 793 = EasyMock.createStrictMock(ITestInvocationListener.class); 794 mockListener.testRunStarted(getTestId(deqpTest), 0); 795 EasyMock.expectLastCall().once(); 796 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 797 EasyMock.expectLastCall().once(); 798 799 EasyMock.replay(mockListener); 800 deqpTest.run(mockListener); 801 EasyMock.verify(mockListener); 802 } 803 804 /** 805 * Test running a unexecutable test. 806 */ testRun_unexecutableTests()807 public void testRun_unexecutableTests() throws Exception { 808 final String instrumentationAnswerNoExecs = 809 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 810 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 811 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 812 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 813 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 814 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 815 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 816 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 817 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 818 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 819 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 820 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 821 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 822 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 823 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 824 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 825 + "INSTRUMENTATION_CODE: 0\r\n"; 826 827 final TestDescription[] testIds = { 828 new TestDescription("dEQP-GLES3.missing", "no"), 829 new TestDescription("dEQP-GLES3.missing", "nope"), 830 new TestDescription("dEQP-GLES3.missing", "donotwant"), 831 }; 832 833 final String[] testPaths = { 834 "dEQP-GLES3.missing.no", 835 "dEQP-GLES3.missing.nope", 836 "dEQP-GLES3.missing.donotwant", 837 }; 838 839 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 840 ITestInvocationListener mockListener 841 = EasyMock.createStrictMock(ITestInvocationListener.class); 842 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 843 844 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 845 846 for (TestDescription id : testIds) { 847 tests.add(id); 848 } 849 850 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 851 852 int version = 3 << 16; 853 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 854 .andReturn(Integer.toString(version)).atLeastOnce(); 855 856 expectRenderConfigQuery(mockDevice, 3, 0); 857 858 String commandLine = String.format( 859 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 860 + "--deqp-screen-rotation=unspecified " 861 + "--deqp-surface-type=window " 862 + "--deqp-log-images=disable " 863 + "--deqp-watchdog=enable", 864 APP_DIR + CASE_LIST_FILE_NAME); 865 866 // first try 867 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 868 "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs); 869 870 // splitting begins 871 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 872 "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs); 873 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 874 "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs); 875 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 876 "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs); 877 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 878 "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs); 879 880 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 881 EasyMock.expectLastCall().once(); 882 883 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 884 andReturn("").once(); 885 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 886 andReturn("").once(); 887 888 for (int i = 0; i < testPaths.length; i++) { 889 mockListener.testStarted(EasyMock.eq(testIds[i])); 890 EasyMock.expectLastCall().once(); 891 892 mockListener.testFailed(EasyMock.eq(testIds[i]), 893 EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n" 894 + "Abort: Test cannot be executed")); 895 EasyMock.expectLastCall().once(); 896 897 mockListener.testEnded(EasyMock.eq(testIds[i]), 898 EasyMock.<HashMap<String, Metric>>notNull()); 899 EasyMock.expectLastCall().once(); 900 } 901 902 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 903 EasyMock.expectLastCall().once(); 904 905 EasyMock.replay(mockDevice, mockIDevice); 906 EasyMock.replay(mockListener); 907 908 deqpTest.setDevice(mockDevice); 909 deqpTest.run(mockListener); 910 911 EasyMock.verify(mockListener); 912 EasyMock.verify(mockDevice, mockIDevice); 913 } 914 915 /** 916 * Test that test are left unexecuted if pm list query fails 917 */ testRun_queryPmListFailure()918 public void testRun_queryPmListFailure() 919 throws Exception { 920 final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test"); 921 922 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 923 ITestInvocationListener mockListener 924 = EasyMock.createStrictMock(ITestInvocationListener.class); 925 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 926 tests.add(testId); 927 928 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 929 OptionSetter setter = new OptionSetter(deqpTest); 930 // Note: If the rotation is the default unspecified, features are not queried at all 931 setter.setOptionValue("deqp-screen-rotation", "90"); 932 933 deqpTest.setDevice(mockDevice); 934 935 int version = 3 << 16; 936 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 937 .andReturn(Integer.toString(version)).atLeastOnce(); 938 939 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 940 .andReturn("not a valid format"); 941 942 mockListener.testRunStarted(getTestId(deqpTest), 1); 943 EasyMock.expectLastCall().once(); 944 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 945 andReturn("").once(); 946 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 947 andReturn("").once(); 948 949 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 950 EasyMock.expectLastCall().once(); 951 952 EasyMock.replay(mockDevice); 953 EasyMock.replay(mockListener); 954 deqpTest.run(mockListener); 955 EasyMock.verify(mockListener); 956 EasyMock.verify(mockDevice); 957 } 958 959 /** 960 * Test that test are left unexecuted if renderablity query fails 961 */ testRun_queryRenderabilityFailure()962 public void testRun_queryRenderabilityFailure() 963 throws Exception { 964 final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test"); 965 966 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 967 ITestInvocationListener mockListener 968 = EasyMock.createStrictMock(ITestInvocationListener.class); 969 970 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 971 tests.add(testId); 972 973 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 974 975 deqpTest.setDevice(mockDevice); 976 977 int version = 3 << 16; 978 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 979 .andReturn(Integer.toString(version)).atLeastOnce(); 980 981 expectRenderConfigQueryAndReturn(mockDevice, 982 "--deqp-gl-config-name=rgba8888d24s8 " 983 + "--deqp-screen-rotation=unspecified " 984 + "--deqp-surface-type=window " 985 + "--deqp-gl-major-version=3 " 986 + "--deqp-gl-minor-version=0", "Maybe?"); 987 988 mockListener.testRunStarted(getTestId(deqpTest), 1); 989 EasyMock.expectLastCall().once(); 990 991 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 992 andReturn("").once(); 993 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 994 andReturn("").once(); 995 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 996 EasyMock.expectLastCall().once(); 997 998 EasyMock.replay(mockDevice); 999 EasyMock.replay(mockListener); 1000 deqpTest.run(mockListener); 1001 EasyMock.verify(mockListener); 1002 EasyMock.verify(mockDevice); 1003 } 1004 1005 /** 1006 * Test that orientation is supplied to runner correctly 1007 */ testOrientation(final String rotation, final String featureString)1008 private void testOrientation(final String rotation, final String featureString) 1009 throws Exception { 1010 final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test"); 1011 final String testPath = "dEQP-GLES3.orientation.test"; 1012 final String testTrie = "{dEQP-GLES3{orientation{test}}}"; 1013 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1014 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1015 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1016 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1017 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1018 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1019 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1020 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1021 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1022 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1023 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1024 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1025 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1026 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1027 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1028 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 1029 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1030 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1031 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1032 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1033 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1034 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1035 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1036 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1037 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1038 + "INSTRUMENTATION_CODE: 0\r\n"; 1039 1040 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1041 ITestInvocationListener mockListener 1042 = EasyMock.createStrictMock(ITestInvocationListener.class); 1043 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1044 1045 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1046 tests.add(testId); 1047 1048 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1049 OptionSetter setter = new OptionSetter(deqpTest); 1050 setter.setOptionValue("deqp-screen-rotation", rotation); 1051 1052 deqpTest.setDevice(mockDevice); 1053 1054 int version = 3 << 16; 1055 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1056 .andReturn(Integer.toString(version)).atLeastOnce(); 1057 1058 if (!rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED)) { 1059 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 1060 .andReturn(featureString); 1061 } 1062 1063 final boolean isPortraitOrientation = 1064 rotation.equals(BatchRunConfiguration.ROTATION_PORTRAIT) || 1065 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT); 1066 final boolean isLandscapeOrientation = 1067 rotation.equals(BatchRunConfiguration.ROTATION_LANDSCAPE) || 1068 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE); 1069 final boolean executable = 1070 rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED) || 1071 (isPortraitOrientation && 1072 featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) || 1073 (isLandscapeOrientation && 1074 featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE)); 1075 1076 if (executable) { 1077 expectRenderConfigQuery(mockDevice, String.format( 1078 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s " 1079 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1080 + "--deqp-gl-minor-version=0", rotation)); 1081 1082 String commandLine = String.format( 1083 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1084 + "--deqp-screen-rotation=%s " 1085 + "--deqp-surface-type=window " 1086 + "--deqp-log-images=disable " 1087 + "--deqp-watchdog=enable", 1088 APP_DIR + CASE_LIST_FILE_NAME, rotation); 1089 1090 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 1091 output); 1092 } 1093 1094 mockListener.testRunStarted(getTestId(deqpTest), 1); 1095 EasyMock.expectLastCall().once(); 1096 1097 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 1098 andReturn("").once(); 1099 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 1100 andReturn("").once(); 1101 1102 mockListener.testStarted(EasyMock.eq(testId)); 1103 EasyMock.expectLastCall().once(); 1104 1105 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 1106 EasyMock.expectLastCall().once(); 1107 1108 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1109 EasyMock.expectLastCall().once(); 1110 1111 EasyMock.replay(mockDevice, mockIDevice); 1112 EasyMock.replay(mockListener); 1113 deqpTest.run(mockListener); 1114 EasyMock.verify(mockListener); 1115 EasyMock.verify(mockDevice, mockIDevice); 1116 } 1117 1118 /** 1119 * Test OpeGL ES3 tests on device with OpenGL ES2. 1120 */ testRun_require30DeviceVersion20()1121 public void testRun_require30DeviceVersion20() throws Exception { 1122 testGlesVersion(3, 0, 2, 0); 1123 } 1124 1125 /** 1126 * Test OpeGL ES3.1 tests on device with OpenGL ES2. 1127 */ testRun_require31DeviceVersion20()1128 public void testRun_require31DeviceVersion20() throws Exception { 1129 testGlesVersion(3, 1, 2, 0); 1130 } 1131 1132 /** 1133 * Test OpeGL ES3 tests on device with OpenGL ES3. 1134 */ testRun_require30DeviceVersion30()1135 public void testRun_require30DeviceVersion30() throws Exception { 1136 testGlesVersion(3, 0, 3, 0); 1137 } 1138 1139 /** 1140 * Test OpeGL ES3.1 tests on device with OpenGL ES3. 1141 */ testRun_require31DeviceVersion30()1142 public void testRun_require31DeviceVersion30() throws Exception { 1143 testGlesVersion(3, 1, 3, 0); 1144 } 1145 1146 /** 1147 * Test OpeGL ES3 tests on device with OpenGL ES3.1. 1148 */ testRun_require30DeviceVersion31()1149 public void testRun_require30DeviceVersion31() throws Exception { 1150 testGlesVersion(3, 0, 3, 1); 1151 } 1152 1153 /** 1154 * Test OpeGL ES3.1 tests on device with OpenGL ES3.1. 1155 */ testRun_require31DeviceVersion31()1156 public void testRun_require31DeviceVersion31() throws Exception { 1157 testGlesVersion(3, 1, 3, 1); 1158 } 1159 1160 /** 1161 * Test dEQP Pass result code. 1162 */ testRun_resultPass()1163 public void testRun_resultPass() throws Exception { 1164 testResultCode("Pass", true); 1165 } 1166 1167 /** 1168 * Test dEQP Fail result code. 1169 */ testRun_resultFail()1170 public void testRun_resultFail() throws Exception { 1171 testResultCode("Fail", false); 1172 } 1173 1174 /** 1175 * Test dEQP NotSupported result code. 1176 */ testRun_resultNotSupported()1177 public void testRun_resultNotSupported() throws Exception { 1178 testResultCode("NotSupported", true); 1179 } 1180 1181 /** 1182 * Test dEQP QualityWarning result code. 1183 */ testRun_resultQualityWarning()1184 public void testRun_resultQualityWarning() throws Exception { 1185 testResultCode("QualityWarning", true); 1186 } 1187 1188 /** 1189 * Test dEQP CompatibilityWarning result code. 1190 */ testRun_resultCompatibilityWarning()1191 public void testRun_resultCompatibilityWarning() throws Exception { 1192 testResultCode("CompatibilityWarning", true); 1193 } 1194 1195 /** 1196 * Test dEQP ResourceError result code. 1197 */ testRun_resultResourceError()1198 public void testRun_resultResourceError() throws Exception { 1199 testResultCode("ResourceError", false); 1200 } 1201 1202 /** 1203 * Test dEQP InternalError result code. 1204 */ testRun_resultInternalError()1205 public void testRun_resultInternalError() throws Exception { 1206 testResultCode("InternalError", false); 1207 } 1208 1209 /** 1210 * Test dEQP Crash result code. 1211 */ testRun_resultCrash()1212 public void testRun_resultCrash() throws Exception { 1213 testResultCode("Crash", false); 1214 } 1215 1216 /** 1217 * Test dEQP Timeout result code. 1218 */ testRun_resultTimeout()1219 public void testRun_resultTimeout() throws Exception { 1220 testResultCode("Timeout", false); 1221 } 1222 /** 1223 * Test dEQP Orientation 1224 */ testRun_orientationLandscape()1225 public void testRun_orientationLandscape() throws Exception { 1226 testOrientation("90", ALL_FEATURES); 1227 } 1228 1229 /** 1230 * Test dEQP Orientation 1231 */ testRun_orientationPortrait()1232 public void testRun_orientationPortrait() throws Exception { 1233 testOrientation("0", ALL_FEATURES); 1234 } 1235 1236 /** 1237 * Test dEQP Orientation 1238 */ testRun_orientationReverseLandscape()1239 public void testRun_orientationReverseLandscape() throws Exception { 1240 testOrientation("270", ALL_FEATURES); 1241 } 1242 1243 /** 1244 * Test dEQP Orientation 1245 */ testRun_orientationReversePortrait()1246 public void testRun_orientationReversePortrait() throws Exception { 1247 testOrientation("180", ALL_FEATURES); 1248 } 1249 1250 /** 1251 * Test dEQP Orientation 1252 */ testRun_orientationUnspecified()1253 public void testRun_orientationUnspecified() throws Exception { 1254 testOrientation("unspecified", ALL_FEATURES); 1255 } 1256 1257 /** 1258 * Test dEQP Orientation with limited features 1259 */ testRun_orientationUnspecifiedLimitedFeatures()1260 public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception { 1261 testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES); 1262 } 1263 1264 /** 1265 * Test dEQP Orientation with limited features 1266 */ testRun_orientationLandscapeLimitedFeatures()1267 public void testRun_orientationLandscapeLimitedFeatures() throws Exception { 1268 testOrientation("90", ONLY_LANDSCAPE_FEATURES); 1269 } 1270 1271 /** 1272 * Test dEQP Orientation with limited features 1273 */ testRun_orientationPortraitLimitedFeatures()1274 public void testRun_orientationPortraitLimitedFeatures() throws Exception { 1275 testOrientation("0", ONLY_LANDSCAPE_FEATURES); 1276 } 1277 1278 /** 1279 * Test dEQP unsupported pixel format 1280 */ testRun_unsupportedPixelFormat()1281 public void testRun_unsupportedPixelFormat() throws Exception { 1282 final String pixelFormat = "rgba5658d16m4"; 1283 final TestDescription testId = new TestDescription("dEQP-GLES3.pixelformat", "test"); 1284 1285 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1286 ITestInvocationListener mockListener 1287 = EasyMock.createStrictMock(ITestInvocationListener.class); 1288 1289 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1290 tests.add(testId); 1291 1292 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1293 OptionSetter setter = new OptionSetter(deqpTest); 1294 setter.setOptionValue("deqp-gl-config-name", pixelFormat); 1295 1296 deqpTest.setDevice(mockDevice); 1297 1298 int version = 3 << 16; 1299 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1300 .andReturn(Integer.toString(version)).atLeastOnce(); 1301 1302 expectRenderConfigQueryAndReturn(mockDevice, String.format( 1303 "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified " 1304 + "--deqp-surface-type=window " 1305 + "--deqp-gl-major-version=3 " 1306 + "--deqp-gl-minor-version=0", pixelFormat), "No"); 1307 1308 mockListener.testRunStarted(getTestId(deqpTest), 1); 1309 EasyMock.expectLastCall().once(); 1310 1311 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 1312 andReturn("").once(); 1313 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 1314 andReturn("").once(); 1315 1316 mockListener.testStarted(EasyMock.eq(testId)); 1317 EasyMock.expectLastCall().once(); 1318 1319 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 1320 EasyMock.expectLastCall().once(); 1321 1322 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1323 EasyMock.expectLastCall().once(); 1324 1325 EasyMock.replay(mockDevice); 1326 EasyMock.replay(mockListener); 1327 deqpTest.run(mockListener); 1328 EasyMock.verify(mockListener); 1329 EasyMock.verify(mockDevice); 1330 } 1331 1332 /** 1333 * Test interface to mock Tradefed device types. 1334 */ 1335 public static interface RecoverableTestDevice extends ITestDevice, IManagedTestDevice { 1336 } 1337 1338 private static enum RecoveryEvent { 1339 PROGRESS, 1340 FAIL_CONNECTION_REFUSED, 1341 FAIL_LINK_KILLED, 1342 } 1343 runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)1344 private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events) 1345 throws DeviceNotAvailableException { 1346 for (RecoveryEvent event : events) { 1347 switch (event) { 1348 case PROGRESS: 1349 recovery.onExecutionProgressed(); 1350 break; 1351 case FAIL_CONNECTION_REFUSED: 1352 recovery.recoverConnectionRefused(); 1353 break; 1354 case FAIL_LINK_KILLED: 1355 recovery.recoverComLinkKilled(); 1356 break; 1357 } 1358 } 1359 } 1360 setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider)1361 private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) { 1362 mockSleepProvider.sleep(EasyMock.gt(0)); 1363 EasyMock.expectLastCall().once(); 1364 } 1365 setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider)1366 private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, 1367 DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException { 1368 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1369 andReturn("root 1234 com.drawelement.deqp").once(); 1370 1371 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1372 andReturn("").once(); 1373 1374 // Recovery checks if kill failed 1375 mockSleepProvider.sleep(EasyMock.gt(0)); 1376 EasyMock.expectLastCall().once(); 1377 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1378 andReturn("").once(); 1379 } 1380 setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)1381 private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice) 1382 throws DeviceNotAvailableException { 1383 mockDevice.recoverDevice(); 1384 EasyMock.expectLastCall().once(); 1385 } 1386 setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)1387 private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice) 1388 throws DeviceNotAvailableException { 1389 mockDevice.reboot(); 1390 EasyMock.expectLastCall().once(); 1391 } 1392 setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)1393 private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, 1394 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 1395 throws DeviceNotAvailableException { 1396 switch (numConsecutiveErrors) { 1397 case 0: 1398 case 1: 1399 setRecoveryExpectationRecovery(mockDevice); 1400 return 2; 1401 case 2: 1402 setRecoveryExpectationReboot(mockDevice); 1403 return 3; 1404 default: 1405 return 4; 1406 } 1407 } 1408 setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)1409 private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, 1410 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 1411 throws DeviceNotAvailableException { 1412 switch (numConsecutiveErrors) { 1413 case 0: 1414 setRecoveryExpectationWait(mockSleepProvider); 1415 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1416 return 1; 1417 case 1: 1418 setRecoveryExpectationRecovery(mockDevice); 1419 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1420 return 2; 1421 case 2: 1422 setRecoveryExpectationReboot(mockDevice); 1423 return 3; 1424 default: 1425 return 4; 1426 } 1427 } 1428 setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)1429 private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, 1430 DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events) 1431 throws DeviceNotAvailableException { 1432 int numConsecutiveErrors = 0; 1433 for (RecoveryEvent event : events) { 1434 switch (event) { 1435 case PROGRESS: 1436 numConsecutiveErrors = 0; 1437 break; 1438 case FAIL_CONNECTION_REFUSED: 1439 numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice, 1440 mockSleepProvider, numConsecutiveErrors); 1441 break; 1442 case FAIL_LINK_KILLED: 1443 numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice, 1444 mockSleepProvider, numConsecutiveErrors); 1445 break; 1446 } 1447 } 1448 } 1449 1450 /** 1451 * Test dEQP runner recovery state machine. 1452 */ testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)1453 private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern) 1454 throws Exception { 1455 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 1456 IMocksControl orderedControl = EasyMock.createStrictControl(); 1457 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 1458 EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL"); 1459 DeqpTestRunner.ISleepProvider mockSleepProvider = 1460 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 1461 1462 setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern); 1463 1464 orderedControl.replay(); 1465 1466 recovery.setDevice(mockDevice); 1467 recovery.setSleepProvider(mockSleepProvider); 1468 try { 1469 runRecoveryWithPattern(recovery, pattern); 1470 if (!expectSuccess) { 1471 fail("Expected DeviceNotAvailableException"); 1472 } 1473 } catch (DeviceNotAvailableException ex) { 1474 if (expectSuccess) { 1475 fail("Did not expect DeviceNotAvailableException"); 1476 } 1477 } 1478 1479 orderedControl.verify(); 1480 } 1481 1482 // basic patterns 1483 testRecovery_NoEvents()1484 public void testRecovery_NoEvents() throws Exception { 1485 testRecoveryWithPattern(true); 1486 } 1487 testRecovery_AllOk()1488 public void testRecovery_AllOk() throws Exception { 1489 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS); 1490 } 1491 1492 // conn fail patterns 1493 testRecovery_OneConnectionFailureBegin()1494 public void testRecovery_OneConnectionFailureBegin() throws Exception { 1495 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1496 RecoveryEvent.PROGRESS); 1497 } 1498 testRecovery_TwoConnectionFailuresBegin()1499 public void testRecovery_TwoConnectionFailuresBegin() throws Exception { 1500 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1501 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1502 } 1503 testRecovery_ThreeConnectionFailuresBegin()1504 public void testRecovery_ThreeConnectionFailuresBegin() throws Exception { 1505 testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1506 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED); 1507 } 1508 testRecovery_OneConnectionFailureMid()1509 public void testRecovery_OneConnectionFailureMid() throws Exception { 1510 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1511 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1512 } 1513 testRecovery_TwoConnectionFailuresMid()1514 public void testRecovery_TwoConnectionFailuresMid() throws Exception { 1515 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1516 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1517 RecoveryEvent.PROGRESS); 1518 } 1519 testRecovery_ThreeConnectionFailuresMid()1520 public void testRecovery_ThreeConnectionFailuresMid() throws Exception { 1521 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, 1522 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1523 RecoveryEvent.FAIL_CONNECTION_REFUSED); 1524 } 1525 1526 // link fail patterns 1527 testRecovery_OneLinkFailureBegin()1528 public void testRecovery_OneLinkFailureBegin() throws Exception { 1529 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1530 RecoveryEvent.PROGRESS); 1531 } 1532 testRecovery_TwoLinkFailuresBegin()1533 public void testRecovery_TwoLinkFailuresBegin() throws Exception { 1534 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1535 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1536 } 1537 testRecovery_ThreeLinkFailuresBegin()1538 public void testRecovery_ThreeLinkFailuresBegin() throws Exception { 1539 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1540 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1541 RecoveryEvent.PROGRESS); 1542 } 1543 testRecovery_FourLinkFailuresBegin()1544 public void testRecovery_FourLinkFailuresBegin() throws Exception { 1545 testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED, 1546 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1547 RecoveryEvent.FAIL_LINK_KILLED); 1548 } 1549 testRecovery_OneLinkFailureMid()1550 public void testRecovery_OneLinkFailureMid() throws Exception { 1551 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1552 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1553 } 1554 testRecovery_TwoLinkFailuresMid()1555 public void testRecovery_TwoLinkFailuresMid() throws Exception { 1556 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1557 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1558 RecoveryEvent.PROGRESS); 1559 } 1560 testRecovery_ThreeLinkFailuresMid()1561 public void testRecovery_ThreeLinkFailuresMid() throws Exception { 1562 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1563 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1564 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1565 } 1566 testRecovery_FourLinkFailuresMid()1567 public void testRecovery_FourLinkFailuresMid() throws Exception { 1568 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1569 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1570 RecoveryEvent.FAIL_LINK_KILLED); 1571 } 1572 1573 // mixed patterns 1574 testRecovery_MixedFailuresProgressBetween()1575 public void testRecovery_MixedFailuresProgressBetween() throws Exception { 1576 testRecoveryWithPattern(true, 1577 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1578 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1579 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1580 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1581 RecoveryEvent.PROGRESS); 1582 } 1583 testRecovery_MixedFailuresNoProgressBetween()1584 public void testRecovery_MixedFailuresNoProgressBetween() throws Exception { 1585 testRecoveryWithPattern(true, 1586 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1587 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED, 1588 RecoveryEvent.PROGRESS); 1589 } 1590 1591 /** 1592 * Test recovery if process cannot be killed 1593 */ testRecovery_unkillableProcess()1594 public void testRecovery_unkillableProcess () throws Exception { 1595 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 1596 IMocksControl orderedControl = EasyMock.createStrictControl(); 1597 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 1598 DeqpTestRunner.ISleepProvider mockSleepProvider = 1599 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 1600 1601 // recovery attempts to kill the process after a timeout 1602 mockSleepProvider.sleep(EasyMock.gt(0)); 1603 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1604 andReturn("root 1234 com.drawelement.deqp").once(); 1605 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1606 andReturn("").once(); 1607 1608 // Recovery checks if kill failed 1609 mockSleepProvider.sleep(EasyMock.gt(0)); 1610 EasyMock.expectLastCall().once(); 1611 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1612 andReturn("root 1234 com.drawelement.deqp").once(); 1613 1614 // Recovery resets the connection 1615 mockDevice.recoverDevice(); 1616 EasyMock.expectLastCall().once(); 1617 1618 // and attempts to kill the process again 1619 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1620 andReturn("root 1234 com.drawelement.deqp").once(); 1621 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1622 andReturn("").once(); 1623 1624 // Recovery checks if kill failed 1625 mockSleepProvider.sleep(EasyMock.gt(0)); 1626 EasyMock.expectLastCall().once(); 1627 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1628 andReturn("root 1234 com.drawelement.deqp").once(); 1629 1630 // recovery reboots the device 1631 mockDevice.reboot(); 1632 EasyMock.expectLastCall().once(); 1633 1634 orderedControl.replay(); 1635 recovery.setDevice(mockDevice); 1636 recovery.setSleepProvider(mockSleepProvider); 1637 recovery.recoverComLinkKilled(); 1638 orderedControl.verify(); 1639 } 1640 1641 /** 1642 * Test external interruption before batch run. 1643 */ testInterrupt_killBeforeBatch()1644 public void testInterrupt_killBeforeBatch() throws Exception { 1645 final TestDescription testId = new TestDescription("dEQP-GLES3.interrupt", "test"); 1646 1647 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1648 tests.add(testId); 1649 1650 ITestInvocationListener mockListener 1651 = EasyMock.createStrictMock(ITestInvocationListener.class); 1652 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1653 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1654 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 1655 1656 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1657 1658 deqpTest.setDevice(mockDevice); 1659 deqpTest.setRunUtil(mockRunUtil); 1660 1661 int version = 3 << 16; 1662 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1663 .andReturn(Integer.toString(version)).atLeastOnce(); 1664 1665 expectRenderConfigQuery(mockDevice, 1666 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 1667 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1668 + "--deqp-gl-minor-version=0"); 1669 1670 mockRunUtil.sleep(0); 1671 EasyMock.expectLastCall().andThrow(new RunInterruptedException()); 1672 1673 mockListener.testRunStarted(getTestId(deqpTest), 1); 1674 EasyMock.expectLastCall().once(); 1675 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 1676 andReturn("").once(); 1677 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 1678 andReturn("").once(); 1679 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject()); 1680 EasyMock.expectLastCall().once(); 1681 1682 EasyMock.replay(mockDevice, mockIDevice); 1683 EasyMock.replay(mockListener); 1684 EasyMock.replay(mockRunUtil); 1685 try { 1686 deqpTest.run(mockListener); 1687 fail("expected RunInterruptedException"); 1688 } catch (RunInterruptedException ex) { 1689 // expected 1690 } 1691 EasyMock.verify(mockRunUtil); 1692 EasyMock.verify(mockListener); 1693 EasyMock.verify(mockDevice, mockIDevice); 1694 } 1695 runShardedTest(TestDescription[] testIds, ArrayList<ArrayList<TestDescription>> testsForShard)1696 private void runShardedTest(TestDescription[] testIds, 1697 ArrayList<ArrayList<TestDescription>> testsForShard) throws Exception { 1698 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1699 for (TestDescription id : testIds) tests.add(id); 1700 1701 DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests, mTestsDir); 1702 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split(); 1703 1704 for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) { 1705 DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex); 1706 shard.setBuildHelper(getMockBuildHelper(mTestsDir)); 1707 1708 ArrayList<TestDescription> shardTests = testsForShard.get(shardIndex); 1709 1710 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1711 ITestInvocationListener mockListener 1712 = EasyMock.createStrictMock(ITestInvocationListener.class); 1713 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1714 int version = 3 << 16; 1715 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1716 .andReturn(Integer.toString(version)).atLeastOnce(); 1717 1718 mockListener.testRunStarted(getTestId(shard), shardTests.size()); 1719 EasyMock.expectLastCall().once(); 1720 1721 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 1722 andReturn("").once(); 1723 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 1724 andReturn("").once(); 1725 1726 expectRenderConfigQuery(mockDevice, 3, 0); 1727 1728 String testOut = buildTestProcessOutput(shardTests); 1729 // NOTE: This assumes that there won't be multiple batches per shard! 1730 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); 1731 1732 for (int i = 0; i < shardTests.size(); i++) { 1733 mockListener.testStarted(EasyMock.eq(shardTests.get(i))); 1734 EasyMock.expectLastCall().once(); 1735 1736 mockListener.testEnded(EasyMock.eq(shardTests.get(i)), 1737 EasyMock.<HashMap<String, Metric>>notNull()); 1738 1739 EasyMock.expectLastCall().once(); 1740 } 1741 1742 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1743 EasyMock.expectLastCall().once(); 1744 1745 EasyMock.replay(mockDevice, mockIDevice); 1746 EasyMock.replay(mockListener); 1747 1748 shard.setDevice(mockDevice); 1749 shard.run(mockListener); 1750 1751 EasyMock.verify(mockListener); 1752 EasyMock.verify(mockDevice, mockIDevice); 1753 } 1754 } 1755 testSharding_smallTrivial()1756 public void testSharding_smallTrivial() throws Exception { 1757 final TestDescription[] testIds = { 1758 new TestDescription("dEQP-GLES3.info", "vendor"), 1759 new TestDescription("dEQP-GLES3.info", "renderer"), 1760 new TestDescription("dEQP-GLES3.info", "version"), 1761 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 1762 new TestDescription("dEQP-GLES3.info", "extensions"), 1763 new TestDescription("dEQP-GLES3.info", "render_target") 1764 }; 1765 ArrayList<ArrayList<TestDescription>> shardedTests = new ArrayList<>(); 1766 ArrayList<TestDescription> shardOne = new ArrayList<>(); 1767 for (int i = 0; i < testIds.length; i++) { 1768 shardOne.add(testIds[i]); 1769 } 1770 shardedTests.add(shardOne); 1771 runShardedTest(testIds, shardedTests); 1772 } 1773 testSharding_twoShards()1774 public void testSharding_twoShards() throws Exception { 1775 final int TEST_COUNT = 1237; 1776 final int SHARD_SIZE = 1000; 1777 1778 ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT); 1779 for (int i = 0; i < TEST_COUNT; i++) { 1780 testIds.add(new TestDescription("dEQP-GLES3.funny.group", String.valueOf(i))); 1781 } 1782 1783 ArrayList<ArrayList<TestDescription>> shardedTests = new ArrayList<>(); 1784 ArrayList<TestDescription> shard = new ArrayList<>(); 1785 for (int i = 0; i < testIds.size(); i++) { 1786 if (i == SHARD_SIZE) { 1787 shardedTests.add(shard); 1788 shard = new ArrayList<>(); 1789 } 1790 shard.add(testIds.get(i)); 1791 } 1792 shardedTests.add(shard); 1793 runShardedTest(testIds.toArray(new TestDescription[testIds.size()]), shardedTests); 1794 } 1795 testSharding_empty()1796 public void testSharding_empty() throws Exception { 1797 DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList<TestDescription>(), mTestsDir); 1798 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split(); 1799 // Returns null when cannot be sharded. 1800 assertNull(shards); 1801 } 1802 1803 /** 1804 * Test external interruption in testFailed(). 1805 */ testInterrupt_killReportTestFailed()1806 public void testInterrupt_killReportTestFailed() throws Exception { 1807 final TestDescription testId = new TestDescription("dEQP-GLES3.interrupt", "test"); 1808 final String testPath = "dEQP-GLES3.interrupt.test"; 1809 final String testTrie = "{dEQP-GLES3{interrupt{test}}}"; 1810 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1811 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1812 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1813 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1814 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1815 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1816 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1817 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1818 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1819 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1820 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1821 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1822 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1823 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1824 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1825 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 1826 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1827 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n" 1828 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n" 1829 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1830 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1831 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1832 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1833 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1834 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1835 + "INSTRUMENTATION_CODE: 0\r\n"; 1836 1837 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1838 tests.add(testId); 1839 1840 ITestInvocationListener mockListener 1841 = EasyMock.createStrictMock(ITestInvocationListener.class); 1842 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1843 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1844 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 1845 1846 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1847 1848 deqpTest.setDevice(mockDevice); 1849 deqpTest.setRunUtil(mockRunUtil); 1850 1851 int version = 3 << 16; 1852 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1853 .andReturn(Integer.toString(version)).atLeastOnce(); 1854 1855 expectRenderConfigQuery(mockDevice, 1856 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 1857 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1858 + "--deqp-gl-minor-version=0"); 1859 1860 mockRunUtil.sleep(0); 1861 EasyMock.expectLastCall().once(); 1862 1863 String commandLine = String.format( 1864 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1865 + "--deqp-screen-rotation=unspecified " 1866 + "--deqp-surface-type=window " 1867 + "--deqp-log-images=disable " 1868 + "--deqp-watchdog=enable", 1869 APP_DIR + CASE_LIST_FILE_NAME); 1870 1871 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 1872 output); 1873 1874 mockListener.testRunStarted(getTestId(deqpTest), 1); 1875 EasyMock.expectLastCall().once(); 1876 1877 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 1878 andReturn("").once(); 1879 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 1880 andReturn("").once(); 1881 1882 mockListener.testStarted(EasyMock.eq(testId)); 1883 EasyMock.expectLastCall().once(); 1884 1885 mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull()); 1886 EasyMock.expectLastCall().andThrow(new RunInterruptedException()); 1887 1888 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject()); 1889 EasyMock.expectLastCall().once(); 1890 EasyMock.replay(mockDevice, mockIDevice); 1891 EasyMock.replay(mockListener); 1892 EasyMock.replay(mockRunUtil); 1893 try { 1894 deqpTest.run(mockListener); 1895 fail("expected RunInterruptedException"); 1896 } catch (RunInterruptedException ex) { 1897 // expected 1898 } 1899 EasyMock.verify(mockRunUtil); 1900 EasyMock.verify(mockListener); 1901 EasyMock.verify(mockDevice, mockIDevice); 1902 } 1903 testRuntimeHint_optionSet()1904 public void testRuntimeHint_optionSet() throws Exception { 1905 /* MultiLineReceiver expects "\r\n" line ending. */ 1906 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1907 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1908 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1909 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1910 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1911 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1912 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1913 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1914 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1915 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1916 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1917 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1918 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1919 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1920 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1921 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 1922 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1923 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1924 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1925 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1926 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1927 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1928 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1929 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1930 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 1931 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1932 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1933 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1934 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1935 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1936 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1937 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1938 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1939 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 1940 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1941 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1942 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1943 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1944 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1945 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1946 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1947 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1948 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 1949 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1950 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1951 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1952 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1953 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1954 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1955 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1956 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1957 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 1958 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1959 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1960 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1961 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1962 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1963 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1964 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1965 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1966 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 1967 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1968 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1969 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1970 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1971 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1972 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1973 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1974 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1975 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1976 + "INSTRUMENTATION_CODE: 0\r\n"; 1977 1978 final TestDescription[] testIds = { 1979 new TestDescription("dEQP-GLES3.info", "vendor"), 1980 new TestDescription("dEQP-GLES3.info", "renderer"), 1981 new TestDescription("dEQP-GLES3.info", "version"), 1982 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 1983 new TestDescription("dEQP-GLES3.info", "extensions"), 1984 new TestDescription("dEQP-GLES3.info", "render_target") 1985 }; 1986 1987 final String[] testPaths = { 1988 "dEQP-GLES3.info.vendor", 1989 "dEQP-GLES3.info.renderer", 1990 "dEQP-GLES3.info.version", 1991 "dEQP-GLES3.info.shading_language_version", 1992 "dEQP-GLES3.info.extensions", 1993 "dEQP-GLES3.info.render_target" 1994 }; 1995 1996 final String testTrie 1997 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 1998 1999 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2000 ITestInvocationListener mockListener 2001 = EasyMock.createStrictMock(ITestInvocationListener.class); 2002 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 2003 2004 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2005 2006 for (TestDescription id : testIds) { 2007 tests.add(id); 2008 } 2009 2010 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2011 OptionSetter setter = new OptionSetter(deqpTest); 2012 final long runtimeMs = 123456; 2013 setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs)); 2014 assertEquals("Wrong expected runtime - option not passed cleanly", runtimeMs, deqpTest.getRuntimeHint()); 2015 2016 // Try running the tests as well. The unit tests do not set the hint be default, 2017 // so that case is covered. 2018 2019 int version = 3 << 16; 2020 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 2021 .andReturn(Integer.toString(version)).atLeastOnce(); 2022 2023 expectRenderConfigQuery(mockDevice, 3, 0); 2024 2025 String commandLine = String.format( 2026 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2027 + "--deqp-screen-rotation=unspecified " 2028 + "--deqp-surface-type=window " 2029 + "--deqp-log-images=disable " 2030 + "--deqp-watchdog=enable", 2031 APP_DIR + CASE_LIST_FILE_NAME); 2032 2033 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 2034 2035 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 2036 EasyMock.expectLastCall().once(); 2037 2038 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 2039 andReturn("").once(); 2040 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 2041 andReturn("").once(); 2042 2043 for (int i = 0; i < testPaths.length; i++) { 2044 mockListener.testStarted(EasyMock.eq(testIds[i])); 2045 EasyMock.expectLastCall().once(); 2046 2047 mockListener.testEnded(EasyMock.eq(testIds[i]), 2048 EasyMock.<HashMap<String, Metric>>notNull()); 2049 2050 EasyMock.expectLastCall().once(); 2051 } 2052 2053 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 2054 EasyMock.expectLastCall().once(); 2055 2056 EasyMock.replay(mockDevice, mockIDevice); 2057 EasyMock.replay(mockListener); 2058 2059 deqpTest.setDevice(mockDevice); 2060 deqpTest.run(mockListener); 2061 2062 EasyMock.verify(mockListener); 2063 EasyMock.verify(mockDevice, mockIDevice); 2064 } 2065 testRuntimeHint_optionSetSharded()2066 public void testRuntimeHint_optionSetSharded() throws Exception { 2067 final int TEST_COUNT = 1237; 2068 final int SHARD_SIZE = 1000; 2069 2070 ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT); 2071 for (int i = 0; i < TEST_COUNT; i++) { 2072 testIds.add(new TestDescription("dEQP-GLES3.funny.group", String.valueOf(i))); 2073 } 2074 2075 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir); 2076 OptionSetter setter = new OptionSetter(deqpTest); 2077 final long fullRuntimeMs = testIds.size()*100; 2078 setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs)); 2079 2080 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)deqpTest.split(); 2081 assertEquals("First shard's time not proportional to test count", 2082 (fullRuntimeMs*SHARD_SIZE)/TEST_COUNT, 2083 ((IRuntimeHintProvider)shards.get(0)).getRuntimeHint()); 2084 assertEquals("Second shard's time not proportional to test count", 2085 (fullRuntimeMs*(TEST_COUNT-SHARD_SIZE))/TEST_COUNT, 2086 ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint()); 2087 } 2088 testRuntimeHint_optionNotSet()2089 public void testRuntimeHint_optionNotSet() throws Exception { 2090 final TestDescription[] testIds = { 2091 new TestDescription("dEQP-GLES3.info", "vendor"), 2092 new TestDescription("dEQP-GLES3.info", "renderer"), 2093 new TestDescription("dEQP-GLES3.info", "version"), 2094 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 2095 new TestDescription("dEQP-GLES3.info", "extensions"), 2096 new TestDescription("dEQP-GLES3.info", "render_target") 2097 }; 2098 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2099 2100 for (TestDescription id : testIds) { 2101 tests.add(id); 2102 } 2103 2104 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2105 2106 long runtime = deqpTest.getRuntimeHint(); 2107 assertTrue("Runtime for tests must be positive", runtime > 0); 2108 assertTrue("Runtime for tests must be reasonable", runtime < (1000 * 10)); // Must be done in 10s 2109 } 2110 2111 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String output)2112 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 2113 final String output) throws Exception { 2114 String cmd = String.format( 2115 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2116 + "--deqp-screen-rotation=unspecified " 2117 + "--deqp-surface-type=window " 2118 + "--deqp-log-images=disable " 2119 + "--deqp-watchdog=enable", 2120 APP_DIR + CASE_LIST_FILE_NAME); 2121 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, cmd, output); 2122 } 2123 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String testTrie, final String cmd, final String output)2124 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 2125 final String testTrie, final String cmd, final String output) throws Exception { 2126 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + CASE_LIST_FILE_NAME))) 2127 .andReturn("").once(); 2128 2129 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + LOG_FILE_NAME))) 2130 .andReturn("").once(); 2131 2132 if (testTrie == null) { 2133 mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(APP_DIR + CASE_LIST_FILE_NAME)); 2134 } 2135 else { 2136 mockDevice.pushString(testTrie + "\n", APP_DIR + CASE_LIST_FILE_NAME); 2137 } 2138 EasyMock.expectLastCall().andReturn(true).once(); 2139 2140 String command = String.format( 2141 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" " 2142 + "-e deqpLogData \"%s\" %s", 2143 AbiUtils.createAbiFlag(ABI.getName()), APP_DIR + LOG_FILE_NAME, cmd, false, 2144 INSTRUMENTATION_NAME); 2145 2146 EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice); 2147 mockIDevice.executeShellCommand(EasyMock.eq(command), 2148 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(), 2149 EasyMock.isA(TimeUnit.class)); 2150 2151 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 2152 @Override 2153 public Object answer() { 2154 IShellOutputReceiver receiver 2155 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 2156 2157 receiver.addOutput(output.getBytes(), 0, output.length()); 2158 receiver.flush(); 2159 2160 return null; 2161 } 2162 }); 2163 } 2164 writeStringsToFile(File target, Set<String> strings)2165 static private void writeStringsToFile(File target, Set<String> strings) throws IOException { 2166 try (PrintWriter out = new PrintWriter(new FileWriter(target))) { 2167 out.print(String.join(System.lineSeparator(), strings)); 2168 out.println(); 2169 } 2170 } 2171 addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option)2172 private void addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option) 2173 throws IOException, ConfigurationException { 2174 String filterFile = option + ".txt"; 2175 writeStringsToFile(new File(mTestsDir, filterFile), filters); 2176 OptionSetter setter = new OptionSetter(test); 2177 setter.setOptionValue(option, filterFile); 2178 } 2179 testIncludeFilterFile()2180 public void testIncludeFilterFile() throws Exception { 2181 final TestDescription[] testIds = { 2182 new TestDescription("dEQP-GLES3.missing", "no"), 2183 new TestDescription("dEQP-GLES3.missing", "nope"), 2184 new TestDescription("dEQP-GLES3.missing", "donotwant"), 2185 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2186 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2187 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2188 }; 2189 2190 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2191 for (TestDescription id : testIds) { 2192 allTests.add(id); 2193 } 2194 2195 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2196 activeTests.add(testIds[3]); 2197 activeTests.add(testIds[4]); 2198 activeTests.add(testIds[5]); 2199 2200 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2201 2202 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2203 Set<String> includes = new HashSet<>(); 2204 includes.add("dEQP-GLES3.pick_me#*"); 2205 addFilterFileForOption(deqpTest, includes, "include-filter-file"); 2206 testFiltering(deqpTest, expectedTrie, activeTests); 2207 } 2208 testMissingIncludeFilterFile()2209 public void testMissingIncludeFilterFile() throws Exception { 2210 final TestDescription[] testIds = { 2211 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2212 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2213 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2214 }; 2215 2216 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2217 for (TestDescription id : testIds) { 2218 allTests.add(id); 2219 } 2220 2221 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2222 2223 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2224 OptionSetter setter = new OptionSetter(deqpTest); 2225 setter.setOptionValue("include-filter-file", "not-a-file.txt"); 2226 try { 2227 testFiltering(deqpTest, expectedTrie, allTests); 2228 fail("Test execution should have aborted with exception."); 2229 } catch (RuntimeException e) { 2230 } 2231 } 2232 testExcludeFilterFile()2233 public void testExcludeFilterFile() throws Exception { 2234 final TestDescription[] testIds = { 2235 new TestDescription("dEQP-GLES3.missing", "no"), 2236 new TestDescription("dEQP-GLES3.missing", "nope"), 2237 new TestDescription("dEQP-GLES3.missing", "donotwant"), 2238 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2239 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2240 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2241 }; 2242 2243 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2244 for (TestDescription id : testIds) { 2245 allTests.add(id); 2246 } 2247 2248 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2249 activeTests.add(testIds[3]); 2250 activeTests.add(testIds[4]); 2251 activeTests.add(testIds[5]); 2252 2253 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2254 2255 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2256 Set<String> excludes = new HashSet<>(); 2257 excludes.add("dEQP-GLES3.missing#*"); 2258 addFilterFileForOption(deqpTest, excludes, "exclude-filter-file"); 2259 testFiltering(deqpTest, expectedTrie, activeTests); 2260 } 2261 testFilterComboWithFiles()2262 public void testFilterComboWithFiles() throws Exception { 2263 final TestDescription[] testIds = { 2264 new TestDescription("dEQP-GLES3.group1", "footah"), 2265 new TestDescription("dEQP-GLES3.group1", "foo"), 2266 new TestDescription("dEQP-GLES3.group1", "nope"), 2267 new TestDescription("dEQP-GLES3.group1", "nonotwant"), 2268 new TestDescription("dEQP-GLES3.group2", "foo"), 2269 new TestDescription("dEQP-GLES3.group2", "yes"), 2270 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"), 2271 }; 2272 2273 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2274 for (TestDescription id : testIds) { 2275 allTests.add(id); 2276 } 2277 2278 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2279 activeTests.add(testIds[0]); 2280 activeTests.add(testIds[5]); 2281 2282 String expectedTrie = "{dEQP-GLES3{group1{footah}group2{yes}}}"; 2283 2284 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2285 2286 Set<String> includes = new HashSet<>(); 2287 includes.add("dEQP-GLES3.group2#*"); 2288 deqpTest.addAllIncludeFilters(includes); 2289 2290 Set<String> fileIncludes = new HashSet<>(); 2291 fileIncludes.add("dEQP-GLES3.group1#no*"); 2292 fileIncludes.add("dEQP-GLES3.group1#foo*"); 2293 addFilterFileForOption(deqpTest, fileIncludes, "include-filter-file"); 2294 2295 Set<String> fileExcludes = new HashSet<>(); 2296 fileExcludes.add("*foo"); 2297 fileExcludes.add("*thoushallnotpass"); 2298 addFilterFileForOption(deqpTest, fileExcludes, "exclude-filter-file"); 2299 2300 deqpTest.addExcludeFilter("dEQP-GLES3.group1#no*"); 2301 2302 testFiltering(deqpTest, expectedTrie, activeTests); 2303 } 2304 testDotToHashConversionInFilters()2305 public void testDotToHashConversionInFilters() throws Exception { 2306 final TestDescription[] testIds = { 2307 new TestDescription("dEQP-GLES3.missing", "no"), 2308 new TestDescription("dEQP-GLES3.pick_me", "donotwant"), 2309 new TestDescription("dEQP-GLES3.pick_me", "yes") 2310 }; 2311 2312 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2313 for (TestDescription id : testIds) { 2314 allTests.add(id); 2315 } 2316 2317 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2318 activeTests.add(testIds[2]); 2319 2320 String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}"; 2321 2322 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2323 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me.yes"); 2324 testFiltering(deqpTest, expectedTrie, activeTests); 2325 } 2326 testCoverageDisabled_noCoverageListener()2327 public void testCoverageDisabled_noCoverageListener() throws Exception { 2328 final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version"); 2329 2330 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2331 ITestInvocationListener mockListener = 2332 EasyMock.createStrictMock(ITestInvocationListener.class); 2333 2334 Collection<TestDescription> tests = new ArrayList<>(); 2335 tests.add(testId); 2336 2337 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2338 deqpTest.setNativeCoverage(false); 2339 2340 ITestInvocationListener listener = 2341 deqpTest.addNativeCoverageListenerIfEnabled(mockDevice, mockListener); 2342 assertSame(listener, mockListener); 2343 } 2344 testCoverageEnabled_newCoverageListener()2345 public void testCoverageEnabled_newCoverageListener() throws Exception { 2346 final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version"); 2347 2348 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2349 ITestInvocationListener mockListener = 2350 EasyMock.createStrictMock(ITestInvocationListener.class); 2351 2352 Collection<TestDescription> tests = new ArrayList<>(); 2353 tests.add(testId); 2354 2355 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2356 deqpTest.setNativeCoverage(true); 2357 2358 ITestInvocationListener listener = 2359 deqpTest.addNativeCoverageListenerIfEnabled(mockDevice, mockListener); 2360 assertTrue(listener instanceof NativeCodeCoverageListener); 2361 } 2362 } 2363