1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.drawelements.deqp.runner; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.ddmlib.IDevice; 20 import com.android.ddmlib.IShellOutputReceiver; 21 import com.android.ddmlib.testrunner.TestIdentifier; 22 import com.android.tradefed.build.IFolderBuildInfo; 23 import com.android.tradefed.config.ConfigurationException; 24 import com.android.tradefed.config.OptionSetter; 25 import com.android.tradefed.device.DeviceNotAvailableException; 26 import com.android.tradefed.device.ITestDevice; 27 import com.android.tradefed.result.ITestInvocationListener; 28 import com.android.tradefed.testtype.Abi; 29 import com.android.tradefed.testtype.IAbi; 30 import com.android.tradefed.testtype.IRemoteTest; 31 import com.android.tradefed.testtype.IRuntimeHintProvider; 32 import com.android.tradefed.util.AbiUtils; 33 import com.android.tradefed.util.FileUtil; 34 import com.android.tradefed.util.IRunUtil; 35 import com.android.tradefed.util.RunInterruptedException; 36 37 import junit.framework.TestCase; 38 39 import org.easymock.EasyMock; 40 import org.easymock.IAnswer; 41 import org.easymock.IMocksControl; 42 43 import java.io.File; 44 import java.io.FileNotFoundException; 45 import java.io.FileWriter; 46 import java.io.IOException; 47 import java.io.PrintWriter; 48 import java.io.StringReader; 49 import java.io.StringWriter; 50 import java.util.ArrayList; 51 import java.util.Collection; 52 import java.util.HashMap; 53 import java.util.HashSet; 54 import java.util.List; 55 import java.util.Map; 56 import java.util.Set; 57 import java.util.concurrent.TimeUnit; 58 59 /** 60 * Unit tests for {@link DeqpTestRunner}. 61 */ 62 public class DeqpTestRunnerTest extends TestCase { 63 private static final String NAME = "dEQP-GLES3"; 64 private static final IAbi ABI = new Abi("armeabi-v7a", "32"); 65 private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt"; 66 private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa"; 67 private static final String INSTRUMENTATION_NAME = 68 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation"; 69 private static final String QUERY_INSTRUMENTATION_NAME = 70 "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation"; 71 private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk"; 72 private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp"; 73 private static final String ONLY_LANDSCAPE_FEATURES = 74 "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE; 75 private static final String ALL_FEATURES = 76 ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT; 77 private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS; 78 79 static { 80 DEFAULT_INSTANCE_ARGS = new ArrayList<>(1); DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>())81 DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>()); 82 DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8"); 83 DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified"); 84 DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window"); 85 } 86 87 private File mTestsDir = null; 88 89 public static class BuildHelperMock extends CompatibilityBuildHelper { 90 private File mTestsDir = null; BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir)91 public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) { 92 super(buildInfo); 93 mTestsDir = testsDir; 94 } 95 @Override getTestsDir()96 public File getTestsDir() throws FileNotFoundException { 97 return mTestsDir; 98 } 99 } 100 101 102 /** 103 * {@inheritDoc} 104 */ 105 @Override setUp()106 protected void setUp() throws Exception { 107 super.setUp(); 108 mTestsDir = FileUtil.createTempDir("deqp-test-cases"); 109 } 110 111 /** 112 * {@inheritDoc} 113 */ 114 @Override tearDown()115 protected void tearDown() throws Exception { 116 FileUtil.recursiveDelete(mTestsDir); 117 super.tearDown(); 118 } 119 buildGlesTestRunner(int majorVersion, int minorVersion, Collection<TestIdentifier> tests, File testsDir)120 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, 121 int minorVersion, 122 Collection<TestIdentifier> tests, 123 File testsDir) throws ConfigurationException, FileNotFoundException { 124 StringWriter testlist = new StringWriter(); 125 for (TestIdentifier test : tests) { 126 testlist.write(test.getClassName() + "." + test.getTestName() + "\n"); 127 } 128 return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString(), testsDir); 129 } 130 getMockBuildHelper(File testsDir)131 private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) { 132 IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class); 133 EasyMock.replay(mockIFolderBuildInfo); 134 return new BuildHelperMock(mockIFolderBuildInfo, testsDir); 135 } 136 buildGlesTestRunner(int majorVersion, int minorVersion, String testlist, File testsDir)137 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, 138 int minorVersion, 139 String testlist, 140 File testsDir) throws ConfigurationException, FileNotFoundException { 141 DeqpTestRunner runner = new DeqpTestRunner(); 142 OptionSetter setter = new OptionSetter(runner); 143 144 String deqpPackage = "dEQP-GLES" + Integer.toString(majorVersion) 145 + (minorVersion > 0 ? Integer.toString(minorVersion) : ""); 146 147 setter.setOptionValue("deqp-package", deqpPackage); 148 setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8"); 149 setter.setOptionValue("deqp-caselist-file", "dummyfile.txt"); 150 setter.setOptionValue("deqp-screen-rotation", "unspecified"); 151 setter.setOptionValue("deqp-surface-type", "window"); 152 153 runner.setCaselistReader(new StringReader(testlist)); 154 runner.setAbi(ABI); 155 runner.setBuildHelper(getMockBuildHelper(testsDir)); 156 157 return runner; 158 } 159 getTestId(DeqpTestRunner runner)160 private static String getTestId(DeqpTestRunner runner) { 161 return AbiUtils.createId(ABI.getName(), runner.getPackageName()); 162 } 163 164 /** 165 * Test version of OpenGL ES. 166 */ testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion)167 private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception { 168 final TestIdentifier testId = new TestIdentifier("dEQP-GLES" 169 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 170 + ".info", "version"); 171 172 final String testPath = "dEQP-GLES" 173 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 174 +".info.version"; 175 176 final String testTrie = "{dEQP-GLES" 177 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 178 + "{info{version}}}"; 179 180 final String resultCode = "Pass"; 181 182 /* MultiLineReceiver expects "\r\n" line ending. */ 183 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 184 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 185 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 186 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 187 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 188 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 189 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 190 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 191 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 192 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 193 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 194 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 195 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 196 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 197 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 198 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 199 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 200 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 201 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 202 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 203 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 204 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 205 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 206 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 207 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 208 + "INSTRUMENTATION_CODE: 0\r\n"; 209 210 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 211 ITestInvocationListener mockListener 212 = EasyMock.createStrictMock(ITestInvocationListener.class); 213 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 214 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 215 tests.add(testId); 216 217 DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests, mTestsDir); 218 219 int version = (majorVersion << 16) | minorVersion; 220 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 221 .andReturn(Integer.toString(version)).atLeastOnce(); 222 223 if (majorVersion > requiredMajorVersion 224 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) { 225 226 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 227 .andReturn("").once(); 228 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 229 EasyMock.eq(true), 230 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))) 231 .andReturn(null).once(); 232 233 expectRenderConfigQuery(mockDevice, requiredMajorVersion, 234 requiredMinorVersion); 235 236 String commandLine = String.format( 237 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 238 + "--deqp-screen-rotation=unspecified " 239 + "--deqp-surface-type=window " 240 + "--deqp-log-images=disable " 241 + "--deqp-watchdog=enable", 242 CASE_LIST_FILE_NAME); 243 244 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 245 output); 246 247 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 248 .andReturn("").once(); 249 } 250 251 mockListener.testRunStarted(getTestId(deqpTest), 1); 252 EasyMock.expectLastCall().once(); 253 254 mockListener.testStarted(EasyMock.eq(testId)); 255 EasyMock.expectLastCall().once(); 256 257 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); 258 EasyMock.expectLastCall().once(); 259 260 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 261 EasyMock.expectLastCall().once(); 262 263 EasyMock.replay(mockDevice, mockIDevice); 264 EasyMock.replay(mockListener); 265 266 deqpTest.setDevice(mockDevice); 267 deqpTest.run(mockListener); 268 269 EasyMock.verify(mockListener); 270 EasyMock.verify(mockDevice, mockIDevice); 271 } 272 expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, int minorVersion)273 private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, 274 int minorVersion) throws Exception { 275 expectRenderConfigQuery(mockDevice, 276 String.format("--deqp-gl-config-name=rgba8888d24s8 " 277 + "--deqp-screen-rotation=unspecified " 278 + "--deqp-surface-type=window " 279 + "--deqp-gl-major-version=%d " 280 + "--deqp-gl-minor-version=%d", majorVersion, minorVersion)); 281 } 282 expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)283 private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine) 284 throws Exception { 285 expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes"); 286 } 287 expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, String output)288 private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, 289 String output) throws Exception { 290 final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n" 291 + "INSTRUMENTATION_CODE: 0\r\n"; 292 final String command = String.format( 293 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine " 294 + "\"%s\" %s", 295 AbiUtils.createAbiFlag(ABI.getName()), commandLine, 296 QUERY_INSTRUMENTATION_NAME); 297 298 mockDevice.executeShellCommand(EasyMock.eq(command), 299 EasyMock.<IShellOutputReceiver>notNull()); 300 301 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 302 @Override 303 public Object answer() { 304 IShellOutputReceiver receiver 305 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 306 307 receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length()); 308 receiver.flush(); 309 310 return null; 311 } 312 }); 313 } 314 315 /** 316 * Test that result code produces correctly pass or fail. 317 */ testResultCode(final String resultCode, boolean pass)318 private void testResultCode(final String resultCode, boolean pass) throws Exception { 319 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.info", "version"); 320 final String testPath = "dEQP-GLES3.info.version"; 321 final String testTrie = "{dEQP-GLES3{info{version}}}"; 322 323 /* MultiLineReceiver expects "\r\n" line ending. */ 324 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 325 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 326 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 327 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 328 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 329 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 330 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 331 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 332 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 333 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 334 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 335 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 336 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 337 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 338 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 339 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 340 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 341 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 342 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 343 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 344 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 345 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 346 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 347 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 348 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 349 + "INSTRUMENTATION_CODE: 0\r\n"; 350 351 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 352 ITestInvocationListener mockListener 353 = EasyMock.createStrictMock(ITestInvocationListener.class); 354 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 355 356 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 357 tests.add(testId); 358 359 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 360 361 int version = 3 << 16; 362 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 363 .andReturn(Integer.toString(version)).atLeastOnce(); 364 365 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 366 .once(); 367 368 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 369 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))) 370 .andReturn(null).once(); 371 372 expectRenderConfigQuery(mockDevice, 3, 0); 373 374 String commandLine = String.format( 375 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 376 + "--deqp-screen-rotation=unspecified " 377 + "--deqp-surface-type=window " 378 + "--deqp-log-images=disable " 379 + "--deqp-watchdog=enable", 380 CASE_LIST_FILE_NAME); 381 382 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 383 384 mockListener.testRunStarted(getTestId(deqpTest), 1); 385 EasyMock.expectLastCall().once(); 386 387 mockListener.testStarted(EasyMock.eq(testId)); 388 EasyMock.expectLastCall().once(); 389 390 if (!pass) { 391 mockListener.testFailed(testId, 392 "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n" 393 + resultCode + ": Detail" + resultCode); 394 395 EasyMock.expectLastCall().once(); 396 } 397 398 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); 399 EasyMock.expectLastCall().once(); 400 401 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 402 EasyMock.expectLastCall().once(); 403 404 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 405 .once(); 406 407 EasyMock.replay(mockDevice, mockIDevice); 408 EasyMock.replay(mockListener); 409 410 deqpTest.setDevice(mockDevice); 411 deqpTest.run(mockListener); 412 413 EasyMock.verify(mockListener); 414 EasyMock.verify(mockDevice, mockIDevice); 415 } 416 417 /** 418 * Test running multiple test cases. 419 */ testRun_multipleTests()420 public void testRun_multipleTests() throws Exception { 421 /* MultiLineReceiver expects "\r\n" line ending. */ 422 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 423 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 424 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 425 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 426 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 427 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 428 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 429 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 430 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 431 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 432 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 433 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 434 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 435 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 436 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 437 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 438 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 439 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 440 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 441 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 442 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 443 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 444 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 445 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 446 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 447 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 448 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 449 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 450 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 451 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 452 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 453 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 454 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 455 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 456 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 457 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 458 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 459 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 460 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 461 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 462 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 463 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 464 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 465 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 466 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 467 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 468 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 469 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 470 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 471 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 472 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 473 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 474 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 475 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 476 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 477 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 478 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 479 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 480 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 481 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 482 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 483 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 484 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 485 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 486 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 487 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 488 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 489 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 490 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 491 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 492 + "INSTRUMENTATION_CODE: 0\r\n"; 493 494 final TestIdentifier[] testIds = { 495 new TestIdentifier("dEQP-GLES3.info", "vendor"), 496 new TestIdentifier("dEQP-GLES3.info", "renderer"), 497 new TestIdentifier("dEQP-GLES3.info", "version"), 498 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"), 499 new TestIdentifier("dEQP-GLES3.info", "extensions"), 500 new TestIdentifier("dEQP-GLES3.info", "render_target") 501 }; 502 503 final String[] testPaths = { 504 "dEQP-GLES3.info.vendor", 505 "dEQP-GLES3.info.renderer", 506 "dEQP-GLES3.info.version", 507 "dEQP-GLES3.info.shading_language_version", 508 "dEQP-GLES3.info.extensions", 509 "dEQP-GLES3.info.render_target" 510 }; 511 512 final String testTrie 513 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 514 515 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 516 ITestInvocationListener mockListener 517 = EasyMock.createStrictMock(ITestInvocationListener.class); 518 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 519 520 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 521 522 for (TestIdentifier id : testIds) { 523 tests.add(id); 524 } 525 526 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 527 528 int version = 3 << 16; 529 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 530 .andReturn(Integer.toString(version)).atLeastOnce(); 531 532 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 533 .once(); 534 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 535 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))) 536 .andReturn(null).once(); 537 538 expectRenderConfigQuery(mockDevice, 3, 0); 539 540 String commandLine = String.format( 541 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 542 + "--deqp-screen-rotation=unspecified " 543 + "--deqp-surface-type=window " 544 + "--deqp-log-images=disable " 545 + "--deqp-watchdog=enable", 546 CASE_LIST_FILE_NAME); 547 548 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 549 550 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 551 EasyMock.expectLastCall().once(); 552 553 for (int i = 0; i < testPaths.length; i++) { 554 mockListener.testStarted(EasyMock.eq(testIds[i])); 555 EasyMock.expectLastCall().once(); 556 557 mockListener.testEnded(EasyMock.eq(testIds[i]), 558 EasyMock.<Map<String, String>>notNull()); 559 560 EasyMock.expectLastCall().once(); 561 } 562 563 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 564 EasyMock.expectLastCall().once(); 565 566 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 567 .once(); 568 569 EasyMock.replay(mockDevice, mockIDevice); 570 EasyMock.replay(mockListener); 571 572 deqpTest.setDevice(mockDevice); 573 deqpTest.run(mockListener); 574 575 EasyMock.verify(mockListener); 576 EasyMock.verify(mockDevice, mockIDevice); 577 } 578 buildTestProcessOutput(List<TestIdentifier> tests)579 static private String buildTestProcessOutput(List<TestIdentifier> tests) { 580 /* MultiLineReceiver expects "\r\n" line ending. */ 581 final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 582 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 583 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 584 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 585 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 586 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 587 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 588 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 589 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 590 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 591 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 592 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 593 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 594 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; 595 596 final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 597 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 598 + "INSTRUMENTATION_CODE: 0\r\n"; 599 600 StringWriter output = new StringWriter(); 601 output.write(outputHeader); 602 for (TestIdentifier test : tests) { 603 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"); 604 output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath="); 605 output.write(test.getClassName()); 606 output.write("."); 607 output.write(test.getTestName()); 608 output.write("\r\n"); 609 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 610 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"); 611 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"); 612 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"); 613 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 614 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"); 615 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 616 } 617 output.write(outputEnd); 618 return output.toString(); 619 } 620 testFiltering(DeqpTestRunner deqpTest, String expectedTrie, List<TestIdentifier> expectedTests)621 private void testFiltering(DeqpTestRunner deqpTest, 622 String expectedTrie, 623 List<TestIdentifier> expectedTests) throws Exception { 624 int version = 3 << 16; 625 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 626 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 627 .andReturn(Integer.toString(version)).atLeastOnce(); 628 629 boolean thereAreTests = !expectedTests.isEmpty(); 630 if (thereAreTests) 631 { 632 // only expect to install/uninstall packages if there are any tests 633 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 634 .once(); 635 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 636 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))) 637 .andReturn(null).once(); 638 } 639 640 ITestInvocationListener mockListener 641 = EasyMock.createStrictMock(ITestInvocationListener.class); 642 mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size()); 643 EasyMock.expectLastCall().once(); 644 645 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 646 if (thereAreTests) 647 { 648 expectRenderConfigQuery(mockDevice, 3, 0); 649 650 String testOut = buildTestProcessOutput(expectedTests); 651 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); 652 653 for (int i = 0; i < expectedTests.size(); i++) { 654 mockListener.testStarted(EasyMock.eq(expectedTests.get(i))); 655 EasyMock.expectLastCall().once(); 656 657 mockListener.testEnded(EasyMock.eq(expectedTests.get(i)), 658 EasyMock.<Map<String, String>>notNull()); 659 660 EasyMock.expectLastCall().once(); 661 } 662 } 663 664 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 665 EasyMock.expectLastCall().once(); 666 667 if (thereAreTests) 668 { 669 // package will only be installed if there are tests to run 670 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 671 .once(); 672 } 673 674 EasyMock.replay(mockDevice, mockIDevice); 675 EasyMock.replay(mockListener); 676 677 deqpTest.setDevice(mockDevice); 678 deqpTest.run(mockListener); 679 680 EasyMock.verify(mockListener); 681 EasyMock.verify(mockDevice, mockIDevice); 682 } 683 testRun_trivialIncludeFilter()684 public void testRun_trivialIncludeFilter() throws Exception { 685 final TestIdentifier[] testIds = { 686 new TestIdentifier("dEQP-GLES3.missing", "no"), 687 new TestIdentifier("dEQP-GLES3.missing", "nope"), 688 new TestIdentifier("dEQP-GLES3.missing", "donotwant"), 689 new TestIdentifier("dEQP-GLES3.pick_me", "yes"), 690 new TestIdentifier("dEQP-GLES3.pick_me", "ok"), 691 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"), 692 }; 693 694 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>(); 695 for (TestIdentifier id : testIds) { 696 allTests.add(id); 697 } 698 699 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>(); 700 activeTests.add(testIds[3]); 701 activeTests.add(testIds[4]); 702 activeTests.add(testIds[5]); 703 704 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 705 706 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 707 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me#*"); 708 testFiltering(deqpTest, expectedTrie, activeTests); 709 } 710 testRun_trivialExcludeFilter()711 public void testRun_trivialExcludeFilter() throws Exception { 712 final TestIdentifier[] testIds = { 713 new TestIdentifier("dEQP-GLES3.missing", "no"), 714 new TestIdentifier("dEQP-GLES3.missing", "nope"), 715 new TestIdentifier("dEQP-GLES3.missing", "donotwant"), 716 new TestIdentifier("dEQP-GLES3.pick_me", "yes"), 717 new TestIdentifier("dEQP-GLES3.pick_me", "ok"), 718 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"), 719 }; 720 721 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>(); 722 for (TestIdentifier id : testIds) { 723 allTests.add(id); 724 } 725 726 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>(); 727 activeTests.add(testIds[3]); 728 activeTests.add(testIds[4]); 729 activeTests.add(testIds[5]); 730 731 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 732 733 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 734 deqpTest.addExcludeFilter("dEQP-GLES3.missing#*"); 735 testFiltering(deqpTest, expectedTrie, activeTests); 736 } 737 testRun_includeAndExcludeFilter()738 public void testRun_includeAndExcludeFilter() throws Exception { 739 final TestIdentifier[] testIds = { 740 new TestIdentifier("dEQP-GLES3.group1", "foo"), 741 new TestIdentifier("dEQP-GLES3.group1", "nope"), 742 new TestIdentifier("dEQP-GLES3.group1", "donotwant"), 743 new TestIdentifier("dEQP-GLES3.group2", "foo"), 744 new TestIdentifier("dEQP-GLES3.group2", "yes"), 745 new TestIdentifier("dEQP-GLES3.group2", "thoushallnotpass"), 746 }; 747 748 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>(); 749 for (TestIdentifier id : testIds) { 750 allTests.add(id); 751 } 752 753 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>(); 754 activeTests.add(testIds[4]); 755 756 String expectedTrie = "{dEQP-GLES3{group2{yes}}}"; 757 758 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 759 760 Set<String> includes = new HashSet<>(); 761 includes.add("dEQP-GLES3.group2#*"); 762 deqpTest.addAllIncludeFilters(includes); 763 764 Set<String> excludes = new HashSet<>(); 765 excludes.add("*foo"); 766 excludes.add("*thoushallnotpass"); 767 deqpTest.addAllExcludeFilters(excludes); 768 testFiltering(deqpTest, expectedTrie, activeTests); 769 } 770 testRun_includeAll()771 public void testRun_includeAll() throws Exception { 772 final TestIdentifier[] testIds = { 773 new TestIdentifier("dEQP-GLES3.group1", "mememe"), 774 new TestIdentifier("dEQP-GLES3.group1", "yeah"), 775 new TestIdentifier("dEQP-GLES3.group1", "takeitall"), 776 new TestIdentifier("dEQP-GLES3.group2", "jeba"), 777 new TestIdentifier("dEQP-GLES3.group2", "yes"), 778 new TestIdentifier("dEQP-GLES3.group2", "granted"), 779 }; 780 781 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>(); 782 for (TestIdentifier id : testIds) { 783 allTests.add(id); 784 } 785 786 String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}"; 787 788 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 789 deqpTest.addIncludeFilter("*"); 790 testFiltering(deqpTest, expectedTrie, allTests); 791 } 792 testRun_excludeAll()793 public void testRun_excludeAll() throws Exception { 794 final TestIdentifier[] testIds = { 795 new TestIdentifier("dEQP-GLES3.group1", "no"), 796 new TestIdentifier("dEQP-GLES3.group1", "nope"), 797 new TestIdentifier("dEQP-GLES3.group1", "nottoday"), 798 new TestIdentifier("dEQP-GLES3.group2", "banned"), 799 new TestIdentifier("dEQP-GLES3.group2", "notrecognized"), 800 new TestIdentifier("dEQP-GLES3.group2", "-2"), 801 }; 802 803 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>(); 804 for (TestIdentifier id : testIds) { 805 allTests.add(id); 806 } 807 808 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 809 deqpTest.addExcludeFilter("*"); 810 ITestInvocationListener mockListener 811 = EasyMock.createStrictMock(ITestInvocationListener.class); 812 mockListener.testRunStarted(getTestId(deqpTest), 0); 813 EasyMock.expectLastCall().once(); 814 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 815 EasyMock.expectLastCall().once(); 816 817 EasyMock.replay(mockListener); 818 deqpTest.run(mockListener); 819 EasyMock.verify(mockListener); 820 } 821 822 /** 823 * Test running a unexecutable test. 824 */ testRun_unexecutableTests()825 public void testRun_unexecutableTests() throws Exception { 826 final String instrumentationAnswerNoExecs = 827 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 828 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 829 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 830 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 831 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 832 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 833 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 834 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 835 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 836 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 837 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 838 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 839 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 840 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 841 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 842 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 843 + "INSTRUMENTATION_CODE: 0\r\n"; 844 845 final TestIdentifier[] testIds = { 846 new TestIdentifier("dEQP-GLES3.missing", "no"), 847 new TestIdentifier("dEQP-GLES3.missing", "nope"), 848 new TestIdentifier("dEQP-GLES3.missing", "donotwant"), 849 }; 850 851 final String[] testPaths = { 852 "dEQP-GLES3.missing.no", 853 "dEQP-GLES3.missing.nope", 854 "dEQP-GLES3.missing.donotwant", 855 }; 856 857 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 858 ITestInvocationListener mockListener 859 = EasyMock.createStrictMock(ITestInvocationListener.class); 860 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 861 862 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 863 864 for (TestIdentifier id : testIds) { 865 tests.add(id); 866 } 867 868 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 869 870 int version = 3 << 16; 871 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 872 .andReturn(Integer.toString(version)).atLeastOnce(); 873 874 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 875 .once(); 876 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 877 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))) 878 .andReturn(null).once(); 879 880 expectRenderConfigQuery(mockDevice, 3, 0); 881 882 String commandLine = String.format( 883 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 884 + "--deqp-screen-rotation=unspecified " 885 + "--deqp-surface-type=window " 886 + "--deqp-log-images=disable " 887 + "--deqp-watchdog=enable", 888 CASE_LIST_FILE_NAME); 889 890 // first try 891 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 892 "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs); 893 894 // splitting begins 895 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 896 "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs); 897 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 898 "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs); 899 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 900 "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs); 901 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 902 "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs); 903 904 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 905 EasyMock.expectLastCall().once(); 906 907 for (int i = 0; i < testPaths.length; i++) { 908 mockListener.testStarted(EasyMock.eq(testIds[i])); 909 EasyMock.expectLastCall().once(); 910 911 mockListener.testFailed(EasyMock.eq(testIds[i]), 912 EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n" 913 + "Abort: Test cannot be executed")); 914 EasyMock.expectLastCall().once(); 915 916 mockListener.testEnded(EasyMock.eq(testIds[i]), 917 EasyMock.<Map<String, String>>notNull()); 918 EasyMock.expectLastCall().once(); 919 } 920 921 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 922 EasyMock.expectLastCall().once(); 923 924 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 925 .once(); 926 927 EasyMock.replay(mockDevice, mockIDevice); 928 EasyMock.replay(mockListener); 929 930 deqpTest.setDevice(mockDevice); 931 deqpTest.run(mockListener); 932 933 EasyMock.verify(mockListener); 934 EasyMock.verify(mockDevice, mockIDevice); 935 } 936 937 /** 938 * Test that test are left unexecuted if pm list query fails 939 */ testRun_queryPmListFailure()940 public void testRun_queryPmListFailure() 941 throws Exception { 942 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test"); 943 944 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 945 ITestInvocationListener mockListener 946 = EasyMock.createStrictMock(ITestInvocationListener.class); 947 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 948 tests.add(testId); 949 950 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 951 OptionSetter setter = new OptionSetter(deqpTest); 952 // Note: If the rotation is the default unspecified, features are not queried at all 953 setter.setOptionValue("deqp-screen-rotation", "90"); 954 955 deqpTest.setDevice(mockDevice); 956 957 int version = 3 << 16; 958 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 959 .andReturn(Integer.toString(version)).atLeastOnce(); 960 961 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 962 .andReturn("not a valid format"); 963 964 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 965 andReturn("").once(); 966 967 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 968 EasyMock.eq(true), 969 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null) 970 .once(); 971 972 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 973 .andReturn("").once(); 974 975 mockListener.testRunStarted(getTestId(deqpTest), 1); 976 EasyMock.expectLastCall().once(); 977 978 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 979 EasyMock.expectLastCall().once(); 980 981 EasyMock.replay(mockDevice); 982 EasyMock.replay(mockListener); 983 deqpTest.run(mockListener); 984 EasyMock.verify(mockListener); 985 EasyMock.verify(mockDevice); 986 } 987 988 /** 989 * Test that test are left unexecuted if renderablity query fails 990 */ testRun_queryRenderabilityFailure()991 public void testRun_queryRenderabilityFailure() 992 throws Exception { 993 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test"); 994 995 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 996 ITestInvocationListener mockListener 997 = EasyMock.createStrictMock(ITestInvocationListener.class); 998 999 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 1000 tests.add(testId); 1001 1002 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1003 1004 deqpTest.setDevice(mockDevice); 1005 1006 int version = 3 << 16; 1007 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1008 .andReturn(Integer.toString(version)).atLeastOnce(); 1009 1010 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 1011 andReturn("").once(); 1012 1013 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 1014 EasyMock.eq(true), 1015 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null) 1016 .once(); 1017 1018 expectRenderConfigQueryAndReturn(mockDevice, 1019 "--deqp-gl-config-name=rgba8888d24s8 " 1020 + "--deqp-screen-rotation=unspecified " 1021 + "--deqp-surface-type=window " 1022 + "--deqp-gl-major-version=3 " 1023 + "--deqp-gl-minor-version=0", "Maybe?"); 1024 1025 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 1026 .andReturn("").once(); 1027 1028 mockListener.testRunStarted(getTestId(deqpTest), 1); 1029 EasyMock.expectLastCall().once(); 1030 1031 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 1032 EasyMock.expectLastCall().once(); 1033 1034 EasyMock.replay(mockDevice); 1035 EasyMock.replay(mockListener); 1036 deqpTest.run(mockListener); 1037 EasyMock.verify(mockListener); 1038 EasyMock.verify(mockDevice); 1039 } 1040 1041 /** 1042 * Test that orientation is supplied to runner correctly 1043 */ testOrientation(final String rotation, final String featureString)1044 private void testOrientation(final String rotation, final String featureString) 1045 throws Exception { 1046 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test"); 1047 final String testPath = "dEQP-GLES3.orientation.test"; 1048 final String testTrie = "{dEQP-GLES3{orientation{test}}}"; 1049 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1050 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1051 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1052 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1053 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1054 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1055 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1056 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1057 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1058 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1059 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1060 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1061 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1062 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1063 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1064 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 1065 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1066 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1067 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1068 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1069 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1070 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1071 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1072 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1073 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1074 + "INSTRUMENTATION_CODE: 0\r\n"; 1075 1076 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1077 ITestInvocationListener mockListener 1078 = EasyMock.createStrictMock(ITestInvocationListener.class); 1079 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1080 1081 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 1082 tests.add(testId); 1083 1084 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1085 OptionSetter setter = new OptionSetter(deqpTest); 1086 setter.setOptionValue("deqp-screen-rotation", rotation); 1087 1088 deqpTest.setDevice(mockDevice); 1089 1090 int version = 3 << 16; 1091 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1092 .andReturn(Integer.toString(version)).atLeastOnce(); 1093 1094 if (!rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED)) { 1095 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 1096 .andReturn(featureString); 1097 } 1098 1099 final boolean isPortraitOrientation = 1100 rotation.equals(BatchRunConfiguration.ROTATION_PORTRAIT) || 1101 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT); 1102 final boolean isLandscapeOrientation = 1103 rotation.equals(BatchRunConfiguration.ROTATION_LANDSCAPE) || 1104 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE); 1105 final boolean executable = 1106 rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED) || 1107 (isPortraitOrientation && 1108 featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) || 1109 (isLandscapeOrientation && 1110 featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE)); 1111 1112 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 1113 andReturn("").once(); 1114 1115 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 1116 EasyMock.eq(true), 1117 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null) 1118 .once(); 1119 1120 if (executable) { 1121 expectRenderConfigQuery(mockDevice, String.format( 1122 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s " 1123 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1124 + "--deqp-gl-minor-version=0", rotation)); 1125 1126 String commandLine = String.format( 1127 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1128 + "--deqp-screen-rotation=%s " 1129 + "--deqp-surface-type=window " 1130 + "--deqp-log-images=disable " 1131 + "--deqp-watchdog=enable", 1132 CASE_LIST_FILE_NAME, rotation); 1133 1134 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 1135 output); 1136 } 1137 1138 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 1139 .andReturn("").once(); 1140 1141 mockListener.testRunStarted(getTestId(deqpTest), 1); 1142 EasyMock.expectLastCall().once(); 1143 1144 mockListener.testStarted(EasyMock.eq(testId)); 1145 EasyMock.expectLastCall().once(); 1146 1147 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); 1148 EasyMock.expectLastCall().once(); 1149 1150 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 1151 EasyMock.expectLastCall().once(); 1152 1153 EasyMock.replay(mockDevice, mockIDevice); 1154 EasyMock.replay(mockListener); 1155 deqpTest.run(mockListener); 1156 EasyMock.verify(mockListener); 1157 EasyMock.verify(mockDevice, mockIDevice); 1158 } 1159 1160 /** 1161 * Test OpeGL ES3 tests on device with OpenGL ES2. 1162 */ testRun_require30DeviceVersion20()1163 public void testRun_require30DeviceVersion20() throws Exception { 1164 testGlesVersion(3, 0, 2, 0); 1165 } 1166 1167 /** 1168 * Test OpeGL ES3.1 tests on device with OpenGL ES2. 1169 */ testRun_require31DeviceVersion20()1170 public void testRun_require31DeviceVersion20() throws Exception { 1171 testGlesVersion(3, 1, 2, 0); 1172 } 1173 1174 /** 1175 * Test OpeGL ES3 tests on device with OpenGL ES3. 1176 */ testRun_require30DeviceVersion30()1177 public void testRun_require30DeviceVersion30() throws Exception { 1178 testGlesVersion(3, 0, 3, 0); 1179 } 1180 1181 /** 1182 * Test OpeGL ES3.1 tests on device with OpenGL ES3. 1183 */ testRun_require31DeviceVersion30()1184 public void testRun_require31DeviceVersion30() throws Exception { 1185 testGlesVersion(3, 1, 3, 0); 1186 } 1187 1188 /** 1189 * Test OpeGL ES3 tests on device with OpenGL ES3.1. 1190 */ testRun_require30DeviceVersion31()1191 public void testRun_require30DeviceVersion31() throws Exception { 1192 testGlesVersion(3, 0, 3, 1); 1193 } 1194 1195 /** 1196 * Test OpeGL ES3.1 tests on device with OpenGL ES3.1. 1197 */ testRun_require31DeviceVersion31()1198 public void testRun_require31DeviceVersion31() throws Exception { 1199 testGlesVersion(3, 1, 3, 1); 1200 } 1201 1202 /** 1203 * Test dEQP Pass result code. 1204 */ testRun_resultPass()1205 public void testRun_resultPass() throws Exception { 1206 testResultCode("Pass", true); 1207 } 1208 1209 /** 1210 * Test dEQP Fail result code. 1211 */ testRun_resultFail()1212 public void testRun_resultFail() throws Exception { 1213 testResultCode("Fail", false); 1214 } 1215 1216 /** 1217 * Test dEQP NotSupported result code. 1218 */ testRun_resultNotSupported()1219 public void testRun_resultNotSupported() throws Exception { 1220 testResultCode("NotSupported", true); 1221 } 1222 1223 /** 1224 * Test dEQP QualityWarning result code. 1225 */ testRun_resultQualityWarning()1226 public void testRun_resultQualityWarning() throws Exception { 1227 testResultCode("QualityWarning", true); 1228 } 1229 1230 /** 1231 * Test dEQP CompatibilityWarning result code. 1232 */ testRun_resultCompatibilityWarning()1233 public void testRun_resultCompatibilityWarning() throws Exception { 1234 testResultCode("CompatibilityWarning", true); 1235 } 1236 1237 /** 1238 * Test dEQP ResourceError result code. 1239 */ testRun_resultResourceError()1240 public void testRun_resultResourceError() throws Exception { 1241 testResultCode("ResourceError", false); 1242 } 1243 1244 /** 1245 * Test dEQP InternalError result code. 1246 */ testRun_resultInternalError()1247 public void testRun_resultInternalError() throws Exception { 1248 testResultCode("InternalError", false); 1249 } 1250 1251 /** 1252 * Test dEQP Crash result code. 1253 */ testRun_resultCrash()1254 public void testRun_resultCrash() throws Exception { 1255 testResultCode("Crash", false); 1256 } 1257 1258 /** 1259 * Test dEQP Timeout result code. 1260 */ testRun_resultTimeout()1261 public void testRun_resultTimeout() throws Exception { 1262 testResultCode("Timeout", false); 1263 } 1264 /** 1265 * Test dEQP Orientation 1266 */ testRun_orientationLandscape()1267 public void testRun_orientationLandscape() throws Exception { 1268 testOrientation("90", ALL_FEATURES); 1269 } 1270 1271 /** 1272 * Test dEQP Orientation 1273 */ testRun_orientationPortrait()1274 public void testRun_orientationPortrait() throws Exception { 1275 testOrientation("0", ALL_FEATURES); 1276 } 1277 1278 /** 1279 * Test dEQP Orientation 1280 */ testRun_orientationReverseLandscape()1281 public void testRun_orientationReverseLandscape() throws Exception { 1282 testOrientation("270", ALL_FEATURES); 1283 } 1284 1285 /** 1286 * Test dEQP Orientation 1287 */ testRun_orientationReversePortrait()1288 public void testRun_orientationReversePortrait() throws Exception { 1289 testOrientation("180", ALL_FEATURES); 1290 } 1291 1292 /** 1293 * Test dEQP Orientation 1294 */ testRun_orientationUnspecified()1295 public void testRun_orientationUnspecified() throws Exception { 1296 testOrientation("unspecified", ALL_FEATURES); 1297 } 1298 1299 /** 1300 * Test dEQP Orientation with limited features 1301 */ testRun_orientationUnspecifiedLimitedFeatures()1302 public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception { 1303 testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES); 1304 } 1305 1306 /** 1307 * Test dEQP Orientation with limited features 1308 */ testRun_orientationLandscapeLimitedFeatures()1309 public void testRun_orientationLandscapeLimitedFeatures() throws Exception { 1310 testOrientation("90", ONLY_LANDSCAPE_FEATURES); 1311 } 1312 1313 /** 1314 * Test dEQP Orientation with limited features 1315 */ testRun_orientationPortraitLimitedFeatures()1316 public void testRun_orientationPortraitLimitedFeatures() throws Exception { 1317 testOrientation("0", ONLY_LANDSCAPE_FEATURES); 1318 } 1319 1320 /** 1321 * Test dEQP unsupported pixel format 1322 */ testRun_unsupportedPixelFormat()1323 public void testRun_unsupportedPixelFormat() throws Exception { 1324 final String pixelFormat = "rgba5658d16m4"; 1325 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.pixelformat", "test"); 1326 1327 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1328 ITestInvocationListener mockListener 1329 = EasyMock.createStrictMock(ITestInvocationListener.class); 1330 1331 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 1332 tests.add(testId); 1333 1334 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1335 OptionSetter setter = new OptionSetter(deqpTest); 1336 setter.setOptionValue("deqp-gl-config-name", pixelFormat); 1337 1338 deqpTest.setDevice(mockDevice); 1339 1340 int version = 3 << 16; 1341 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1342 .andReturn(Integer.toString(version)).atLeastOnce(); 1343 1344 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 1345 andReturn("").once(); 1346 1347 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 1348 EasyMock.eq(true), 1349 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null) 1350 .once(); 1351 1352 expectRenderConfigQueryAndReturn(mockDevice, String.format( 1353 "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified " 1354 + "--deqp-surface-type=window " 1355 + "--deqp-gl-major-version=3 " 1356 + "--deqp-gl-minor-version=0", pixelFormat), "No"); 1357 1358 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 1359 .andReturn("").once(); 1360 1361 mockListener.testRunStarted(getTestId(deqpTest), 1); 1362 EasyMock.expectLastCall().once(); 1363 1364 mockListener.testStarted(EasyMock.eq(testId)); 1365 EasyMock.expectLastCall().once(); 1366 1367 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); 1368 EasyMock.expectLastCall().once(); 1369 1370 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 1371 EasyMock.expectLastCall().once(); 1372 1373 EasyMock.replay(mockDevice); 1374 EasyMock.replay(mockListener); 1375 deqpTest.run(mockListener); 1376 EasyMock.verify(mockListener); 1377 EasyMock.verify(mockDevice); 1378 } 1379 1380 public static interface RecoverableTestDevice extends ITestDevice { recoverDevice()1381 public void recoverDevice() throws DeviceNotAvailableException; 1382 } 1383 1384 private static enum RecoveryEvent { 1385 PROGRESS, 1386 FAIL_CONNECTION_REFUSED, 1387 FAIL_LINK_KILLED, 1388 } 1389 runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)1390 private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events) 1391 throws DeviceNotAvailableException { 1392 for (RecoveryEvent event : events) { 1393 switch (event) { 1394 case PROGRESS: 1395 recovery.onExecutionProgressed(); 1396 break; 1397 case FAIL_CONNECTION_REFUSED: 1398 recovery.recoverConnectionRefused(); 1399 break; 1400 case FAIL_LINK_KILLED: 1401 recovery.recoverComLinkKilled(); 1402 break; 1403 } 1404 } 1405 } 1406 setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider)1407 private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) { 1408 mockSleepProvider.sleep(EasyMock.gt(0)); 1409 EasyMock.expectLastCall().once(); 1410 } 1411 setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider)1412 private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, 1413 DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException { 1414 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1415 andReturn("root 1234 com.drawelement.deqp").once(); 1416 1417 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1418 andReturn("").once(); 1419 1420 // Recovery checks if kill failed 1421 mockSleepProvider.sleep(EasyMock.gt(0)); 1422 EasyMock.expectLastCall().once(); 1423 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1424 andReturn("").once(); 1425 } 1426 setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)1427 private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice) 1428 throws DeviceNotAvailableException { 1429 mockDevice.recoverDevice(); 1430 EasyMock.expectLastCall().once(); 1431 } 1432 setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)1433 private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice) 1434 throws DeviceNotAvailableException { 1435 mockDevice.reboot(); 1436 EasyMock.expectLastCall().once(); 1437 } 1438 setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)1439 private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, 1440 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 1441 throws DeviceNotAvailableException { 1442 switch (numConsecutiveErrors) { 1443 case 0: 1444 case 1: 1445 setRecoveryExpectationRecovery(mockDevice); 1446 return 2; 1447 case 2: 1448 setRecoveryExpectationReboot(mockDevice); 1449 return 3; 1450 default: 1451 return 4; 1452 } 1453 } 1454 setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)1455 private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, 1456 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 1457 throws DeviceNotAvailableException { 1458 switch (numConsecutiveErrors) { 1459 case 0: 1460 setRecoveryExpectationWait(mockSleepProvider); 1461 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1462 return 1; 1463 case 1: 1464 setRecoveryExpectationRecovery(mockDevice); 1465 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1466 return 2; 1467 case 2: 1468 setRecoveryExpectationReboot(mockDevice); 1469 return 3; 1470 default: 1471 return 4; 1472 } 1473 } 1474 setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)1475 private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, 1476 DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events) 1477 throws DeviceNotAvailableException { 1478 int numConsecutiveErrors = 0; 1479 for (RecoveryEvent event : events) { 1480 switch (event) { 1481 case PROGRESS: 1482 numConsecutiveErrors = 0; 1483 break; 1484 case FAIL_CONNECTION_REFUSED: 1485 numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice, 1486 mockSleepProvider, numConsecutiveErrors); 1487 break; 1488 case FAIL_LINK_KILLED: 1489 numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice, 1490 mockSleepProvider, numConsecutiveErrors); 1491 break; 1492 } 1493 } 1494 } 1495 1496 /** 1497 * Test dEQP runner recovery state machine. 1498 */ testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)1499 private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern) 1500 throws Exception { 1501 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 1502 IMocksControl orderedControl = EasyMock.createStrictControl(); 1503 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 1504 EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL"); 1505 DeqpTestRunner.ISleepProvider mockSleepProvider = 1506 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 1507 1508 setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern); 1509 1510 orderedControl.replay(); 1511 1512 recovery.setDevice(mockDevice); 1513 recovery.setSleepProvider(mockSleepProvider); 1514 try { 1515 runRecoveryWithPattern(recovery, pattern); 1516 if (!expectSuccess) { 1517 fail("Expected DeviceNotAvailableException"); 1518 } 1519 } catch (DeviceNotAvailableException ex) { 1520 if (expectSuccess) { 1521 fail("Did not expect DeviceNotAvailableException"); 1522 } 1523 } 1524 1525 orderedControl.verify(); 1526 } 1527 1528 // basic patterns 1529 testRecovery_NoEvents()1530 public void testRecovery_NoEvents() throws Exception { 1531 testRecoveryWithPattern(true); 1532 } 1533 testRecovery_AllOk()1534 public void testRecovery_AllOk() throws Exception { 1535 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS); 1536 } 1537 1538 // conn fail patterns 1539 testRecovery_OneConnectionFailureBegin()1540 public void testRecovery_OneConnectionFailureBegin() throws Exception { 1541 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1542 RecoveryEvent.PROGRESS); 1543 } 1544 testRecovery_TwoConnectionFailuresBegin()1545 public void testRecovery_TwoConnectionFailuresBegin() throws Exception { 1546 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1547 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1548 } 1549 testRecovery_ThreeConnectionFailuresBegin()1550 public void testRecovery_ThreeConnectionFailuresBegin() throws Exception { 1551 testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1552 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED); 1553 } 1554 testRecovery_OneConnectionFailureMid()1555 public void testRecovery_OneConnectionFailureMid() throws Exception { 1556 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1557 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1558 } 1559 testRecovery_TwoConnectionFailuresMid()1560 public void testRecovery_TwoConnectionFailuresMid() throws Exception { 1561 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1562 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1563 RecoveryEvent.PROGRESS); 1564 } 1565 testRecovery_ThreeConnectionFailuresMid()1566 public void testRecovery_ThreeConnectionFailuresMid() throws Exception { 1567 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, 1568 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1569 RecoveryEvent.FAIL_CONNECTION_REFUSED); 1570 } 1571 1572 // link fail patterns 1573 testRecovery_OneLinkFailureBegin()1574 public void testRecovery_OneLinkFailureBegin() throws Exception { 1575 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1576 RecoveryEvent.PROGRESS); 1577 } 1578 testRecovery_TwoLinkFailuresBegin()1579 public void testRecovery_TwoLinkFailuresBegin() throws Exception { 1580 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1581 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1582 } 1583 testRecovery_ThreeLinkFailuresBegin()1584 public void testRecovery_ThreeLinkFailuresBegin() throws Exception { 1585 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1586 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1587 RecoveryEvent.PROGRESS); 1588 } 1589 testRecovery_FourLinkFailuresBegin()1590 public void testRecovery_FourLinkFailuresBegin() throws Exception { 1591 testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED, 1592 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1593 RecoveryEvent.FAIL_LINK_KILLED); 1594 } 1595 testRecovery_OneLinkFailureMid()1596 public void testRecovery_OneLinkFailureMid() throws Exception { 1597 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1598 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1599 } 1600 testRecovery_TwoLinkFailuresMid()1601 public void testRecovery_TwoLinkFailuresMid() throws Exception { 1602 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1603 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1604 RecoveryEvent.PROGRESS); 1605 } 1606 testRecovery_ThreeLinkFailuresMid()1607 public void testRecovery_ThreeLinkFailuresMid() throws Exception { 1608 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1609 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1610 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1611 } 1612 testRecovery_FourLinkFailuresMid()1613 public void testRecovery_FourLinkFailuresMid() throws Exception { 1614 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1615 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1616 RecoveryEvent.FAIL_LINK_KILLED); 1617 } 1618 1619 // mixed patterns 1620 testRecovery_MixedFailuresProgressBetween()1621 public void testRecovery_MixedFailuresProgressBetween() throws Exception { 1622 testRecoveryWithPattern(true, 1623 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1624 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1625 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1626 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1627 RecoveryEvent.PROGRESS); 1628 } 1629 testRecovery_MixedFailuresNoProgressBetween()1630 public void testRecovery_MixedFailuresNoProgressBetween() throws Exception { 1631 testRecoveryWithPattern(true, 1632 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1633 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED, 1634 RecoveryEvent.PROGRESS); 1635 } 1636 1637 /** 1638 * Test recovery if process cannot be killed 1639 */ testRecovery_unkillableProcess()1640 public void testRecovery_unkillableProcess () throws Exception { 1641 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 1642 IMocksControl orderedControl = EasyMock.createStrictControl(); 1643 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 1644 DeqpTestRunner.ISleepProvider mockSleepProvider = 1645 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 1646 1647 // recovery attempts to kill the process after a timeout 1648 mockSleepProvider.sleep(EasyMock.gt(0)); 1649 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1650 andReturn("root 1234 com.drawelement.deqp").once(); 1651 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1652 andReturn("").once(); 1653 1654 // Recovery checks if kill failed 1655 mockSleepProvider.sleep(EasyMock.gt(0)); 1656 EasyMock.expectLastCall().once(); 1657 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1658 andReturn("root 1234 com.drawelement.deqp").once(); 1659 1660 // Recovery resets the connection 1661 mockDevice.recoverDevice(); 1662 EasyMock.expectLastCall().once(); 1663 1664 // and attempts to kill the process again 1665 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1666 andReturn("root 1234 com.drawelement.deqp").once(); 1667 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1668 andReturn("").once(); 1669 1670 // Recovery checks if kill failed 1671 mockSleepProvider.sleep(EasyMock.gt(0)); 1672 EasyMock.expectLastCall().once(); 1673 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1674 andReturn("root 1234 com.drawelement.deqp").once(); 1675 1676 // recovery reboots the device 1677 mockDevice.reboot(); 1678 EasyMock.expectLastCall().once(); 1679 1680 orderedControl.replay(); 1681 recovery.setDevice(mockDevice); 1682 recovery.setSleepProvider(mockSleepProvider); 1683 recovery.recoverComLinkKilled(); 1684 orderedControl.verify(); 1685 } 1686 1687 /** 1688 * Test external interruption before batch run. 1689 */ testInterrupt_killBeforeBatch()1690 public void testInterrupt_killBeforeBatch() throws Exception { 1691 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test"); 1692 1693 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 1694 tests.add(testId); 1695 1696 ITestInvocationListener mockListener 1697 = EasyMock.createStrictMock(ITestInvocationListener.class); 1698 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1699 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1700 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 1701 1702 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1703 1704 deqpTest.setDevice(mockDevice); 1705 deqpTest.setRunUtil(mockRunUtil); 1706 1707 int version = 3 << 16; 1708 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1709 .andReturn(Integer.toString(version)).atLeastOnce(); 1710 1711 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 1712 andReturn("").once(); 1713 1714 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 1715 EasyMock.eq(true), 1716 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null) 1717 .once(); 1718 1719 expectRenderConfigQuery(mockDevice, 1720 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 1721 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1722 + "--deqp-gl-minor-version=0"); 1723 1724 mockRunUtil.sleep(0); 1725 EasyMock.expectLastCall().andThrow(new RunInterruptedException()); 1726 1727 mockListener.testRunStarted(getTestId(deqpTest), 1); 1728 EasyMock.expectLastCall().once(); 1729 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject()); 1730 EasyMock.expectLastCall().once(); 1731 1732 EasyMock.replay(mockDevice, mockIDevice); 1733 EasyMock.replay(mockListener); 1734 EasyMock.replay(mockRunUtil); 1735 try { 1736 deqpTest.run(mockListener); 1737 fail("expected RunInterruptedException"); 1738 } catch (RunInterruptedException ex) { 1739 // expected 1740 } 1741 EasyMock.verify(mockRunUtil); 1742 EasyMock.verify(mockListener); 1743 EasyMock.verify(mockDevice, mockIDevice); 1744 } 1745 runShardedTest(TestIdentifier[] testIds, ArrayList<ArrayList<TestIdentifier>> testsForShard)1746 private void runShardedTest(TestIdentifier[] testIds, 1747 ArrayList<ArrayList<TestIdentifier>> testsForShard) throws Exception { 1748 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 1749 for (TestIdentifier id : testIds) tests.add(id); 1750 1751 DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests, mTestsDir); 1752 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split(); 1753 1754 for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) { 1755 DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex); 1756 shard.setBuildHelper(getMockBuildHelper(mTestsDir)); 1757 1758 ArrayList<TestIdentifier> shardTests = testsForShard.get(shardIndex); 1759 1760 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1761 ITestInvocationListener mockListener 1762 = EasyMock.createStrictMock(ITestInvocationListener.class); 1763 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1764 int version = 3 << 16; 1765 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1766 .andReturn(Integer.toString(version)).atLeastOnce(); 1767 1768 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 1769 .once(); 1770 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 1771 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))) 1772 .andReturn(null).once(); 1773 1774 mockListener.testRunStarted(getTestId(shard), shardTests.size()); 1775 EasyMock.expectLastCall().once(); 1776 1777 expectRenderConfigQuery(mockDevice, 3, 0); 1778 1779 String testOut = buildTestProcessOutput(shardTests); 1780 // NOTE: This assumes that there won't be multiple batches per shard! 1781 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); 1782 1783 for (int i = 0; i < shardTests.size(); i++) { 1784 mockListener.testStarted(EasyMock.eq(shardTests.get(i))); 1785 EasyMock.expectLastCall().once(); 1786 1787 mockListener.testEnded(EasyMock.eq(shardTests.get(i)), 1788 EasyMock.<Map<String, String>>notNull()); 1789 1790 EasyMock.expectLastCall().once(); 1791 } 1792 1793 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 1794 EasyMock.expectLastCall().once(); 1795 1796 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 1797 .once(); 1798 1799 EasyMock.replay(mockDevice, mockIDevice); 1800 EasyMock.replay(mockListener); 1801 1802 shard.setDevice(mockDevice); 1803 shard.run(mockListener); 1804 1805 EasyMock.verify(mockListener); 1806 EasyMock.verify(mockDevice, mockIDevice); 1807 } 1808 } 1809 testSharding_smallTrivial()1810 public void testSharding_smallTrivial() throws Exception { 1811 final TestIdentifier[] testIds = { 1812 new TestIdentifier("dEQP-GLES3.info", "vendor"), 1813 new TestIdentifier("dEQP-GLES3.info", "renderer"), 1814 new TestIdentifier("dEQP-GLES3.info", "version"), 1815 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"), 1816 new TestIdentifier("dEQP-GLES3.info", "extensions"), 1817 new TestIdentifier("dEQP-GLES3.info", "render_target") 1818 }; 1819 ArrayList<ArrayList<TestIdentifier>> shardedTests = new ArrayList<>(); 1820 ArrayList<TestIdentifier> shardOne = new ArrayList<>(); 1821 for (int i = 0; i < testIds.length; i++) { 1822 shardOne.add(testIds[i]); 1823 } 1824 shardedTests.add(shardOne); 1825 runShardedTest(testIds, shardedTests); 1826 } 1827 testSharding_twoShards()1828 public void testSharding_twoShards() throws Exception { 1829 final int TEST_COUNT = 1237; 1830 final int SHARD_SIZE = 1000; 1831 1832 ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT); 1833 for (int i = 0; i < TEST_COUNT; i++) { 1834 testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i))); 1835 } 1836 1837 ArrayList<ArrayList<TestIdentifier>> shardedTests = new ArrayList<>(); 1838 ArrayList<TestIdentifier> shard = new ArrayList<>(); 1839 for (int i = 0; i < testIds.size(); i++) { 1840 if (i == SHARD_SIZE) { 1841 shardedTests.add(shard); 1842 shard = new ArrayList<>(); 1843 } 1844 shard.add(testIds.get(i)); 1845 } 1846 shardedTests.add(shard); 1847 runShardedTest(testIds.toArray(new TestIdentifier[testIds.size()]), shardedTests); 1848 } 1849 testSharding_empty()1850 public void testSharding_empty() throws Exception { 1851 DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList<TestIdentifier>(), mTestsDir); 1852 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split(); 1853 // Returns null when cannot be sharded. 1854 assertNull(shards); 1855 } 1856 1857 /** 1858 * Test external interruption in testFailed(). 1859 */ testInterrupt_killReportTestFailed()1860 public void testInterrupt_killReportTestFailed() throws Exception { 1861 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test"); 1862 final String testPath = "dEQP-GLES3.interrupt.test"; 1863 final String testTrie = "{dEQP-GLES3{interrupt{test}}}"; 1864 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1865 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1866 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1867 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1868 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1869 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1870 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1871 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1872 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1873 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1874 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1875 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1876 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1877 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1878 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1879 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 1880 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1881 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n" 1882 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n" 1883 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1884 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1885 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1886 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1887 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1888 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1889 + "INSTRUMENTATION_CODE: 0\r\n"; 1890 1891 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 1892 tests.add(testId); 1893 1894 ITestInvocationListener mockListener 1895 = EasyMock.createStrictMock(ITestInvocationListener.class); 1896 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1897 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1898 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 1899 1900 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1901 1902 deqpTest.setDevice(mockDevice); 1903 deqpTest.setRunUtil(mockRunUtil); 1904 1905 int version = 3 << 16; 1906 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1907 .andReturn(Integer.toString(version)).atLeastOnce(); 1908 1909 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 1910 andReturn("").once(); 1911 1912 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 1913 EasyMock.eq(true), 1914 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null) 1915 .once(); 1916 1917 expectRenderConfigQuery(mockDevice, 1918 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 1919 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1920 + "--deqp-gl-minor-version=0"); 1921 1922 mockRunUtil.sleep(0); 1923 EasyMock.expectLastCall().once(); 1924 1925 String commandLine = String.format( 1926 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1927 + "--deqp-screen-rotation=unspecified " 1928 + "--deqp-surface-type=window " 1929 + "--deqp-log-images=disable " 1930 + "--deqp-watchdog=enable", 1931 CASE_LIST_FILE_NAME); 1932 1933 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 1934 output); 1935 1936 mockListener.testRunStarted(getTestId(deqpTest), 1); 1937 EasyMock.expectLastCall().once(); 1938 1939 mockListener.testStarted(EasyMock.eq(testId)); 1940 EasyMock.expectLastCall().once(); 1941 1942 mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull()); 1943 EasyMock.expectLastCall().andThrow(new RunInterruptedException()); 1944 1945 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject()); 1946 EasyMock.expectLastCall().once(); 1947 EasyMock.replay(mockDevice, mockIDevice); 1948 EasyMock.replay(mockListener); 1949 EasyMock.replay(mockRunUtil); 1950 try { 1951 deqpTest.run(mockListener); 1952 fail("expected RunInterruptedException"); 1953 } catch (RunInterruptedException ex) { 1954 // expected 1955 } 1956 EasyMock.verify(mockRunUtil); 1957 EasyMock.verify(mockListener); 1958 EasyMock.verify(mockDevice, mockIDevice); 1959 } 1960 testRuntimeHint_optionSet()1961 public void testRuntimeHint_optionSet() throws Exception { 1962 /* MultiLineReceiver expects "\r\n" line ending. */ 1963 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1964 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1965 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1966 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1967 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1968 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1969 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1970 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1971 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1972 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1973 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1974 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1975 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1976 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1977 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1978 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 1979 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1980 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1981 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1982 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1983 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1984 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1985 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1986 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1987 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 1988 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1989 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1990 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1991 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1992 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1993 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1994 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1995 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1996 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 1997 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1998 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1999 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2000 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2001 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2002 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2003 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2004 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2005 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 2006 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2007 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2008 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2009 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2010 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2011 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2012 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2013 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2014 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 2015 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2016 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2017 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2018 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2019 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2020 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2021 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2022 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2023 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 2024 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2025 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 2026 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 2027 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2028 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2029 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2030 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2031 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 2032 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2033 + "INSTRUMENTATION_CODE: 0\r\n"; 2034 2035 final TestIdentifier[] testIds = { 2036 new TestIdentifier("dEQP-GLES3.info", "vendor"), 2037 new TestIdentifier("dEQP-GLES3.info", "renderer"), 2038 new TestIdentifier("dEQP-GLES3.info", "version"), 2039 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"), 2040 new TestIdentifier("dEQP-GLES3.info", "extensions"), 2041 new TestIdentifier("dEQP-GLES3.info", "render_target") 2042 }; 2043 2044 final String[] testPaths = { 2045 "dEQP-GLES3.info.vendor", 2046 "dEQP-GLES3.info.renderer", 2047 "dEQP-GLES3.info.version", 2048 "dEQP-GLES3.info.shading_language_version", 2049 "dEQP-GLES3.info.extensions", 2050 "dEQP-GLES3.info.render_target" 2051 }; 2052 2053 final String testTrie 2054 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 2055 2056 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2057 ITestInvocationListener mockListener 2058 = EasyMock.createStrictMock(ITestInvocationListener.class); 2059 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 2060 2061 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 2062 2063 for (TestIdentifier id : testIds) { 2064 tests.add(id); 2065 } 2066 2067 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2068 OptionSetter setter = new OptionSetter(deqpTest); 2069 final long runtimeMs = 123456; 2070 setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs)); 2071 assertEquals("Wrong expected runtime - option not passed cleanly", runtimeMs, deqpTest.getRuntimeHint()); 2072 2073 // Try running the tests as well. The unit tests do not set the hint be default, 2074 // so that case is covered. 2075 2076 int version = 3 << 16; 2077 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 2078 .andReturn(Integer.toString(version)).atLeastOnce(); 2079 2080 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 2081 .once(); 2082 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 2083 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))) 2084 .andReturn(null).once(); 2085 2086 expectRenderConfigQuery(mockDevice, 3, 0); 2087 2088 String commandLine = String.format( 2089 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2090 + "--deqp-screen-rotation=unspecified " 2091 + "--deqp-surface-type=window " 2092 + "--deqp-log-images=disable " 2093 + "--deqp-watchdog=enable", 2094 CASE_LIST_FILE_NAME); 2095 2096 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 2097 2098 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 2099 EasyMock.expectLastCall().once(); 2100 2101 for (int i = 0; i < testPaths.length; i++) { 2102 mockListener.testStarted(EasyMock.eq(testIds[i])); 2103 EasyMock.expectLastCall().once(); 2104 2105 mockListener.testEnded(EasyMock.eq(testIds[i]), 2106 EasyMock.<Map<String, String>>notNull()); 2107 2108 EasyMock.expectLastCall().once(); 2109 } 2110 2111 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 2112 EasyMock.expectLastCall().once(); 2113 2114 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 2115 .once(); 2116 2117 EasyMock.replay(mockDevice, mockIDevice); 2118 EasyMock.replay(mockListener); 2119 2120 deqpTest.setDevice(mockDevice); 2121 deqpTest.run(mockListener); 2122 2123 EasyMock.verify(mockListener); 2124 EasyMock.verify(mockDevice, mockIDevice); 2125 } 2126 testRuntimeHint_optionSetSharded()2127 public void testRuntimeHint_optionSetSharded() throws Exception { 2128 final int TEST_COUNT = 1237; 2129 final int SHARD_SIZE = 1000; 2130 2131 ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT); 2132 for (int i = 0; i < TEST_COUNT; i++) { 2133 testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i))); 2134 } 2135 2136 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir); 2137 OptionSetter setter = new OptionSetter(deqpTest); 2138 final long fullRuntimeMs = testIds.size()*100; 2139 setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs)); 2140 2141 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)deqpTest.split(); 2142 assertEquals("First shard's time not proportional to test count", 2143 (fullRuntimeMs*SHARD_SIZE)/TEST_COUNT, 2144 ((IRuntimeHintProvider)shards.get(0)).getRuntimeHint()); 2145 assertEquals("Second shard's time not proportional to test count", 2146 (fullRuntimeMs*(TEST_COUNT-SHARD_SIZE))/TEST_COUNT, 2147 ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint()); 2148 } 2149 2150 /** 2151 * Test that strict shardable is able to split deterministically the set of tests. 2152 */ testGetTestShard()2153 public void testGetTestShard() throws Exception { 2154 final int TEST_COUNT = 2237; 2155 final int SHARD_COUNT = 4; 2156 2157 ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT); 2158 for (int i = 0; i < TEST_COUNT; i++) { 2159 testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i))); 2160 } 2161 2162 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir); 2163 OptionSetter setter = new OptionSetter(deqpTest); 2164 final long fullRuntimeMs = testIds.size()*100; 2165 setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs)); 2166 2167 DeqpTestRunner shard1 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 0); 2168 assertEquals(559, shard1.getTestInstance().size()); 2169 int j = 0; 2170 // Ensure numbers, and that order is stable 2171 for (TestIdentifier t : shard1.getTestInstance().keySet()) { 2172 assertEquals(String.format("dEQP-GLES3.funny.group#%s", j), 2173 String.format("%s#%s", t.getClassName(), t.getTestName())); 2174 j++; 2175 } 2176 DeqpTestRunner shard2 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 1); 2177 assertEquals(559, shard2.getTestInstance().size()); 2178 for (TestIdentifier t : shard2.getTestInstance().keySet()) { 2179 assertEquals(String.format("dEQP-GLES3.funny.group#%s", j), 2180 String.format("%s#%s", t.getClassName(), t.getTestName())); 2181 j++; 2182 } 2183 DeqpTestRunner shard3 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 2); 2184 assertEquals(559, shard3.getTestInstance().size()); 2185 for (TestIdentifier t : shard3.getTestInstance().keySet()) { 2186 assertEquals(String.format("dEQP-GLES3.funny.group#%s", j), 2187 String.format("%s#%s", t.getClassName(), t.getTestName())); 2188 j++; 2189 } 2190 DeqpTestRunner shard4 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 3); 2191 assertEquals(560, shard4.getTestInstance().size()); 2192 for (TestIdentifier t : shard4.getTestInstance().keySet()) { 2193 assertEquals(String.format("dEQP-GLES3.funny.group#%s", j), 2194 String.format("%s#%s", t.getClassName(), t.getTestName())); 2195 j++; 2196 } 2197 assertEquals(TEST_COUNT, j); 2198 } 2199 2200 /** 2201 * Test that strict shardable is creating an empty shard of the runner when too many shards 2202 * are requested. 2203 */ testGetTestShard_tooManyShardRequested()2204 public void testGetTestShard_tooManyShardRequested() throws Exception { 2205 final int TEST_COUNT = 2; 2206 final int SHARD_COUNT = 3; 2207 2208 ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT); 2209 for (int i = 0; i < TEST_COUNT; i++) { 2210 testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i))); 2211 } 2212 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir); 2213 OptionSetter setter = new OptionSetter(deqpTest); 2214 final long fullRuntimeMs = testIds.size()*100; 2215 setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs)); 2216 DeqpTestRunner shard1 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 0); 2217 assertEquals(1, shard1.getTestInstance().size()); 2218 int j = 0; 2219 // Ensure numbers, and that order is stable 2220 for (TestIdentifier t : shard1.getTestInstance().keySet()) { 2221 assertEquals(String.format("dEQP-GLES3.funny.group#%s", j), 2222 String.format("%s#%s", t.getClassName(), t.getTestName())); 2223 j++; 2224 } 2225 DeqpTestRunner shard2 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 1); 2226 assertEquals(1, shard2.getTestInstance().size()); 2227 for (TestIdentifier t : shard2.getTestInstance().keySet()) { 2228 assertEquals(String.format("dEQP-GLES3.funny.group#%s", j), 2229 String.format("%s#%s", t.getClassName(), t.getTestName())); 2230 j++; 2231 } 2232 DeqpTestRunner shard3 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 2); 2233 assertTrue(shard3.getTestInstance().isEmpty()); 2234 assertEquals(TEST_COUNT, j); 2235 ITestInvocationListener mockListener 2236 = EasyMock.createStrictMock(ITestInvocationListener.class); 2237 mockListener.testRunStarted(EasyMock.anyObject(), EasyMock.eq(0)); 2238 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject()); 2239 EasyMock.replay(mockListener); 2240 shard3.run(mockListener); 2241 EasyMock.verify(mockListener); 2242 } 2243 testRuntimeHint_optionNotSet()2244 public void testRuntimeHint_optionNotSet() throws Exception { 2245 final TestIdentifier[] testIds = { 2246 new TestIdentifier("dEQP-GLES3.info", "vendor"), 2247 new TestIdentifier("dEQP-GLES3.info", "renderer"), 2248 new TestIdentifier("dEQP-GLES3.info", "version"), 2249 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"), 2250 new TestIdentifier("dEQP-GLES3.info", "extensions"), 2251 new TestIdentifier("dEQP-GLES3.info", "render_target") 2252 }; 2253 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 2254 2255 for (TestIdentifier id : testIds) { 2256 tests.add(id); 2257 } 2258 2259 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2260 2261 long runtime = deqpTest.getRuntimeHint(); 2262 assertTrue("Runtime for tests must be positive", runtime > 0); 2263 assertTrue("Runtime for tests must be reasonable", runtime < (1000 * 10)); // Must be done in 10s 2264 } 2265 2266 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String output)2267 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 2268 final String output) throws Exception { 2269 String cmd = String.format( 2270 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2271 + "--deqp-screen-rotation=unspecified " 2272 + "--deqp-surface-type=window " 2273 + "--deqp-log-images=disable " 2274 + "--deqp-watchdog=enable", 2275 CASE_LIST_FILE_NAME); 2276 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, cmd, output); 2277 } 2278 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String testTrie, final String cmd, final String output)2279 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 2280 final String testTrie, final String cmd, final String output) throws Exception { 2281 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME))) 2282 .andReturn("").once(); 2283 2284 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME))) 2285 .andReturn("").once(); 2286 2287 if (testTrie == null) { 2288 mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(CASE_LIST_FILE_NAME)); 2289 } 2290 else { 2291 mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME); 2292 } 2293 EasyMock.expectLastCall().andReturn(true).once(); 2294 2295 String command = String.format( 2296 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" " 2297 + "-e deqpLogData \"%s\" %s", 2298 AbiUtils.createAbiFlag(ABI.getName()), LOG_FILE_NAME, cmd, false, 2299 INSTRUMENTATION_NAME); 2300 2301 EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice); 2302 mockIDevice.executeShellCommand(EasyMock.eq(command), 2303 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(), 2304 EasyMock.isA(TimeUnit.class)); 2305 2306 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 2307 @Override 2308 public Object answer() { 2309 IShellOutputReceiver receiver 2310 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 2311 2312 receiver.addOutput(output.getBytes(), 0, output.length()); 2313 receiver.flush(); 2314 2315 return null; 2316 } 2317 }); 2318 } 2319 writeStringsToFile(File target, Set<String> strings)2320 static private void writeStringsToFile(File target, Set<String> strings) throws IOException { 2321 try (PrintWriter out = new PrintWriter(new FileWriter(target))) { 2322 out.print(String.join(System.lineSeparator(), strings)); 2323 out.println(); 2324 } 2325 } 2326 addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option)2327 private void addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option) 2328 throws IOException, ConfigurationException { 2329 String filterFile = option + ".txt"; 2330 writeStringsToFile(new File(mTestsDir, filterFile), filters); 2331 OptionSetter setter = new OptionSetter(test); 2332 setter.setOptionValue(option, filterFile); 2333 } 2334 testIncludeFilterFile()2335 public void testIncludeFilterFile() throws Exception { 2336 final TestIdentifier[] testIds = { 2337 new TestIdentifier("dEQP-GLES3.missing", "no"), 2338 new TestIdentifier("dEQP-GLES3.missing", "nope"), 2339 new TestIdentifier("dEQP-GLES3.missing", "donotwant"), 2340 new TestIdentifier("dEQP-GLES3.pick_me", "yes"), 2341 new TestIdentifier("dEQP-GLES3.pick_me", "ok"), 2342 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"), 2343 }; 2344 2345 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>(); 2346 for (TestIdentifier id : testIds) { 2347 allTests.add(id); 2348 } 2349 2350 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>(); 2351 activeTests.add(testIds[3]); 2352 activeTests.add(testIds[4]); 2353 activeTests.add(testIds[5]); 2354 2355 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2356 2357 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2358 Set<String> includes = new HashSet<>(); 2359 includes.add("dEQP-GLES3.pick_me#*"); 2360 addFilterFileForOption(deqpTest, includes, "include-filter-file"); 2361 testFiltering(deqpTest, expectedTrie, activeTests); 2362 } 2363 testMissingIncludeFilterFile()2364 public void testMissingIncludeFilterFile() throws Exception { 2365 final TestIdentifier[] testIds = { 2366 new TestIdentifier("dEQP-GLES3.pick_me", "yes"), 2367 new TestIdentifier("dEQP-GLES3.pick_me", "ok"), 2368 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"), 2369 }; 2370 2371 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>(); 2372 for (TestIdentifier id : testIds) { 2373 allTests.add(id); 2374 } 2375 2376 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2377 2378 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2379 OptionSetter setter = new OptionSetter(deqpTest); 2380 setter.setOptionValue("include-filter-file", "not-a-file.txt"); 2381 try { 2382 testFiltering(deqpTest, expectedTrie, allTests); 2383 fail("Test execution should have aborted with exception."); 2384 } catch (RuntimeException e) { 2385 } 2386 } 2387 testExcludeFilterFile()2388 public void testExcludeFilterFile() throws Exception { 2389 final TestIdentifier[] testIds = { 2390 new TestIdentifier("dEQP-GLES3.missing", "no"), 2391 new TestIdentifier("dEQP-GLES3.missing", "nope"), 2392 new TestIdentifier("dEQP-GLES3.missing", "donotwant"), 2393 new TestIdentifier("dEQP-GLES3.pick_me", "yes"), 2394 new TestIdentifier("dEQP-GLES3.pick_me", "ok"), 2395 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"), 2396 }; 2397 2398 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>(); 2399 for (TestIdentifier id : testIds) { 2400 allTests.add(id); 2401 } 2402 2403 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>(); 2404 activeTests.add(testIds[3]); 2405 activeTests.add(testIds[4]); 2406 activeTests.add(testIds[5]); 2407 2408 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2409 2410 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2411 Set<String> excludes = new HashSet<>(); 2412 excludes.add("dEQP-GLES3.missing#*"); 2413 addFilterFileForOption(deqpTest, excludes, "exclude-filter-file"); 2414 testFiltering(deqpTest, expectedTrie, activeTests); 2415 } 2416 testFilterComboWithFiles()2417 public void testFilterComboWithFiles() throws Exception { 2418 final TestIdentifier[] testIds = { 2419 new TestIdentifier("dEQP-GLES3.group1", "footah"), 2420 new TestIdentifier("dEQP-GLES3.group1", "foo"), 2421 new TestIdentifier("dEQP-GLES3.group1", "nope"), 2422 new TestIdentifier("dEQP-GLES3.group1", "nonotwant"), 2423 new TestIdentifier("dEQP-GLES3.group2", "foo"), 2424 new TestIdentifier("dEQP-GLES3.group2", "yes"), 2425 new TestIdentifier("dEQP-GLES3.group2", "thoushallnotpass"), 2426 }; 2427 2428 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>(); 2429 for (TestIdentifier id : testIds) { 2430 allTests.add(id); 2431 } 2432 2433 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>(); 2434 activeTests.add(testIds[0]); 2435 activeTests.add(testIds[5]); 2436 2437 String expectedTrie = "{dEQP-GLES3{group1{footah}group2{yes}}}"; 2438 2439 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2440 2441 Set<String> includes = new HashSet<>(); 2442 includes.add("dEQP-GLES3.group2#*"); 2443 deqpTest.addAllIncludeFilters(includes); 2444 2445 Set<String> fileIncludes = new HashSet<>(); 2446 fileIncludes.add("dEQP-GLES3.group1#no*"); 2447 fileIncludes.add("dEQP-GLES3.group1#foo*"); 2448 addFilterFileForOption(deqpTest, fileIncludes, "include-filter-file"); 2449 2450 Set<String> fileExcludes = new HashSet<>(); 2451 fileExcludes.add("*foo"); 2452 fileExcludes.add("*thoushallnotpass"); 2453 addFilterFileForOption(deqpTest, fileExcludes, "exclude-filter-file"); 2454 2455 deqpTest.addExcludeFilter("dEQP-GLES3.group1#no*"); 2456 2457 testFiltering(deqpTest, expectedTrie, activeTests); 2458 } 2459 testDotToHashConversionInFilters()2460 public void testDotToHashConversionInFilters() throws Exception { 2461 final TestIdentifier[] testIds = { 2462 new TestIdentifier("dEQP-GLES3.missing", "no"), 2463 new TestIdentifier("dEQP-GLES3.pick_me", "donotwant"), 2464 new TestIdentifier("dEQP-GLES3.pick_me", "yes") 2465 }; 2466 2467 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>(); 2468 for (TestIdentifier id : testIds) { 2469 allTests.add(id); 2470 } 2471 2472 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>(); 2473 activeTests.add(testIds[2]); 2474 2475 String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}"; 2476 2477 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2478 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me.yes"); 2479 testFiltering(deqpTest, expectedTrie, activeTests); 2480 } 2481 } 2482