1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.platform.test.longevity; 17 18 import static org.mockito.ArgumentMatchers.any; 19 import static org.mockito.ArgumentMatchers.anyLong; 20 import static org.mockito.ArgumentMatchers.longThat; 21 import static org.mockito.Mockito.atLeastOnce; 22 import static org.mockito.Mockito.never; 23 import static org.mockito.Mockito.mock; 24 import static org.mockito.Mockito.spy; 25 import static org.mockito.Mockito.times; 26 import static org.mockito.Mockito.verify; 27 import static org.mockito.MockitoAnnotations.initMocks; 28 import static java.lang.Math.abs; 29 30 import android.os.Bundle; 31 import android.platform.test.longevity.proto.Configuration.Scenario; 32 import android.platform.test.longevity.proto.Configuration.Scenario.AfterTest; 33 import android.platform.test.longevity.proto.Configuration.Scenario.ExtraArg; 34 import android.platform.test.longevity.samples.testing.SampleProfileSuite; 35 import androidx.test.InstrumentationRegistry; 36 37 import org.junit.Assert; 38 import org.junit.After; 39 import org.junit.Before; 40 import org.junit.Test; 41 import org.junit.runner.RunWith; 42 import org.junit.runner.notification.Failure; 43 import org.junit.runner.notification.RunListener; 44 import org.junit.runner.notification.RunNotifier; 45 import org.junit.runners.JUnit4; 46 import org.junit.runners.model.InitializationError; 47 import org.junit.runners.model.TestTimedOutException; 48 import org.mockito.ArgumentCaptor; 49 import org.mockito.Mock; 50 import org.mockito.exceptions.base.MockitoAssertionError; 51 52 import java.util.HashSet; 53 import java.util.List; 54 import java.util.concurrent.TimeUnit; 55 56 57 /** Unit tests for the {@link ScheduledScenarioRunner} runner. */ 58 @RunWith(JUnit4.class) 59 public class ScheduledScenarioRunnerTest { 60 61 @Mock private RunNotifier mRunNotifier; 62 63 private static final String ASSERTION_FAILURE_MESSAGE = "Test assertion failed"; 64 65 public static class ArgumentTest { 66 public static final String TEST_ARG = "test-arg-test-only"; 67 public static final String TEST_ARG_DEFAULT = "default"; 68 public static final String TEST_ARG_OVERRIDE = "not default"; 69 70 @Before setUp()71 public void setUp() { 72 // The actual argument testing happens here as this is where instrumentation args are 73 // parsed in the CUJs. 74 String argValue = 75 InstrumentationRegistry.getArguments().getString(TEST_ARG, TEST_ARG_DEFAULT); 76 Assert.assertEquals(ASSERTION_FAILURE_MESSAGE, argValue, TEST_ARG_OVERRIDE); 77 } 78 79 @Test dummyTest()80 public void dummyTest() { 81 // Does nothing; always passes. 82 } 83 } 84 85 // Threshold above which missing a schedule is considered a failure. 86 private static final long TIMEOUT_ERROR_MARGIN_MS = 500; 87 88 // Holds the state of the instrumentation args before each test for restoring after, as one test 89 // might affect the state of another otherwise. 90 // TODO(b/124239142): Avoid manipulating the instrumentation args here. 91 private Bundle mArgumentsBeforeTest; 92 93 @Before setUpSuite()94 public void setUpSuite() throws InitializationError { 95 initMocks(this); 96 mArgumentsBeforeTest = InstrumentationRegistry.getArguments(); 97 } 98 99 @After restoreSuite()100 public void restoreSuite() { 101 InstrumentationRegistry.registerInstance( 102 InstrumentationRegistry.getInstrumentation(), mArgumentsBeforeTest); 103 } 104 105 /** 106 * Test that an over time test causes a JUnit TestTimedOutException with the correct exception 107 * timeout. 108 */ 109 @Test testOverTimeTest_throwsTestTimedOutException()110 public void testOverTimeTest_throwsTestTimedOutException() throws InitializationError { 111 ArgumentCaptor<Failure> failureCaptor = ArgumentCaptor.forClass(Failure.class); 112 // Set a over time test with a 5-second window that will idle until the end of the window is 113 // reached. 114 long timeoutMs = TimeUnit.SECONDS.toMillis(5); 115 Scenario testScenario = 116 Scenario.newBuilder() 117 .setAt("00:00:00") 118 .setJourney(SampleProfileSuite.LongIdleTest.class.getName()) 119 .setAfterTest(AfterTest.STAY_IN_APP) 120 .build(); 121 ScheduledScenarioRunner runner = 122 spy( 123 new ScheduledScenarioRunner( 124 SampleProfileSuite.LongIdleTest.class, 125 testScenario, 126 timeoutMs, 127 true)); 128 runner.run(mRunNotifier); 129 // Verify that a TestTimedOutException is fired and that the timeout is correct. 130 verify(mRunNotifier, atLeastOnce()).fireTestFailure(failureCaptor.capture()); 131 List<Failure> failures = failureCaptor.getAllValues(); 132 boolean correctTestTimedOutExceptionFired = 133 failures.stream() 134 .anyMatch( 135 f -> { 136 if (!(f.getException() instanceof TestTimedOutException)) { 137 return false; 138 } 139 TestTimedOutException exception = 140 (TestTimedOutException) f.getException(); 141 long exceptionTimeout = 142 exception 143 .getTimeUnit() 144 .toMillis(exception.getTimeout()); 145 long expectedTimeout = 146 timeoutMs - ScheduledScenarioRunner.ENDTIME_LEEWAY_MS; 147 return abs(exceptionTimeout - expectedTimeout) 148 <= TIMEOUT_ERROR_MARGIN_MS; 149 }); 150 Assert.assertTrue(correctTestTimedOutExceptionFired); 151 } 152 153 /** Test that an over time test does not idle before teardown. */ 154 @Test testOverTimeTest_doesNotIdleBeforeTeardown()155 public void testOverTimeTest_doesNotIdleBeforeTeardown() throws InitializationError { 156 // Set a over time test with a 5-second window that will idle until the end of the window is 157 // reached. 158 Scenario testScenario = 159 Scenario.newBuilder() 160 .setAt("00:00:00") 161 .setJourney(SampleProfileSuite.LongIdleTest.class.getName()) 162 .setAfterTest(AfterTest.STAY_IN_APP) 163 .build(); 164 ScheduledScenarioRunner runner = 165 spy( 166 new ScheduledScenarioRunner( 167 SampleProfileSuite.LongIdleTest.class, 168 testScenario, 169 TimeUnit.SECONDS.toMillis(5), 170 true)); 171 runner.run(mRunNotifier); 172 // There should not be idle before teardown as the test should not have left itself enough 173 // time for that. 174 verify(runner, never()).performIdleBeforeTeardown(anyLong()); 175 } 176 177 /** Test that an over time test still idles until tne next scenario is supposed to begin. */ 178 @Test testOverTimeTest_idlesAfterTeardownUntilNextScenario()179 public void testOverTimeTest_idlesAfterTeardownUntilNextScenario() throws InitializationError { 180 // Set a over time test with a 5-second window that will idle until the end of the window is 181 // reached. 182 Scenario testScenario = 183 Scenario.newBuilder() 184 .setAt("00:00:00") 185 .setJourney(SampleProfileSuite.LongIdleTest.class.getName()) 186 .setAfterTest(AfterTest.STAY_IN_APP) 187 .build(); 188 ScheduledScenarioRunner runner = 189 spy( 190 new ScheduledScenarioRunner( 191 SampleProfileSuite.LongIdleTest.class, 192 testScenario, 193 TimeUnit.SECONDS.toMillis(5), 194 true)); 195 runner.run(mRunNotifier); 196 // Verify that it still idles until the next scenario; duration should be roughly equal to 197 // the leeway set in @{link ScheduledScenarioRunner}. 198 verify(runner, times(1)) 199 .performIdleBeforeNextScenario( 200 getWithinMarginMatcher( 201 ScheduledScenarioRunner.ENDTIME_LEEWAY_MS, 202 TIMEOUT_ERROR_MARGIN_MS)); 203 } 204 205 /** Test that a test set to stay in the app after the test idles after its @Test method. */ 206 @Test testRespectsAfterTestPolicy_stayInApp()207 public void testRespectsAfterTestPolicy_stayInApp() throws InitializationError { 208 // Set a passing test with a 5-second timeout that will idle after its @Test method and 209 // idle until the end of the timeout is reached. 210 long timeoutMs = TimeUnit.SECONDS.toMillis(5); 211 Scenario testScenario = 212 Scenario.newBuilder() 213 .setAt("00:00:00") 214 .setJourney(SampleProfileSuite.PassingTest.class.getName()) 215 .setAfterTest(AfterTest.STAY_IN_APP) 216 .build(); 217 ScheduledScenarioRunner runner = 218 spy( 219 new ScheduledScenarioRunner( 220 SampleProfileSuite.PassingTest.class, 221 testScenario, 222 timeoutMs, 223 true)); 224 runner.run(mRunNotifier); 225 // Idles before teardown; duration should be roughly equal to the timeout minus the leeway 226 // set in {@link ScheduledScenarioRunner}. 227 verify(runner, times(1)) 228 .performIdleBeforeTeardown( 229 getWithinMarginMatcher( 230 timeoutMs - ScheduledScenarioRunner.ENDTIME_LEEWAY_MS, 231 TIMEOUT_ERROR_MARGIN_MS)); 232 } 233 234 /** Test that a test set to exit the app after the test does not idle after its @Test method. */ 235 @Test testRespectsAfterTestPolicy_exit()236 public void testRespectsAfterTestPolicy_exit() throws InitializationError { 237 // Set a passing test with a 5-second timeout that does not idle after its @Test method and 238 // will idle until the end of the timeout is reached. 239 long timeoutMs = TimeUnit.SECONDS.toMillis(5); 240 Scenario testScenario = 241 Scenario.newBuilder() 242 .setAt("00:00:00") 243 .setJourney(SampleProfileSuite.PassingTest.class.getName()) 244 .setAfterTest(AfterTest.EXIT) 245 .build(); 246 ScheduledScenarioRunner runner = 247 spy( 248 new ScheduledScenarioRunner( 249 SampleProfileSuite.PassingTest.class, 250 testScenario, 251 timeoutMs, 252 true)); 253 runner.run(mRunNotifier); 254 // There should not be idle before teardown. 255 verify(runner, never()).performIdleBeforeTeardown(anyLong()); 256 // Idles before the next scenario; duration should be roughly equal to the timeout. 257 verify(runner, times(1)) 258 .performIdleBeforeNextScenario( 259 getWithinMarginMatcher(timeoutMs, TIMEOUT_ERROR_MARGIN_MS)); 260 } 261 262 /** Test that an ignored scenario still includes the timeout dictated in a profile. */ 263 @Test testIgnoredScenario_doesIdle()264 public void testIgnoredScenario_doesIdle() throws InitializationError, Exception { 265 long timeoutMs = TimeUnit.SECONDS.toMillis(5); 266 Scenario testScenario = 267 Scenario.newBuilder() 268 .setAt("00:00:00") 269 .setJourney(SampleProfileSuite.PassingTest.class.getName()) 270 .setAfterTest(AfterTest.EXIT) 271 .build(); 272 Bundle ignores = new Bundle(); 273 ignores.putString( 274 LongevityClassRunner.FILTER_OPTION, 275 SampleProfileSuite.PassingTest.class.getCanonicalName()); 276 ScheduledScenarioRunner runner = 277 spy( 278 new ScheduledScenarioRunner( 279 SampleProfileSuite.PassingTest.class, 280 testScenario, 281 timeoutMs, 282 true, 283 ignores)); 284 RunNotifier notifier = spy(new RunNotifier()); 285 RunListener listener = mock(RunListener.class); 286 notifier.addListener(listener); 287 runner.run(notifier); 288 // There should not be idle before teardown. 289 verify(runner, never()).performIdleBeforeTeardown(anyLong()); 290 // Ensure the test was ignored via listener. 291 verify(listener, times(1)).testIgnored(any()); 292 // Idles before the next scenario; duration should be roughly equal to the timeout. 293 verify(runner, times(1)) 294 .performIdleBeforeNextScenario( 295 getWithinMarginMatcher(timeoutMs, TIMEOUT_ERROR_MARGIN_MS)); 296 } 297 298 /** Test that the last test does not have idle after it, regardless of its AfterTest policy. */ 299 @Test testLastScenarioDoesNotIdle()300 public void testLastScenarioDoesNotIdle() throws InitializationError { 301 // Set a passing test with a 5-second timeout that is set to idle after its @Test method and 302 // but should not idle as it will be the last test in practice. 303 Scenario testScenario = 304 Scenario.newBuilder() 305 .setAt("00:00:00") 306 .setJourney(SampleProfileSuite.PassingTest.class.getName()) 307 .setAfterTest(AfterTest.STAY_IN_APP) 308 .build(); 309 ScheduledScenarioRunner runner = 310 spy( 311 new ScheduledScenarioRunner( 312 SampleProfileSuite.PassingTest.class, 313 testScenario, 314 TimeUnit.SECONDS.toMillis(5), 315 false)); 316 runner.run(mRunNotifier); 317 // There should not be idle of any form. 318 verify(runner, never()).performIdleBeforeTeardown(anyLong()); 319 verify(runner, never()).performIdleBeforeNextScenario(anyLong()); 320 } 321 322 323 /** Test that the "extras" in a scenario is properly registered before the test. */ 324 @Test testExtraArgs_registeredBeforeTest()325 public void testExtraArgs_registeredBeforeTest() throws Throwable { 326 Scenario testScenario = 327 Scenario.newBuilder() 328 .setAt("00:00:00") 329 .setJourney(ArgumentTest.class.getName()) 330 .setAfterTest(AfterTest.STAY_IN_APP) 331 .addExtras( 332 ExtraArg.newBuilder() 333 .setKey(ArgumentTest.TEST_ARG) 334 .setValue(ArgumentTest.TEST_ARG_OVERRIDE)) 335 .build(); 336 ScheduledScenarioRunner runner = 337 spy( 338 new ScheduledScenarioRunner( 339 ArgumentTest.class, 340 testScenario, 341 TimeUnit.SECONDS.toMillis(5), 342 false)); 343 runner.run(mRunNotifier); 344 verifyForAssertionFailures(mRunNotifier); 345 } 346 347 /** Test that the "extras" in a scenario is properly un-registered after the test. */ 348 @Test testExtraArgs_unregisteredAfterTest()349 public void testExtraArgs_unregisteredAfterTest() throws Throwable { 350 Bundle argsBeforeTest = InstrumentationRegistry.getArguments(); 351 Scenario testScenario = 352 Scenario.newBuilder() 353 .setAt("00:00:00") 354 .setJourney(ArgumentTest.class.getName()) 355 .setAfterTest(AfterTest.STAY_IN_APP) 356 .addExtras( 357 ExtraArg.newBuilder() 358 .setKey(ArgumentTest.TEST_ARG) 359 .setValue(ArgumentTest.TEST_ARG_OVERRIDE)) 360 .build(); 361 ScheduledScenarioRunner runner = 362 new ScheduledScenarioRunner( 363 ArgumentTest.class, testScenario, TimeUnit.SECONDS.toMillis(5), false); 364 runner.run(mRunNotifier); 365 Bundle argsAfterTest = InstrumentationRegistry.getArguments(); 366 Assert.assertTrue(bundlesContainSameStringKeyValuePairs(argsBeforeTest, argsAfterTest)); 367 } 368 369 /** 370 * Helper method to get an argument matcher that checks whether the input value is equal to 371 * expected value within a margin. 372 */ getWithinMarginMatcher(long expected, long margin)373 private long getWithinMarginMatcher(long expected, long margin) { 374 return longThat(duration -> abs(duration - expected) <= margin); 375 } 376 377 /** 378 * Verify that no test failure is fired because of an assertion failure in the stubbed methods. 379 * If the verfication fails, check whether it's due the injected assertions failing. If yes, 380 * throw that exception out; otherwise, throw the first exception. 381 */ verifyForAssertionFailures(final RunNotifier notifier)382 private void verifyForAssertionFailures(final RunNotifier notifier) throws Throwable { 383 try { 384 verify(notifier, never()).fireTestFailure(any()); 385 } catch (MockitoAssertionError e) { 386 ArgumentCaptor<Failure> failureCaptor = ArgumentCaptor.forClass(Failure.class); 387 verify(notifier, atLeastOnce()).fireTestFailure(failureCaptor.capture()); 388 List<Failure> failures = failureCaptor.getAllValues(); 389 // Go through the failures, look for an known failure case from the above exceptions 390 // and throw the exception in the first one out if any. 391 for (Failure failure : failures) { 392 if (failure.getException().getMessage().contains(ASSERTION_FAILURE_MESSAGE)) { 393 throw failure.getException(); 394 } 395 } 396 // Otherwise, throw the exception from the first failure reported. 397 throw failures.get(0).getException(); 398 } 399 } 400 401 /** 402 * Helper method to check whether two {@link Bundle}s are equal since the built-in {@code 403 * equals} is not properly overriden. 404 */ bundlesContainSameStringKeyValuePairs(Bundle b1, Bundle b2)405 private boolean bundlesContainSameStringKeyValuePairs(Bundle b1, Bundle b2) { 406 if (b1.size() != b2.size()) { 407 return false; 408 } 409 HashSet<String> allKeys = new HashSet<String>(b1.keySet()); 410 allKeys.addAll(b2.keySet()); 411 for (String key : allKeys) { 412 if (b1.getString(key) != null) { 413 // If key is in b1 and corresponds to a string, check whether this key corresponds 414 // to the same value in b2. 415 if (!b1.getString(key).equals(b2.getString(key))) { 416 return false; 417 } 418 } else if (b2.getString(key) != null) { 419 // Otherwise if b2 has a string at this key, return false since we know that b1 does 420 // not have a string at this key. 421 return false; 422 } 423 } 424 return true; 425 } 426 } 427