• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.server;
18 
19 import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig;
20 
21 import static com.android.dx.mockito.inline.extended.ExtendedMockito.doAnswer;
22 
23 import static com.google.common.truth.Truth.assertThat;
24 
25 import static org.junit.Assert.assertFalse;
26 import static org.mockito.ArgumentMatchers.anyBoolean;
27 import static org.mockito.ArgumentMatchers.anyInt;
28 import static org.mockito.ArgumentMatchers.anyLong;
29 import static org.mockito.ArgumentMatchers.anyString;
30 import static org.mockito.Mockito.never;
31 import static org.mockito.Mockito.reset;
32 import static org.mockito.Mockito.spy;
33 import static org.mockito.Mockito.verify;
34 import static org.mockito.Mockito.when;
35 
36 import android.Manifest;
37 import android.content.Context;
38 import android.content.pm.PackageInfo;
39 import android.content.pm.PackageManager;
40 import android.content.pm.VersionedPackage;
41 import android.content.rollback.PackageRollbackInfo;
42 import android.content.rollback.RollbackInfo;
43 import android.content.rollback.RollbackManager;
44 import android.crashrecovery.flags.Flags;
45 import android.net.ConnectivityModuleConnector;
46 import android.net.ConnectivityModuleConnector.ConnectivityModuleHealthListener;
47 import android.os.Handler;
48 import android.os.SystemProperties;
49 import android.os.test.TestLooper;
50 import android.platform.test.flag.junit.SetFlagsRule;
51 import android.provider.DeviceConfig;
52 import android.util.AtomicFile;
53 
54 import androidx.test.InstrumentationRegistry;
55 
56 import com.android.dx.mockito.inline.extended.ExtendedMockito;
57 import com.android.server.RescueParty.RescuePartyObserver;
58 import com.android.server.pm.ApexManager;
59 import com.android.server.rollback.RollbackPackageHealthObserver;
60 
61 import org.junit.After;
62 import org.junit.Before;
63 import org.junit.Rule;
64 import org.junit.Test;
65 import org.mockito.Answers;
66 import org.mockito.ArgumentCaptor;
67 import org.mockito.Captor;
68 import org.mockito.Mock;
69 import org.mockito.Mockito;
70 import org.mockito.MockitoAnnotations;
71 import org.mockito.MockitoSession;
72 import org.mockito.quality.Strictness;
73 import org.mockito.stubbing.Answer;
74 
75 import java.io.File;
76 import java.lang.reflect.Field;
77 import java.util.ArrayList;
78 import java.util.Collections;
79 import java.util.HashMap;
80 import java.util.List;
81 import java.util.Set;
82 import java.util.concurrent.TimeUnit;
83 import java.util.function.Consumer;
84 
85 /**
86  * Test CrashRecovery, integration tests that include PackageWatchdog, RescueParty and
87  * RollbackPackageHealthObserver
88  */
89 public class CrashRecoveryTest {
90     private static final String PROP_DEVICE_CONFIG_DISABLE_FLAG =
91             "persist.device_config.configuration.disable_rescue_party";
92 
93     private static final String APP_A = "com.package.a";
94     private static final String APP_B = "com.package.b";
95     private static final String APP_C = "com.package.c";
96     private static final long VERSION_CODE = 1L;
97     private static final long SHORT_DURATION = TimeUnit.SECONDS.toMillis(1);
98 
99     private static final RollbackInfo ROLLBACK_INFO_LOW = getRollbackInfo(APP_A, VERSION_CODE, 1,
100                 PackageManager.ROLLBACK_USER_IMPACT_LOW);
101     private static final RollbackInfo ROLLBACK_INFO_HIGH = getRollbackInfo(APP_B, VERSION_CODE, 2,
102             PackageManager.ROLLBACK_USER_IMPACT_HIGH);
103     private static final RollbackInfo ROLLBACK_INFO_MANUAL = getRollbackInfo(APP_C, VERSION_CODE, 3,
104             PackageManager.ROLLBACK_USER_IMPACT_ONLY_MANUAL);
105 
106     @Rule
107     public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();
108 
109     private final TestClock mTestClock = new TestClock();
110     private TestLooper mTestLooper;
111     private Context mSpyContext;
112     // Keep track of all created watchdogs to apply device config changes
113     private List<PackageWatchdog> mAllocatedWatchdogs;
114     @Mock
115     private ConnectivityModuleConnector mConnectivityModuleConnector;
116     @Mock
117     private PackageManager mMockPackageManager;
118     @Mock(answer = Answers.RETURNS_DEEP_STUBS)
119     private ApexManager mApexManager;
120     @Mock
121     RollbackManager mRollbackManager;
122     // Mock only sysprop apis
123     private PackageWatchdog.BootThreshold mSpyBootThreshold;
124     @Captor
125     private ArgumentCaptor<ConnectivityModuleHealthListener> mConnectivityModuleCallbackCaptor;
126     private MockitoSession mSession;
127     private HashMap<String, String> mSystemSettingsMap;
128     private HashMap<String, String> mCrashRecoveryPropertiesMap;
129 
130     @Before
setUp()131     public void setUp() throws Exception {
132         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
133         MockitoAnnotations.initMocks(this);
134         new File(InstrumentationRegistry.getContext().getFilesDir(),
135                 "package-watchdog.xml").delete();
136         adoptShellPermissions(Manifest.permission.READ_DEVICE_CONFIG,
137                 Manifest.permission.WRITE_DEVICE_CONFIG);
138         mTestLooper = new TestLooper();
139         mSpyContext = spy(InstrumentationRegistry.getContext());
140         when(mSpyContext.getPackageManager()).thenReturn(mMockPackageManager);
141         when(mMockPackageManager.getPackageInfo(anyString(), anyInt())).then(inv -> {
142             final PackageInfo res = new PackageInfo();
143             res.packageName = inv.getArgument(0);
144             res.setLongVersionCode(VERSION_CODE);
145             return res;
146         });
147         mSession = ExtendedMockito.mockitoSession()
148                 .initMocks(this)
149                 .strictness(Strictness.LENIENT)
150                 .spyStatic(SystemProperties.class)
151                 .spyStatic(RescueParty.class)
152                 .startMocking();
153         mSystemSettingsMap = new HashMap<>();
154 
155         // Mock SystemProperties setter and various getters
156         doAnswer((Answer<Void>) invocationOnMock -> {
157                     String key = invocationOnMock.getArgument(0);
158                     String value = invocationOnMock.getArgument(1);
159 
160                     mSystemSettingsMap.put(key, value);
161                     return null;
162                 }
163         ).when(() -> SystemProperties.set(anyString(), anyString()));
164 
165         doAnswer((Answer<Integer>) invocationOnMock -> {
166                     String key = invocationOnMock.getArgument(0);
167                     int defaultValue = invocationOnMock.getArgument(1);
168 
169                     String storedValue = mSystemSettingsMap.get(key);
170                     return storedValue == null ? defaultValue : Integer.parseInt(storedValue);
171                 }
172         ).when(() -> SystemProperties.getInt(anyString(), anyInt()));
173 
174         doAnswer((Answer<Long>) invocationOnMock -> {
175                     String key = invocationOnMock.getArgument(0);
176                     long defaultValue = invocationOnMock.getArgument(1);
177 
178                     String storedValue = mSystemSettingsMap.get(key);
179                     return storedValue == null ? defaultValue : Long.parseLong(storedValue);
180                 }
181         ).when(() -> SystemProperties.getLong(anyString(), anyLong()));
182 
183         doAnswer((Answer<Boolean>) invocationOnMock -> {
184                     String key = invocationOnMock.getArgument(0);
185                     boolean defaultValue = invocationOnMock.getArgument(1);
186 
187                     String storedValue = mSystemSettingsMap.get(key);
188                     return storedValue == null ? defaultValue : Boolean.parseBoolean(storedValue);
189                 }
190         ).when(() -> SystemProperties.getBoolean(anyString(), anyBoolean()));
191 
192         SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true));
193         SystemProperties.set(PROP_DEVICE_CONFIG_DISABLE_FLAG, Boolean.toString(false));
194 
195         DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
196                 PackageWatchdog.PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED,
197                 Boolean.toString(true), false);
198 
199         DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
200                 PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
201                 Integer.toString(PackageWatchdog.DEFAULT_TRIGGER_FAILURE_COUNT), false);
202 
203         mAllocatedWatchdogs = new ArrayList<>();
204         RescuePartyObserver.reset();
205     }
206 
207     @After
tearDown()208     public void tearDown() throws Exception {
209         dropShellPermissions();
210         mSession.finishMocking();
211         // Clean up listeners since too many listeners will delay notifications significantly
212         for (PackageWatchdog watchdog : mAllocatedWatchdogs) {
213             watchdog.removePropertyChangedListener();
214         }
215         mAllocatedWatchdogs.clear();
216     }
217 
218     @Test
testBootLoopWithRescueParty()219     public void testBootLoopWithRescueParty() throws Exception {
220         PackageWatchdog watchdog = createWatchdog();
221         RescuePartyObserver rescuePartyObserver = setUpRescuePartyObserver(watchdog);
222 
223         verify(rescuePartyObserver, never()).executeBootLoopMitigation(1);
224 
225         for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
226             watchdog.noteBoot();
227         }
228 
229         verify(rescuePartyObserver).executeBootLoopMitigation(1);
230         verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);
231 
232         watchdog.noteBoot();
233 
234         verify(rescuePartyObserver).executeBootLoopMitigation(2);
235         verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);
236 
237         watchdog.noteBoot();
238 
239         verify(rescuePartyObserver).executeBootLoopMitigation(3);
240         verify(rescuePartyObserver, never()).executeBootLoopMitigation(4);
241 
242         watchdog.noteBoot();
243 
244         verify(rescuePartyObserver).executeBootLoopMitigation(4);
245         verify(rescuePartyObserver, never()).executeBootLoopMitigation(5);
246 
247         watchdog.noteBoot();
248 
249         verify(rescuePartyObserver).executeBootLoopMitigation(5);
250         verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);
251 
252         watchdog.noteBoot();
253 
254         verify(rescuePartyObserver).executeBootLoopMitigation(6);
255         verify(rescuePartyObserver, never()).executeBootLoopMitigation(7);
256     }
257 
258     @Test
testBootLoopWithRollbackPackageHealthObserver()259     public void testBootLoopWithRollbackPackageHealthObserver() throws Exception {
260         PackageWatchdog watchdog = createWatchdog();
261         RollbackPackageHealthObserver rollbackObserver =
262                 setUpRollbackPackageHealthObserver(watchdog);
263 
264         verify(rollbackObserver, never()).executeBootLoopMitigation(1);
265 
266         for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
267             watchdog.noteBoot();
268         }
269 
270         verify(rollbackObserver).executeBootLoopMitigation(1);
271         verify(rollbackObserver, never()).executeBootLoopMitigation(2);
272 
273         // Update the list of available rollbacks after executing bootloop mitigation once
274         when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_HIGH,
275                 ROLLBACK_INFO_MANUAL));
276 
277         watchdog.noteBoot();
278 
279         verify(rollbackObserver).executeBootLoopMitigation(2);
280         verify(rollbackObserver, never()).executeBootLoopMitigation(3);
281 
282         // Update the list of available rollbacks after executing bootloop mitigation once
283         when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_MANUAL));
284 
285         watchdog.noteBoot();
286 
287         verify(rollbackObserver, never()).executeBootLoopMitigation(3);
288     }
289 
290     @Test
testBootLoopWithRescuePartyAndRollbackPackageHealthObserver()291     public void testBootLoopWithRescuePartyAndRollbackPackageHealthObserver() throws Exception {
292         PackageWatchdog watchdog = createWatchdog();
293         RescuePartyObserver rescuePartyObserver = setUpRescuePartyObserver(watchdog);
294         RollbackPackageHealthObserver rollbackObserver =
295                 setUpRollbackPackageHealthObserver(watchdog);
296 
297         verify(rescuePartyObserver, never()).executeBootLoopMitigation(1);
298         verify(rollbackObserver, never()).executeBootLoopMitigation(1);
299         for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
300             watchdog.noteBoot();
301         }
302         verify(rescuePartyObserver).executeBootLoopMitigation(1);
303         verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);
304         verify(rollbackObserver, never()).executeBootLoopMitigation(1);
305 
306         watchdog.noteBoot();
307 
308         verify(rescuePartyObserver).executeBootLoopMitigation(2);
309         verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);
310         verify(rollbackObserver, never()).executeBootLoopMitigation(2);
311 
312         watchdog.noteBoot();
313 
314         verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);
315         verify(rollbackObserver).executeBootLoopMitigation(1);
316         verify(rollbackObserver, never()).executeBootLoopMitigation(2);
317         // Update the list of available rollbacks after executing bootloop mitigation once
318         when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_HIGH,
319                 ROLLBACK_INFO_MANUAL));
320 
321         watchdog.noteBoot();
322 
323         verify(rescuePartyObserver).executeBootLoopMitigation(3);
324         verify(rescuePartyObserver, never()).executeBootLoopMitigation(4);
325         verify(rollbackObserver, never()).executeBootLoopMitigation(2);
326 
327         watchdog.noteBoot();
328 
329         verify(rescuePartyObserver).executeBootLoopMitigation(4);
330         verify(rescuePartyObserver, never()).executeBootLoopMitigation(5);
331         verify(rollbackObserver, never()).executeBootLoopMitigation(2);
332 
333         watchdog.noteBoot();
334 
335         verify(rescuePartyObserver).executeBootLoopMitigation(5);
336         verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);
337         verify(rollbackObserver, never()).executeBootLoopMitigation(2);
338 
339         watchdog.noteBoot();
340 
341         verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);
342         verify(rollbackObserver).executeBootLoopMitigation(2);
343         verify(rollbackObserver, never()).executeBootLoopMitigation(3);
344         // Update the list of available rollbacks after executing bootloop mitigation
345         when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_MANUAL));
346 
347         watchdog.noteBoot();
348 
349         verify(rescuePartyObserver).executeBootLoopMitigation(6);
350         verify(rescuePartyObserver, never()).executeBootLoopMitigation(7);
351         verify(rollbackObserver, never()).executeBootLoopMitigation(3);
352 
353         moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);
354         Mockito.reset(rescuePartyObserver);
355 
356         for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
357             watchdog.noteBoot();
358         }
359         verify(rescuePartyObserver).executeBootLoopMitigation(1);
360         verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);
361     }
362 
setUpRollbackPackageHealthObserver(PackageWatchdog watchdog)363     RollbackPackageHealthObserver setUpRollbackPackageHealthObserver(PackageWatchdog watchdog) {
364         RollbackPackageHealthObserver rollbackObserver =
365                 spy(new RollbackPackageHealthObserver(mSpyContext, mApexManager));
366         when(mSpyContext.getSystemService(RollbackManager.class)).thenReturn(mRollbackManager);
367         when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_LOW,
368                 ROLLBACK_INFO_HIGH, ROLLBACK_INFO_MANUAL));
369         when(mSpyContext.getPackageManager()).thenReturn(mMockPackageManager);
370 
371         watchdog.registerHealthObserver(rollbackObserver);
372         return rollbackObserver;
373     }
374 
setUpRescuePartyObserver(PackageWatchdog watchdog)375     RescuePartyObserver setUpRescuePartyObserver(PackageWatchdog watchdog) {
376         setCrashRecoveryPropRescueBootCount(0);
377         RescuePartyObserver rescuePartyObserver = spy(RescuePartyObserver.getInstance(mSpyContext));
378         assertFalse(RescueParty.isRebootPropertySet());
379         watchdog.registerHealthObserver(rescuePartyObserver);
380         return rescuePartyObserver;
381     }
382 
getRollbackInfo(String packageName, long versionCode, int rollbackId, int rollbackUserImpact)383     private static RollbackInfo getRollbackInfo(String packageName, long versionCode,
384             int rollbackId, int rollbackUserImpact) {
385         VersionedPackage appFrom = new VersionedPackage(packageName, versionCode + 1);
386         VersionedPackage appTo = new VersionedPackage(packageName, versionCode);
387         PackageRollbackInfo packageRollbackInfo = new PackageRollbackInfo(appFrom, appTo, null,
388                 null, false, false, null);
389         RollbackInfo rollbackInfo = new RollbackInfo(rollbackId, List.of(packageRollbackInfo),
390                 false, null, 111, rollbackUserImpact);
391         return rollbackInfo;
392     }
393 
adoptShellPermissions(String... permissions)394     private void adoptShellPermissions(String... permissions) {
395         androidx.test.platform.app.InstrumentationRegistry
396                 .getInstrumentation()
397                 .getUiAutomation()
398                 .adoptShellPermissionIdentity(permissions);
399     }
400 
dropShellPermissions()401     private void dropShellPermissions() {
402         androidx.test.platform.app.InstrumentationRegistry
403                 .getInstrumentation()
404                 .getUiAutomation()
405                 .dropShellPermissionIdentity();
406     }
407 
408 
createWatchdog()409     private PackageWatchdog createWatchdog() {
410         return createWatchdog(new TestController(), true /* withPackagesReady */);
411     }
412 
createWatchdog(TestController controller, boolean withPackagesReady)413     private PackageWatchdog createWatchdog(TestController controller, boolean withPackagesReady) {
414         AtomicFile policyFile =
415                 new AtomicFile(new File(mSpyContext.getFilesDir(), "package-watchdog.xml"));
416         Handler handler = new Handler(mTestLooper.getLooper());
417         PackageWatchdog watchdog =
418                 new PackageWatchdog(mSpyContext, policyFile, handler, handler, controller,
419                         mConnectivityModuleConnector, mTestClock);
420         mockCrashRecoveryProperties(watchdog);
421 
422         // Verify controller is not automatically started
423         assertThat(controller.mIsEnabled).isFalse();
424         if (withPackagesReady) {
425             // Only capture the NetworkStack callback for the latest registered watchdog
426             reset(mConnectivityModuleConnector);
427             watchdog.onPackagesReady();
428             // Verify controller by default is started when packages are ready
429             assertThat(controller.mIsEnabled).isTrue();
430 
431             verify(mConnectivityModuleConnector).registerHealthListener(
432                     mConnectivityModuleCallbackCaptor.capture());
433         }
434         mAllocatedWatchdogs.add(watchdog);
435         return watchdog;
436     }
437 
438     // Mock CrashRecoveryProperties as they cannot be accessed due to SEPolicy restrictions
mockCrashRecoveryProperties(PackageWatchdog watchdog)439     private void mockCrashRecoveryProperties(PackageWatchdog watchdog) {
440         mCrashRecoveryPropertiesMap = new HashMap<>();
441 
442         // mock properties in RescueParty
443         try {
444 
445             doAnswer((Answer<Boolean>) invocationOnMock -> {
446                 String storedValue = mCrashRecoveryPropertiesMap
447                         .getOrDefault("crashrecovery.attempting_factory_reset", "false");
448                 return Boolean.parseBoolean(storedValue);
449             }).when(() -> RescueParty.isFactoryResetPropertySet());
450             doAnswer((Answer<Void>) invocationOnMock -> {
451                 boolean value = invocationOnMock.getArgument(0);
452                 mCrashRecoveryPropertiesMap.put("crashrecovery.attempting_factory_reset",
453                         Boolean.toString(value));
454                 return null;
455             }).when(() -> RescueParty.setFactoryResetProperty(anyBoolean()));
456 
457             doAnswer((Answer<Boolean>) invocationOnMock -> {
458                 String storedValue = mCrashRecoveryPropertiesMap
459                         .getOrDefault("crashrecovery.attempting_reboot", "false");
460                 return Boolean.parseBoolean(storedValue);
461             }).when(() -> RescueParty.isRebootPropertySet());
462             doAnswer((Answer<Void>) invocationOnMock -> {
463                 boolean value = invocationOnMock.getArgument(0);
464                 setCrashRecoveryPropAttemptingReboot(value);
465                 return null;
466             }).when(() -> RescueParty.setRebootProperty(anyBoolean()));
467 
468             doAnswer((Answer<Long>) invocationOnMock -> {
469                 String storedValue = mCrashRecoveryPropertiesMap
470                         .getOrDefault("persist.crashrecovery.last_factory_reset", "0");
471                 return Long.parseLong(storedValue);
472             }).when(() -> RescueParty.getLastFactoryResetTimeMs());
473             doAnswer((Answer<Void>) invocationOnMock -> {
474                 long value = invocationOnMock.getArgument(0);
475                 setCrashRecoveryPropLastFactoryReset(value);
476                 return null;
477             }).when(() -> RescueParty.setLastFactoryResetTimeMs(anyLong()));
478 
479             doAnswer((Answer<Integer>) invocationOnMock -> {
480                 String storedValue = mCrashRecoveryPropertiesMap
481                         .getOrDefault("crashrecovery.max_rescue_level_attempted", "0");
482                 return Integer.parseInt(storedValue);
483             }).when(() -> RescueParty.getMaxRescueLevelAttempted());
484             doAnswer((Answer<Void>) invocationOnMock -> {
485                 int value = invocationOnMock.getArgument(0);
486                 mCrashRecoveryPropertiesMap.put("crashrecovery.max_rescue_level_attempted",
487                         Integer.toString(value));
488                 return null;
489             }).when(() -> RescueParty.setMaxRescueLevelAttempted(anyInt()));
490 
491         } catch (Exception e) {
492             // tests will fail, just printing the error
493             System.out.println("Error while mocking crashrecovery properties " + e.getMessage());
494         }
495 
496         try {
497             mSpyBootThreshold = spy(watchdog.new BootThreshold(
498                     PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
499                     PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
500 
501             doAnswer((Answer<Integer>) invocationOnMock -> {
502                 String storedValue = mCrashRecoveryPropertiesMap
503                         .getOrDefault("crashrecovery.rescue_boot_count", "0");
504                 return Integer.parseInt(storedValue);
505             }).when(mSpyBootThreshold).getCount();
506             doAnswer((Answer<Void>) invocationOnMock -> {
507                 int count = invocationOnMock.getArgument(0);
508                 mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_count",
509                         Integer.toString(count));
510                 return null;
511             }).when(mSpyBootThreshold).setCount(anyInt());
512 
513             doAnswer((Answer<Integer>) invocationOnMock -> {
514                 String storedValue = mCrashRecoveryPropertiesMap
515                         .getOrDefault("crashrecovery.boot_mitigation_count", "0");
516                 return Integer.parseInt(storedValue);
517             }).when(mSpyBootThreshold).getMitigationCount();
518             doAnswer((Answer<Void>) invocationOnMock -> {
519                 int count = invocationOnMock.getArgument(0);
520                 mCrashRecoveryPropertiesMap.put("crashrecovery.boot_mitigation_count",
521                         Integer.toString(count));
522                 return null;
523             }).when(mSpyBootThreshold).setMitigationCount(anyInt());
524 
525             doAnswer((Answer<Long>) invocationOnMock -> {
526                 String storedValue = mCrashRecoveryPropertiesMap
527                         .getOrDefault("crashrecovery.rescue_boot_start", "0");
528                 return Long.parseLong(storedValue);
529             }).when(mSpyBootThreshold).getStart();
530             doAnswer((Answer<Void>) invocationOnMock -> {
531                 long count = invocationOnMock.getArgument(0);
532                 mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_start",
533                         Long.toString(count));
534                 return null;
535             }).when(mSpyBootThreshold).setStart(anyLong());
536 
537             doAnswer((Answer<Long>) invocationOnMock -> {
538                 String storedValue = mCrashRecoveryPropertiesMap
539                         .getOrDefault("crashrecovery.boot_mitigation_start", "0");
540                 return Long.parseLong(storedValue);
541             }).when(mSpyBootThreshold).getMitigationStart();
542             doAnswer((Answer<Void>) invocationOnMock -> {
543                 long count = invocationOnMock.getArgument(0);
544                 mCrashRecoveryPropertiesMap.put("crashrecovery.boot_mitigation_start",
545                         Long.toString(count));
546                 return null;
547             }).when(mSpyBootThreshold).setMitigationStart(anyLong());
548 
549             Field mBootThresholdField = watchdog.getClass().getDeclaredField("mBootThreshold");
550             mBootThresholdField.setAccessible(true);
551             mBootThresholdField.set(watchdog, mSpyBootThreshold);
552         } catch (Exception e) {
553             // tests will fail, just printing the error
554             System.out.println("Error detected while spying BootThreshold" + e.getMessage());
555         }
556     }
557 
setCrashRecoveryPropRescueBootCount(int count)558     private void setCrashRecoveryPropRescueBootCount(int count) {
559         mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_count",
560                 Integer.toString(count));
561     }
562 
setCrashRecoveryPropAttemptingReboot(boolean value)563     private void setCrashRecoveryPropAttemptingReboot(boolean value) {
564         mCrashRecoveryPropertiesMap.put("crashrecovery.attempting_reboot",
565                 Boolean.toString(value));
566     }
567 
setCrashRecoveryPropLastFactoryReset(long value)568     private void setCrashRecoveryPropLastFactoryReset(long value) {
569         mCrashRecoveryPropertiesMap.put("persist.crashrecovery.last_factory_reset",
570                 Long.toString(value));
571     }
572 
573     private static class TestController extends ExplicitHealthCheckController {
TestController()574         TestController() {
575             super(null /* controller */);
576         }
577 
578         private boolean mIsEnabled;
579         private List<String> mSupportedPackages = new ArrayList<>();
580         private List<String> mRequestedPackages = new ArrayList<>();
581         private Consumer<List<PackageConfig>> mSupportedConsumer;
582         private List<Set> mSyncRequests = new ArrayList<>();
583 
584         @Override
setEnabled(boolean enabled)585         public void setEnabled(boolean enabled) {
586             mIsEnabled = enabled;
587             if (!mIsEnabled) {
588                 mSupportedPackages.clear();
589             }
590         }
591 
592         @Override
setCallbacks(Consumer<String> passedConsumer, Consumer<List<PackageConfig>> supportedConsumer, Runnable notifySyncRunnable)593         public void setCallbacks(Consumer<String> passedConsumer,
594                 Consumer<List<PackageConfig>> supportedConsumer, Runnable notifySyncRunnable) {
595             mSupportedConsumer = supportedConsumer;
596         }
597 
598         @Override
syncRequests(Set<String> packages)599         public void syncRequests(Set<String> packages) {
600             mSyncRequests.add(packages);
601             mRequestedPackages.clear();
602             if (mIsEnabled) {
603                 packages.retainAll(mSupportedPackages);
604                 mRequestedPackages.addAll(packages);
605                 List<PackageConfig> packageConfigs = new ArrayList<>();
606                 for (String packageName: packages) {
607                     packageConfigs.add(new PackageConfig(packageName, SHORT_DURATION));
608                 }
609                 mSupportedConsumer.accept(packageConfigs);
610             } else {
611                 mSupportedConsumer.accept(Collections.emptyList());
612             }
613         }
614     }
615 
616     private static class TestClock implements PackageWatchdog.SystemClock {
617         // Note 0 is special to the internal clock of PackageWatchdog. We need to start from
618         // a non-zero value in order not to disrupt the logic of PackageWatchdog.
619         private long mUpTimeMillis = 1;
620         @Override
uptimeMillis()621         public long uptimeMillis() {
622             return mUpTimeMillis;
623         }
moveTimeForward(long milliSeconds)624         public void moveTimeForward(long milliSeconds) {
625             mUpTimeMillis += milliSeconds;
626         }
627     }
628 
moveTimeForwardAndDispatch(long milliSeconds)629     private void moveTimeForwardAndDispatch(long milliSeconds) {
630         // Exhaust all due runnables now which shouldn't be executed after time-leap
631         mTestLooper.dispatchAll();
632         mTestClock.moveTimeForward(milliSeconds);
633         mTestLooper.moveTimeForward(milliSeconds);
634         mTestLooper.dispatchAll();
635     }
636 }
637