• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.invoker;
17 
18 import static org.junit.Assert.assertEquals;
19 import static org.junit.Assert.assertFalse;
20 import static org.junit.Assert.assertTrue;
21 import static org.junit.Assert.fail;
22 import static org.mockito.Mockito.atLeast;
23 import static org.mockito.Mockito.calls;
24 import static org.mockito.Mockito.doReturn;
25 import static org.mockito.Mockito.doThrow;
26 import static org.mockito.Mockito.inOrder;
27 import static org.mockito.Mockito.lenient;
28 import static org.mockito.Mockito.mock;
29 import static org.mockito.Mockito.never;
30 import static org.mockito.Mockito.times;
31 import static org.mockito.Mockito.verify;
32 import static org.mockito.Mockito.when;
33 
34 import com.android.tradefed.build.BuildInfo;
35 import com.android.tradefed.build.BuildRetrievalError;
36 import com.android.tradefed.build.IBuildInfo;
37 import com.android.tradefed.build.IBuildInfo.BuildInfoProperties;
38 import com.android.tradefed.build.IBuildProvider;
39 import com.android.tradefed.build.IDeviceBuildInfo;
40 import com.android.tradefed.build.IDeviceBuildProvider;
41 import com.android.tradefed.command.CommandOptions;
42 import com.android.tradefed.command.CommandRunner.ExitCode;
43 import com.android.tradefed.command.FatalHostError;
44 import com.android.tradefed.command.remote.DeviceDescriptor;
45 import com.android.tradefed.config.Configuration;
46 import com.android.tradefed.config.ConfigurationDef;
47 import com.android.tradefed.config.ConfigurationException;
48 import com.android.tradefed.config.DeviceConfigurationHolder;
49 import com.android.tradefed.config.GlobalConfiguration;
50 import com.android.tradefed.config.IConfiguration;
51 import com.android.tradefed.config.IConfigurationFactory;
52 import com.android.tradefed.config.IDeviceConfiguration;
53 import com.android.tradefed.config.IGlobalConfiguration;
54 import com.android.tradefed.config.Option;
55 import com.android.tradefed.config.OptionSetter;
56 import com.android.tradefed.device.DeviceAllocationState;
57 import com.android.tradefed.device.DeviceNotAvailableException;
58 import com.android.tradefed.device.IDeviceRecovery;
59 import com.android.tradefed.device.ITestDevice;
60 import com.android.tradefed.device.ITestDevice.RecoveryMode;
61 import com.android.tradefed.device.StubDevice;
62 import com.android.tradefed.device.TestDeviceOptions;
63 import com.android.tradefed.device.TestDeviceState;
64 import com.android.tradefed.device.metric.BaseDeviceMetricCollector;
65 import com.android.tradefed.device.metric.DeviceMetricData;
66 import com.android.tradefed.device.metric.IMetricCollector;
67 import com.android.tradefed.error.HarnessRuntimeException;
68 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey;
69 import com.android.tradefed.invoker.shard.IShardHelper;
70 import com.android.tradefed.invoker.shard.ShardHelper;
71 import com.android.tradefed.log.ILeveledLogOutput;
72 import com.android.tradefed.log.ILogRegistry;
73 import com.android.tradefed.log.ITestLogger;
74 import com.android.tradefed.metrics.proto.MetricMeasurement.Measurements;
75 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
76 import com.android.tradefed.postprocessor.BasePostProcessor;
77 import com.android.tradefed.result.ActionInProgress;
78 import com.android.tradefed.result.ByteArrayInputStreamSource;
79 import com.android.tradefed.result.FailureDescription;
80 import com.android.tradefed.result.ILogSaver;
81 import com.android.tradefed.result.ILogSaverListener;
82 import com.android.tradefed.result.ITestInvocationListener;
83 import com.android.tradefed.result.ITestSummaryListener;
84 import com.android.tradefed.result.InputStreamSource;
85 import com.android.tradefed.result.InvocationStatus;
86 import com.android.tradefed.result.LogDataType;
87 import com.android.tradefed.result.LogFile;
88 import com.android.tradefed.result.TestSummary;
89 import com.android.tradefed.result.error.InfraErrorIdentifier;
90 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
91 import com.android.tradefed.targetprep.BuildError;
92 import com.android.tradefed.targetprep.ITargetCleaner;
93 import com.android.tradefed.targetprep.ITargetPreparer;
94 import com.android.tradefed.targetprep.TargetSetupError;
95 import com.android.tradefed.testtype.IDeviceTest;
96 import com.android.tradefed.testtype.IInvocationContextReceiver;
97 import com.android.tradefed.testtype.IRemoteTest;
98 import com.android.tradefed.testtype.IShardableTest;
99 import com.android.tradefed.util.keystore.IKeyStoreClient;
100 
101 import org.junit.Before;
102 import org.junit.BeforeClass;
103 import org.junit.Ignore;
104 import org.junit.Test;
105 import org.junit.runner.RunWith;
106 import org.junit.runners.JUnit4;
107 import org.mockito.ArgumentCaptor;
108 import org.mockito.Captor;
109 import org.mockito.InOrder;
110 import org.mockito.Mock;
111 import org.mockito.Mockito;
112 import org.mockito.MockitoAnnotations;
113 
114 import java.io.IOException;
115 import java.io.InputStream;
116 import java.util.ArrayList;
117 import java.util.Collections;
118 import java.util.HashMap;
119 import java.util.HashSet;
120 import java.util.LinkedHashMap;
121 import java.util.List;
122 import java.util.Map;
123 
124 /** Unit tests for {@link TestInvocation}. */
125 @SuppressWarnings("MustBeClosedChecker")
126 @RunWith(JUnit4.class)
127 public class TestInvocationTest {
128 
129     private static final String SERIAL = "serial";
130     private static final Map<String, String> EMPTY_MAP = Collections.emptyMap();
131     private static final String PATH = "path";
132     private static final String URL = "url";
133     private static final TestSummary mSummary = new TestSummary("http://www.url.com/report.txt");
134     private static final String LOGCAT_NAME_ERROR =
135             TestInvocation.getDeviceLogName(TestInvocation.Stage.ERROR);
136     private static final String LOGCAT_NAME_SETUP =
137             TestInvocation.getDeviceLogName(TestInvocation.Stage.SETUP);
138     private static final String LOGCAT_NAME_TEST =
139             TestInvocation.getDeviceLogName(TestInvocation.Stage.TEST);
140     private static final String LOGCAT_NAME_TEARDOWN =
141             TestInvocation.getDeviceLogName(TestInvocation.Stage.TEARDOWN);
142     private static final String CONFIG_LOG_NAME = TestInvocation.TRADEFED_CONFIG_NAME;
143     /** The {@link TestInvocation} under test, with all dependencies mocked out */
144     private TestInvocation mTestInvocation;
145 
146     private FailureStatus mExceptedStatus = null;
147     private boolean mShardingEarlyFailure = false;
148 
149     private IConfiguration mStubConfiguration;
150     private IConfiguration mStubMultiConfiguration;
151     private IGlobalConfiguration mGlobalConfiguration;
152 
153     private IInvocationContext mStubInvocationMetadata;
154 
155     // Log streams, they need a new instance per test
156     private InputStreamSource mLogcatSetupSource =
157             new ByteArrayInputStreamSource("logcat_setup".getBytes());
158     private InputStreamSource mLogcatTestSource =
159             new ByteArrayInputStreamSource("logcat_test".getBytes());
160     private InputStreamSource mLogcatTeardownSource =
161             new ByteArrayInputStreamSource("logcat_test".getBytes());
162     private InputStreamSource mHostLogSource =
163             new ByteArrayInputStreamSource("host_log".getBytes());
164 
165     // The mock objects.
166     @Mock ITestDevice mMockDevice;
167     @Mock ITargetPreparer mMockPreparer;
168     @Mock IBuildProvider mMockBuildProvider;
169     @Mock IBuildInfo mMockBuildInfo;
170     @Mock ITestInvocationListener mMockTestListener;
171     @Mock ITestSummaryListener mMockSummaryListener;
172     @Mock ILeveledLogOutput mMockLogger;
173     @Mock ILogSaver mMockLogSaver;
174     @Mock IDeviceRecovery mMockRecovery;
175     @Captor ArgumentCaptor<List<TestSummary>> mUriCapture;
176     @Mock ILogRegistry mMockLogRegistry;
177     @Mock IConfigurationFactory mMockConfigFactory;
178     @Mock IRescheduler mockRescheduler;
179     @Mock DeviceDescriptor mFakeDescriptor;
180 
181     // inOrder verifiers
182     private InOrder mInOrderSummaryListener;
183     private InOrder mInOrderTestListener;
184     private InOrder mInOrderLogSaver;
185 
186     private InOrder mTestOrder;
187 
188     @BeforeClass
setUpClass()189     public static void setUpClass() throws Exception {
190         try {
191             GlobalConfiguration.createGlobalConfiguration(new String[] {"empty"});
192         } catch (IllegalStateException e) {
193             // Avoid exception in case of multi-init
194         }
195     }
196 
197     @SuppressWarnings("unchecked")
198     @Before
setUp()199     public void setUp() throws Exception {
200         MockitoAnnotations.initMocks(this);
201         mInOrderSummaryListener = inOrder(mMockSummaryListener);
202         mInOrderTestListener = inOrder(mMockTestListener);
203         mInOrderLogSaver = inOrder(mMockLogSaver);
204 
205         mStubConfiguration =
206                 new Configuration("foo", "bar") {
207                     @Override
208                     public IConfiguration partialDeepClone(
209                             List<String> objectToDeepClone, IKeyStoreClient client)
210                             throws ConfigurationException {
211                         return new Configuration(this.getName(), this.getDescription());
212                     }
213                 };
214         OptionSetter setter = new OptionSetter(mStubConfiguration.getCommandOptions());
215         setter.setOptionValue("report-passed-tests", "false");
216         mStubMultiConfiguration = new Configuration("foo", "bar");
217 
218         mStubConfiguration.setDeviceRecovery(mMockRecovery);
219         mStubConfiguration.setTargetPreparer(mMockPreparer);
220         mStubConfiguration.setBuildProvider(mMockBuildProvider);
221 
222         lenient().when(mMockPreparer.isDisabled()).thenReturn(false);
223         lenient().when(mMockPreparer.isTearDownDisabled()).thenReturn(false);
224 
225         List<IDeviceConfiguration> deviceConfigs = new ArrayList<IDeviceConfiguration>();
226         IDeviceConfiguration device1 =
227                 new DeviceConfigurationHolder(ConfigurationDef.DEFAULT_DEVICE_NAME);
228         device1.addSpecificConfig(mMockRecovery);
229         device1.addSpecificConfig(mMockPreparer);
230         device1.addSpecificConfig(mMockBuildProvider);
231         deviceConfigs.add(device1);
232         mStubMultiConfiguration.setDeviceConfigList(deviceConfigs);
233 
234         mStubConfiguration.setLogSaver(mMockLogSaver);
235         mStubMultiConfiguration.setLogSaver(mMockLogSaver);
236 
237         List<ITestInvocationListener> listenerList = new ArrayList<ITestInvocationListener>(1);
238         listenerList.add(mMockTestListener);
239         listenerList.add(mMockSummaryListener);
240         mStubConfiguration.setTestInvocationListeners(listenerList);
241         mStubMultiConfiguration.setTestInvocationListeners(listenerList);
242 
243         mStubConfiguration.setLogOutput(mMockLogger);
244         mStubMultiConfiguration.setLogOutput(mMockLogger);
245         lenient().when(mMockDevice.getSerialNumber()).thenReturn(SERIAL);
246         lenient().when(mMockDevice.getIDevice()).thenReturn(null);
247         lenient().when(mMockDevice.getBattery()).thenReturn(null);
248         lenient().when(mMockDevice.getDeviceState()).thenReturn(TestDeviceState.NOT_AVAILABLE);
249         mFakeDescriptor =
250                 new DeviceDescriptor(
251                         SERIAL,
252                         false,
253                         DeviceAllocationState.Available,
254                         "unknown",
255                         "unknown",
256                         "unknown",
257                         "unknown",
258                         "unknown");
259         lenient().when(mMockDevice.getDeviceDescriptor()).thenReturn(mFakeDescriptor);
260 
261         lenient().when(mMockBuildInfo.getBuildId()).thenReturn("1");
262         lenient().when(mMockBuildInfo.getBuildAttributes()).thenReturn(EMPTY_MAP);
263         lenient().when(mMockBuildInfo.getBuildBranch()).thenReturn("branch");
264         lenient().when(mMockBuildInfo.getBuildFlavor()).thenReturn("flavor");
265         lenient().when(mMockBuildInfo.getProperties()).thenReturn(new HashSet<>());
266 
267         mStubInvocationMetadata = new InvocationContext();
268         mStubInvocationMetadata.addAllocatedDevice(
269                 ConfigurationDef.DEFAULT_DEVICE_NAME, mMockDevice);
270         mStubInvocationMetadata.addDeviceBuildInfo(
271                 ConfigurationDef.DEFAULT_DEVICE_NAME, mMockBuildInfo);
272         TestDeviceOptions options = mock(TestDeviceOptions.class);
273         when(options.shouldSkipTearDown()).thenReturn(false);
274         when(mMockDevice.getOptions()).thenReturn(options);
275 
276         // create the BaseTestInvocation to test
277         mTestInvocation =
278                 new TestInvocation() {
279                     @Override
280                     ILogRegistry getLogRegistry() {
281                         return mMockLogRegistry;
282                     }
283 
284                     @Override
285                     public IInvocationExecution createInvocationExec(RunMode mode) {
286                         return new InvocationExecution() {
287                             @Override
288                             protected IShardHelper createShardHelper() {
289                                 return new ShardHelper() {
290                                     @Override
291                                     protected IGlobalConfiguration getGlobalConfiguration() {
292                                         return mGlobalConfiguration;
293                                     }
294                                 };
295                             }
296 
297                             @Override
298                             protected String getAdbVersion() {
299                                 return null;
300                             }
301 
302                             @Override
303                             protected void collectAutoInfo(
304                                     IConfiguration config, TestInformation info)
305                                     throws DeviceNotAvailableException {
306                                 // Inop for the command test case.
307                             }
308 
309                             @Override
310                             protected void logHostAdb(IConfiguration config, ITestLogger logger) {
311                                 // inop for the common test case.
312                             }
313                         };
314                     }
315 
316                     @Override
317                     protected void setExitCode(ExitCode code, Throwable stack) {
318                         // Empty on purpose
319                     }
320 
321                     @Override
322                     public void registerExecutionFiles(ExecutionFiles executionFiles) {
323                         // Empty on purpose
324                     }
325 
326                     @Override
327                     protected void applyAutomatedReporters(IConfiguration config) {
328                         // Empty on purpose
329                     }
330 
331                     @Override
332                     protected void addInvocationMetric(InvocationMetricKey key, long value) {}
333 
334                     @Override
335                     protected void addInvocationMetric(InvocationMetricKey key, String value) {}
336                 };
337     }
338 
339     /** Helper to set the expectation for N number of shards. */
stubNShardInvocation(int numShard, String commandLine)340     private void stubNShardInvocation(int numShard, String commandLine) throws Exception {
341         /*
342         when(mMockBuildProvider.getBuild()).thenReturn(mMockBuildInfo);
343         Mockito.lenient().when(mMockBuildInfo.getTestTag()).thenReturn("");
344         when(mMockBuildInfo.clone()).thenReturn(mMockBuildInfo);
345         when(mMockBuildInfo.getVersionedFileKeys()).thenReturn(new HashSet<>());
346         when(mockRescheduler.scheduleConfig(Mockito.any())).thenReturn(true);
347         */
348     }
349 
350     /** Helper to set the expectation for N number of shards. */
verifyNShardInvocation(int numShard, String commandLine)351     private void verifyNShardInvocation(int numShard, String commandLine) throws Exception {
352         /*
353         verify(mMockBuildInfo).setTestTag(Mockito.eq("stub"));
354         verify(mMockBuildProvider.getBuild());
355         verify(mMockBuildInfo)
356                 .addBuildAttribute(Mockito.eq("command_line_args"), Mockito.eq(commandLine));
357 
358         mInOrderTestListener.verify(mMockTestListener).invocationStarted(Mockito.any());
359         mInOrderSummaryListener.verify(mMockSummaryListener).invocationStarted(Mockito.any());
360 
361         verify(mMockBuildInfo, times(numShard)).clone();
362         verify(mMockBuildInfo, times(numShard)).getVersionedFileKeys();
363         verify(mockRescheduler, times(numShard)).scheduleConfig(Mockito.any());
364         verify(mMockBuildInfo).setDeviceSerial(Mockito.eq(SERIAL));
365         verify(mMockBuildProvider).cleanUp(Mockito.any());
366         */
367     }
368 
stubMockSuccessListeners()369     private void stubMockSuccessListeners() throws IOException {
370         stubMockListeners(InvocationStatus.SUCCESS, null, false, true, false);
371     }
372 
verifyMockSuccessListeners()373     private void verifyMockSuccessListeners() throws IOException {
374         verifyMockListeners(InvocationStatus.SUCCESS, null, false, true, false, false);
375     }
376 
stubMockFailureListeners(Throwable throwable)377     private void stubMockFailureListeners(Throwable throwable) throws IOException {
378         stubMockListeners(InvocationStatus.FAILED, throwable, false, true, false);
379     }
380 
verifyMockFailureListeners(Throwable throwable)381     private void verifyMockFailureListeners(Throwable throwable) throws IOException {
382         verifyMockListeners(InvocationStatus.FAILED, throwable, false, true, false, false);
383     }
384 
stubMockFailureListenersAny(Throwable throwable, boolean stubFailures)385     private void stubMockFailureListenersAny(Throwable throwable, boolean stubFailures)
386             throws IOException {
387         stubMockListeners(InvocationStatus.FAILED, throwable, stubFailures, true, false);
388     }
389 
verifyMockFailureListenersAny(Throwable throwable, boolean stubFailures)390     private void verifyMockFailureListenersAny(Throwable throwable, boolean stubFailures)
391             throws IOException {
392         verifyMockListeners(InvocationStatus.FAILED, throwable, stubFailures, true, false, false);
393     }
394 
stubMockFailureListeners( Throwable throwable, boolean stubFailures, boolean reportHostLog)395     private void stubMockFailureListeners(
396             Throwable throwable, boolean stubFailures, boolean reportHostLog) throws IOException {
397         stubMockListeners(InvocationStatus.FAILED, throwable, stubFailures, reportHostLog, false);
398     }
399 
verifyMockFailureListeners( Throwable throwable, boolean stubFailures, boolean reportHostLog)400     private void verifyMockFailureListeners(
401             Throwable throwable, boolean stubFailures, boolean reportHostLog) throws IOException {
402         verifyMockListeners(
403                 InvocationStatus.FAILED, throwable, stubFailures, reportHostLog, false, false);
404     }
405 
stubMockStoppedListeners()406     private void stubMockStoppedListeners() throws IOException {
407         stubMockListeners(InvocationStatus.SUCCESS, null, false, true, true);
408     }
409 
verifyMockStoppedListeners(boolean testSkipped)410     private void verifyMockStoppedListeners(boolean testSkipped) throws IOException {
411         verifyMockListeners(InvocationStatus.SUCCESS, null, false, true, true, testSkipped);
412     }
413 
verifySummaryListener()414     private void verifySummaryListener() {
415         // Check that we captured the expected uris List
416         List<TestSummary> summaries = mUriCapture.getValue();
417         assertEquals(1, summaries.size());
418         assertEquals(mSummary, summaries.get(0));
419     }
420 
421     /**
422      * Set up expected conditions for normal run up to the part where tests are run.
423      *
424      * @param test the {@link Test} to use.
425      */
stubNormalInvoke(IRemoteTest test)426     private void stubNormalInvoke(IRemoteTest test) throws Throwable {
427         stubInvokeWithBuild();
428         mStubConfiguration.setTest(test);
429         mStubMultiConfiguration.setTest(test);
430         when(mMockBuildProvider.getBuild()).thenReturn(mMockBuildInfo);
431     }
432 
433     /**
434      * Set up expected conditions for normal run up to the part where tests are run.
435      *
436      * @param test the {@link Test} to use.
437      */
verifyNormalInvoke(IRemoteTest test)438     private void verifyNormalInvoke(IRemoteTest test) throws Throwable {
439         try {
440             verify(mMockLogger).getLog();
441             verify(mMockLogger, times(3)).init();
442             verify(mMockLogger, times(2)).closeLog();
443 
444             verify(mMockBuildProvider, atLeast(1)).cleanUp(Mockito.eq(mMockBuildInfo));
445             verify(mMockBuildProvider).getBuild();
446 
447             verify(mMockBuildInfo).setDeviceSerial(Mockito.eq(SERIAL));
448             verify(mMockBuildInfo).setTestTag(Mockito.eq("stub"));
449 
450             // always expect logger initialization and cleanup calls
451             verify(mMockLogRegistry, times(2)).unregisterLogger();
452             verify(mMockLogRegistry, times(3)).registerLogger(mMockLogger);
453 
454             verify(mMockPreparer).setUp(Mockito.any());
455 
456         } catch (IOException | TargetSetupError | DeviceNotAvailableException e) {
457             // Never happens since these are all verify calls
458         }
459     }
460 
461     /** Set up expected calls that occur on every invoke, regardless of result */
verifyInvoke()462     private void verifyInvoke() throws DeviceNotAvailableException {
463         // TODO(murj) Probably needs the in-order treatment
464         try {
465             // always expect logger initialization and cleanup calls
466             verify(mMockLogRegistry, times(2)).registerLogger(mMockLogger);
467             verify(mMockLogRegistry, times(2)).unregisterLogger();
468 
469             verify(mMockLogger, times(2)).init();
470             verify(mMockLogger, times(2)).closeLog();
471         } catch (IOException e) {
472             // Never happens since these are all verify calls
473         }
474     }
475 
476     /** Set up expected calls that occur on every invoke that gets a valid build */
stubInvokeWithBuild()477     private void stubInvokeWithBuild() throws DeviceNotAvailableException {
478         when(mMockDevice.getLogcat())
479                 .thenReturn(mLogcatSetupSource)
480                 .thenReturn(mLogcatTestSource)
481                 .thenReturn(mLogcatTeardownSource);
482         when(mMockDevice.waitForDeviceAvailable(TestInvocation.AVAILABILITY_CHECK_TIMEOUT))
483                 .thenReturn(true);
484 
485         when(mMockLogger.getLog()).thenReturn(mHostLogSource);
486         Mockito.lenient().when(mMockBuildInfo.getTestTag()).thenReturn("");
487     }
488 
489     /** Set up expected calls that occur on every invoke that gets a valid build */
verifyInvokeWithBuild()490     private void verifyInvokeWithBuild() {
491         try {
492             verify(mMockLogger).getLog();
493             verify(mMockLogger, times(3)).init();
494             verify(mMockLogger, times(2)).closeLog();
495 
496             verify(mMockBuildInfo).setDeviceSerial(Mockito.eq(SERIAL));
497             verify(mMockBuildInfo).setTestTag(Mockito.eq("stub"));
498 
499             verify(mMockBuildProvider, atLeast(1)).cleanUp(Mockito.eq(mMockBuildInfo));
500 
501             // always expect logger initialization and cleanup calls
502             verify(mMockLogRegistry, times(3)).registerLogger(mMockLogger);
503             verify(mMockLogRegistry, times(2)).unregisterLogger();
504 
505         } catch (IOException e) {
506             // Never happens since these are all verify calls
507         }
508     }
509 
510     /**
511      * Set up expected conditions for the test InvocationListener and SummaryListener
512      *
513      * <p>The order of calls for a single listener should be:
514      *
515      * <ol>
516      *   <li>invocationStarted
517      *   <li>testLog(LOGCAT_NAME_SETUP, ...) (if no build or retrieval error)
518      *   <li>invocationFailed (if run failed)
519      *   <li>testLog(LOGCAT_NAME_ERROR, ...) (if build retrieval error)
520      *   <li>testLog(LOGCAT_NAME_TEST, ...) (otherwise)
521      *   <li>testLog(build error bugreport, ...) (otherwise and if build error)
522      *   <li>testLog(LOGCAT_NAME_TEARDOWN, ...) (otherwise)
523      *   <li>testLog(TRADEFED_LOG_NAME, ...)
524      *   <li>putSummary (for an ITestSummaryListener)
525      *   <li>invocationEnded
526      *   <li>getSummary (for an ITestInvocationListener)
527      * </ol>
528      *
529      * However note that, across all listeners, any getSummary call will precede all putSummary
530      * calls.
531      */
stubMockListeners( InvocationStatus status, Throwable throwable, boolean stubFailures, boolean reportHostLog, boolean stopped)532     private void stubMockListeners(
533             InvocationStatus status,
534             Throwable throwable,
535             boolean stubFailures,
536             boolean reportHostLog,
537             boolean stopped)
538             throws IOException {
539         // Very important, the second and beyond calls to mMockTestListener.getSummary() will
540         // actually
541         // return the summary which was originally stubbed at the "end" of this method.
542         doReturn(null).doReturn(mSummary).when(mMockTestListener).getSummary();
543         doReturn(null).when(mMockSummaryListener).getSummary();
544 
545         doReturn(new LogFile(PATH, URL, LogDataType.TEXT))
546                 .when(mMockLogSaver)
547                 .saveLogData(
548                         Mockito.contains(CONFIG_LOG_NAME),
549                         Mockito.eq(LogDataType.HARNESS_CONFIG),
550                         (InputStream) Mockito.any());
551 
552         if (!(throwable instanceof BuildRetrievalError) && !mShardingEarlyFailure) {
553             doReturn(new LogFile(PATH, URL, LogDataType.TEXT))
554                     .when(mMockLogSaver)
555                     .saveLogData(
556                             Mockito.startsWith(LOGCAT_NAME_SETUP),
557                             Mockito.eq(LogDataType.LOGCAT),
558                             (InputStream) Mockito.any());
559         }
560 
561         if (throwable instanceof BuildRetrievalError) {
562             // Handle logcat error listeners
563             doReturn(new LogFile(PATH, URL, LogDataType.TEXT))
564                     .when(mMockLogSaver)
565                     .saveLogData(
566                             Mockito.startsWith(LOGCAT_NAME_ERROR),
567                             Mockito.eq(LogDataType.LOGCAT),
568                             (InputStream) Mockito.any());
569         } else {
570             // Handle build error bugreport listeners
571             if (throwable instanceof BuildError) {
572                 doReturn(true)
573                         .when(mMockDevice)
574                         .logBugreport(
575                                 Mockito.eq(
576                                         TestInvocation.BUILD_ERROR_BUGREPORT_NAME + "_" + SERIAL),
577                                 Mockito.any());
578             } else if (!(throwable instanceof TargetSetupError) && !mShardingEarlyFailure) {
579                 // Handle test logcat listeners
580                 doReturn(new LogFile(PATH, URL, LogDataType.TEXT))
581                         .when(mMockLogSaver)
582                         .saveLogData(
583                                 Mockito.startsWith(LOGCAT_NAME_TEST),
584                                 Mockito.eq(LogDataType.LOGCAT),
585                                 (InputStream) Mockito.any());
586             }
587             // Handle teardown logcat listeners
588             if (!mShardingEarlyFailure) {
589                 doReturn(new LogFile(PATH, URL, LogDataType.TEXT))
590                         .when(mMockLogSaver)
591                         .saveLogData(
592                                 Mockito.startsWith(LOGCAT_NAME_TEARDOWN),
593                                 Mockito.eq(LogDataType.LOGCAT),
594                                 (InputStream) Mockito.any());
595             }
596         }
597 
598         doReturn(new LogFile(PATH, URL, LogDataType.HOST_LOG))
599                 .when(mMockLogSaver)
600                 .saveLogData(
601                         Mockito.eq(TestInvocation.TRADEFED_END_HOST_LOG),
602                         Mockito.eq(LogDataType.HOST_LOG),
603                         (InputStream) Mockito.any());
604 
605         if (reportHostLog) {
606             doReturn(new LogFile(PATH, URL, LogDataType.HOST_LOG))
607                     .when(mMockLogSaver)
608                     .saveLogData(
609                             Mockito.eq(TestInvocation.TRADEFED_LOG_NAME),
610                             Mockito.eq(LogDataType.HOST_LOG),
611                             (InputStream) Mockito.any());
612         }
613     }
614 
verifyMockListeners( InvocationStatus status, Throwable throwable, boolean stubFailures, boolean reportHostLog, boolean stopped, boolean testSkipped)615     private void verifyMockListeners(
616             InvocationStatus status,
617             Throwable throwable,
618             boolean stubFailures,
619             boolean reportHostLog,
620             boolean stopped,
621             boolean testSkipped)
622             throws IOException {
623         // invocationStarted
624         mInOrderTestListener
625                 .verify(mMockTestListener)
626                 .invocationStarted(Mockito.eq(mStubInvocationMetadata));
627         mInOrderTestListener.verify(mMockTestListener).getSummary();
628         mInOrderSummaryListener.verify(mMockSummaryListener).putEarlySummary(Mockito.any());
629         mInOrderSummaryListener
630                 .verify(mMockSummaryListener)
631                 .invocationStarted(mStubInvocationMetadata);
632         mInOrderSummaryListener.verify(mMockSummaryListener).getSummary();
633         mInOrderLogSaver
634                 .verify(mMockLogSaver)
635                 .invocationStarted(Mockito.eq(mStubInvocationMetadata));
636 
637         mInOrderTestListener
638                 .verify(mMockTestListener)
639                 .testLog(
640                         Mockito.contains(CONFIG_LOG_NAME),
641                         Mockito.eq(LogDataType.HARNESS_CONFIG),
642                         (InputStreamSource) Mockito.any());
643         mInOrderSummaryListener
644                 .verify(mMockSummaryListener)
645                 .testLog(
646                         Mockito.contains(CONFIG_LOG_NAME),
647                         Mockito.eq(LogDataType.HARNESS_CONFIG),
648                         (InputStreamSource) Mockito.any());
649 
650         if (throwable == null) {
651             verify(mMockDevice, atLeast(0)).postInvocationTearDown(null);
652         } else {
653             verify(mMockDevice, atLeast(0)).postInvocationTearDown(throwable);
654         }
655 
656         if (!(throwable instanceof BuildRetrievalError) && !mShardingEarlyFailure) {
657             mInOrderTestListener
658                     .verify(mMockTestListener)
659                     .testLog(
660                             Mockito.startsWith(LOGCAT_NAME_SETUP),
661                             Mockito.eq(LogDataType.LOGCAT),
662                             (InputStreamSource) Mockito.any());
663 
664             mInOrderSummaryListener
665                     .verify(mMockSummaryListener)
666                     .testLog(
667                             Mockito.startsWith(LOGCAT_NAME_SETUP),
668                             Mockito.eq(LogDataType.LOGCAT),
669                             (InputStreamSource) Mockito.any());
670         }
671 
672         // invocationFailed
673         if (!status.equals(InvocationStatus.SUCCESS)) {
674             if (stubFailures) {
675                 mInOrderTestListener
676                         .verify(mMockTestListener)
677                         .invocationFailed((FailureDescription) Mockito.any());
678 
679                 mInOrderSummaryListener
680                         .verify(mMockSummaryListener)
681                         .invocationFailed((FailureDescription) Mockito.any());
682 
683             } else {
684                 FailureStatus failureStatus = FailureStatus.INFRA_FAILURE;
685                 if (throwable instanceof BuildError) {
686                     failureStatus = FailureStatus.DEPENDENCY_ISSUE;
687                 }
688                 FailureDescription failure =
689                         FailureDescription.create(throwable.getMessage(), failureStatus)
690                                 .setCause(throwable);
691                 if (throwable instanceof BuildRetrievalError) {
692                     failure.setActionInProgress(ActionInProgress.FETCHING_ARTIFACTS);
693                 } else if (throwable instanceof BuildError
694                         || throwable instanceof TargetSetupError) {
695                     failure.setActionInProgress(ActionInProgress.SETUP);
696                 } else {
697                     failure.setActionInProgress(ActionInProgress.TEST);
698                 }
699                 if (mExceptedStatus != null) {
700                     failure.setFailureStatus(mExceptedStatus);
701                 }
702 
703                 mInOrderTestListener
704                         .verify(mMockTestListener)
705                         .invocationFailed(Mockito.eq(failure));
706                 mInOrderSummaryListener
707                         .verify(mMockSummaryListener)
708                         .invocationFailed(Mockito.eq(failure));
709             }
710         }
711 
712         if (throwable instanceof BuildRetrievalError) {
713             // Handle logcat error listeners
714             mInOrderTestListener
715                     .verify(mMockTestListener)
716                     .testLog(
717                             Mockito.startsWith(LOGCAT_NAME_ERROR),
718                             Mockito.eq(LogDataType.LOGCAT),
719                             (InputStreamSource) Mockito.any());
720             mInOrderSummaryListener
721                     .verify(mMockSummaryListener)
722                     .testLog(
723                             Mockito.startsWith(LOGCAT_NAME_ERROR),
724                             Mockito.eq(LogDataType.LOGCAT),
725                             (InputStreamSource) Mockito.any());
726         } else {
727             // Handle build error bugreport listeners
728             if (throwable instanceof BuildError) {
729             } else if (!(throwable instanceof TargetSetupError)
730                     && !mShardingEarlyFailure
731                     && !testSkipped) {
732                 // Handle test logcat listeners
733                 mInOrderTestListener
734                         .verify(mMockTestListener)
735                         .testLog(
736                                 Mockito.startsWith(LOGCAT_NAME_TEST),
737                                 Mockito.eq(LogDataType.LOGCAT),
738                                 (InputStreamSource) Mockito.any());
739                 mInOrderSummaryListener
740                         .verify(mMockSummaryListener)
741                         .testLog(
742                                 Mockito.startsWith(LOGCAT_NAME_TEST),
743                                 Mockito.eq(LogDataType.LOGCAT),
744                                 (InputStreamSource) Mockito.any());
745             }
746             // Handle teardown logcat listeners
747             if (!mShardingEarlyFailure) {
748                 mInOrderTestListener
749                         .verify(mMockTestListener)
750                         .testLog(
751                                 Mockito.startsWith(LOGCAT_NAME_TEARDOWN),
752                                 Mockito.eq(LogDataType.LOGCAT),
753                                 (InputStreamSource) Mockito.any());
754                 mInOrderSummaryListener
755                         .verify(mMockSummaryListener)
756                         .testLog(
757                                 Mockito.startsWith(LOGCAT_NAME_TEARDOWN),
758                                 Mockito.eq(LogDataType.LOGCAT),
759                                 (InputStreamSource) Mockito.any());
760             }
761 
762             if (stopped) {
763                 mInOrderTestListener
764                         .verify(mMockTestListener)
765                         .invocationFailed((FailureDescription) Mockito.any());
766                 mInOrderSummaryListener
767                         .verify(mMockSummaryListener)
768                         .invocationFailed((FailureDescription) Mockito.any());
769             }
770         }
771 
772         if (reportHostLog) {
773             mInOrderTestListener
774                     .verify(mMockTestListener)
775                     .testLog(
776                             Mockito.eq(TestInvocation.TRADEFED_LOG_NAME),
777                             Mockito.eq(LogDataType.HOST_LOG),
778                             (InputStreamSource) Mockito.any());
779             mInOrderSummaryListener
780                     .verify(mMockSummaryListener)
781                     .testLog(
782                             Mockito.eq(TestInvocation.TRADEFED_LOG_NAME),
783                             Mockito.eq(LogDataType.HOST_LOG),
784                             (InputStreamSource) Mockito.any());
785         }
786 
787         // invocationEnded, getSummary (mMockTestListener)
788         mInOrderTestListener.verify(mMockTestListener).invocationEnded(Mockito.anyLong());
789 
790         // putSummary, invocationEnded (mMockSummaryListener)
791         mInOrderSummaryListener.verify(mMockSummaryListener).putSummary(mUriCapture.capture());
792         mInOrderSummaryListener.verify(mMockSummaryListener).invocationEnded(Mockito.anyLong());
793 
794         mInOrderLogSaver.verify(mMockLogSaver).invocationEnded(Mockito.anyLong());
795     }
796 
797     /** Helper tests class to expose all the interfaces needed for the tests. */
798     private interface IFakeBuildProvider extends IDeviceBuildProvider, IInvocationContextReceiver {}
799 
800     @SuppressWarnings("deprecation")
createTargetSetupError(String reason)801     private TargetSetupError createTargetSetupError(String reason) {
802         // Use the deprecated constructor on purpose to simulate missing DeviceDescriptor.
803         return new TargetSetupError(reason);
804     }
805 
806     /** Interface for testing device config pass through. */
807     private interface DeviceConfigTest extends IRemoteTest, IDeviceTest {}
808 
809     public static class TestableCollector extends BaseDeviceMetricCollector {
810 
811         @Option(name = "name")
812         private String mName;
813 
TestableCollector()814         public TestableCollector() {}
815 
TestableCollector(String name)816         public TestableCollector(String name) {
817             mName = name;
818         }
819 
820         @Override
onTestRunEnd( DeviceMetricData runData, final Map<String, Metric> currentRunMetrics)821         public void onTestRunEnd(
822                 DeviceMetricData runData, final Map<String, Metric> currentRunMetrics) {
823             runData.addMetric(
824                     mName,
825                     Metric.newBuilder()
826                             .setMeasurements(
827                                     Measurements.newBuilder().setSingleString(mName).build()));
828         }
829     }
830 
831     public static class TestableProcessor extends BasePostProcessor {
832 
833         @Option(name = "name")
834         private String mName;
835 
TestableProcessor()836         public TestableProcessor() {}
837 
TestableProcessor(String name)838         public TestableProcessor(String name) {
839             mName = name;
840         }
841 
842         @Override
processRunMetricsAndLogs( HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs)843         public Map<String, Metric.Builder> processRunMetricsAndLogs(
844                 HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs) {
845             Map<String, Metric.Builder> post = new LinkedHashMap<>();
846             post.put(mName, Metric.newBuilder());
847             return post;
848         }
849     }
850 
851     private class TestLogSaverListener implements ILogSaverListener {
852 
853         public boolean mWasLoggerSet = false;
854 
855         @Override
setLogSaver(ILogSaver logSaver)856         public void setLogSaver(ILogSaver logSaver) {
857             mWasLoggerSet = true;
858         }
859     }
860 
stubEarlyDeviceReleaseExpectation()861     private void stubEarlyDeviceReleaseExpectation() {
862         when(mMockDevice.getDeviceState()).thenReturn(TestDeviceState.ONLINE);
863         when(mMockDevice.waitForDeviceShell(30000L)).thenReturn(true);
864     }
865 
verifyEarlyDeviceReleaseExpectation()866     private void verifyEarlyDeviceReleaseExpectation() {
867         verify(mMockDevice, atLeast(1)).getDeviceState();
868         verify(mMockDevice).waitForDeviceShell(30000L);
869         verify(mMockDevice).setRecoveryMode(RecoveryMode.AVAILABLE);
870     }
871 
872     /**
873      * Test the normal case invoke scenario with a {@link IRemoteTest}.
874      *
875      * <p>Verifies that all external interfaces get notified as expected.
876      */
877     @Test
testInvoke_RemoteTest()878     public void testInvoke_RemoteTest() throws Throwable {
879         IRemoteTest test = Mockito.mock(IRemoteTest.class);
880         stubEarlyDeviceReleaseExpectation();
881         stubMockSuccessListeners();
882         stubNormalInvoke(test);
883 
884         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
885 
886         verify(test).run(Mockito.any(), Mockito.any());
887 
888         verifyNormalInvoke(test);
889         verify(mMockPreparer).tearDown(Mockito.any(), Mockito.isNull());
890 
891         verifyMockSuccessListeners();
892         verifyEarlyDeviceReleaseExpectation();
893 
894         verifySummaryListener();
895     }
896 
897     /**
898      * Test the normal case for multi invoke scenario with a {@link IRemoteTest}.
899      *
900      * <p>Verifies that all external interfaces get notified as expected.
901      */
902     @Test
testInvokeMulti_RemoteTest()903     public void testInvokeMulti_RemoteTest() throws Throwable {
904         IRemoteTest test = Mockito.mock(IRemoteTest.class);
905 
906         stubEarlyDeviceReleaseExpectation();
907         stubMockSuccessListeners();
908         stubNormalInvoke(test);
909 
910         mTestInvocation.invoke(mStubInvocationMetadata, mStubMultiConfiguration, mockRescheduler);
911 
912         verify(test).run(Mockito.any(), Mockito.any());
913 
914         verifyNormalInvoke(test);
915         verify(mMockPreparer).tearDown(Mockito.any(), Mockito.isNull());
916 
917         verifyMockSuccessListeners();
918 
919         verifySummaryListener();
920     }
921 
922     /**
923      * Test the normal case invoke scenario with an {@link ITestSummaryListener} masquerading as an
924      * {@link ITestInvocationListener}.
925      *
926      * <p>Verifies that all external interfaces get notified as expected.
927      */
928     @Test
testInvoke_twoSummary()929     public void testInvoke_twoSummary() throws Throwable {
930         IRemoteTest test = Mockito.mock(IRemoteTest.class);
931 
932         stubEarlyDeviceReleaseExpectation();
933         stubMockSuccessListeners();
934         stubNormalInvoke(test);
935 
936         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
937 
938         verify(test).run(Mockito.any(), Mockito.any());
939 
940         verifyNormalInvoke(test);
941         verify(mMockPreparer).tearDown(Mockito.any(), Mockito.isNull());
942 
943         verifyMockSuccessListeners();
944 
945         verifySummaryListener();
946     }
947 
948     /**
949      * Test the invoke scenario where build retrieve fails.
950      *
951      * <p>An invocation will be started in this scenario.
952      */
953     @Test
testInvoke_buildFailed()954     public void testInvoke_buildFailed() throws Throwable {
955         BuildRetrievalError exception =
956                 new BuildRetrievalError("testInvoke_buildFailed", null, mMockBuildInfo);
957         when(mMockBuildProvider.getBuild()).thenThrow(exception);
958         when(mMockBuildInfo.getTestTag()).thenReturn(null);
959 
960         stubEarlyDeviceReleaseExpectation();
961         stubMockFailureListeners(exception);
962         when(mMockLogger.getLog()).thenReturn(mHostLogSource);
963         when(mMockDevice.getLogcat()).thenReturn(mLogcatSetupSource).thenReturn(mLogcatTestSource);
964 
965         IRemoteTest test = Mockito.mock(IRemoteTest.class);
966         CommandOptions cmdOptions = new CommandOptions();
967         final String expectedTestTag = "TEST_TAG";
968         cmdOptions.setTestTag(expectedTestTag);
969         mStubConfiguration.setCommandOptions(cmdOptions);
970         mStubConfiguration.setTest(test);
971 
972         try {
973             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
974             fail("Should have thrown an exception.");
975         } catch (BuildRetrievalError expected) {
976             // Expected
977         }
978 
979         // Needed a full custom set of verifications because it is messy
980         try {
981             // always expect logger initialization and cleanup calls
982             verify(mMockLogRegistry, times(3)).registerLogger(mMockLogger);
983             verify(mMockLogRegistry, times(2)).unregisterLogger();
984 
985             verify(mMockLogger, times(3)).init();
986             verify(mMockLogger, times(2)).closeLog();
987             verify(mMockLogger).getLog();
988 
989             verify(mMockBuildProvider).cleanUp(mMockBuildInfo);
990 
991             verify(mMockBuildInfo).setTestTag(expectedTestTag);
992         } catch (IOException e) {
993             // Never happens since these are all verify calls
994         }
995 
996         verifyMockFailureListeners(exception);
997     }
998 
999     /** Ensure we get a build info when we get a runtime exception in fetch build. */
1000     @Test
testInvoke_buildFailed_runtimeException()1001     public void testInvoke_buildFailed_runtimeException() throws Throwable {
1002         RuntimeException runtimeException = new RuntimeException("failed to get build.");
1003         when(mMockBuildProvider.getBuild()).thenThrow(runtimeException);
1004 
1005         BuildRetrievalError error =
1006                 new BuildRetrievalError("fake", InfraErrorIdentifier.ARTIFACT_DOWNLOAD_ERROR);
1007         stubMockFailureListenersAny(error, true);
1008 
1009         when(mMockLogger.getLog()).thenReturn(mHostLogSource);
1010         when(mMockDevice.getLogcat())
1011                 .thenReturn(mLogcatSetupSource)
1012                 .thenReturn(mLogcatTeardownSource);
1013         ArgumentCaptor<IBuildInfo> captured = ArgumentCaptor.forClass(IBuildInfo.class);
1014 
1015         mStubConfiguration.setCommandLine(new String[] {"empty", "--build-id", "5"});
1016 
1017         try {
1018             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1019             fail("Should have thrown an exception.");
1020         } catch (RuntimeException expected) {
1021             // Expected
1022         }
1023 
1024         verify(mMockBuildProvider).cleanUp(captured.capture());
1025         verify(mMockLogRegistry, times(3)).registerLogger(mMockLogger);
1026         verify(mMockLogRegistry, times(2)).unregisterLogger();
1027         verify(mMockLogger, times(3)).init();
1028         verify(mMockLogger, times(2)).closeLog();
1029 
1030         verifyMockFailureListenersAny(error, true);
1031 
1032         IBuildInfo stubBuild = captured.getValue();
1033         assertEquals("5", stubBuild.getBuildId());
1034         stubBuild.cleanUp();
1035     }
1036 
1037     /** Test the invoke scenario where there is no build to test. */
1038     @Test
testInvoke_noBuild()1039     public void testInvoke_noBuild() throws Throwable {
1040         when(mMockBuildProvider.getBuild()).thenReturn(null);
1041         BuildRetrievalError error =
1042                 new BuildRetrievalError(
1043                         "No build found to test.", InfraErrorIdentifier.ARTIFACT_NOT_FOUND);
1044 
1045         stubMockFailureListenersAny(error, true);
1046 
1047         when(mMockLogger.getLog()).thenReturn(mHostLogSource);
1048         when(mMockDevice.getLogcat()).thenReturn(mLogcatSetupSource).thenReturn(mLogcatTestSource);
1049         ArgumentCaptor<IBuildInfo> captured = ArgumentCaptor.forClass(IBuildInfo.class);
1050 
1051         try {
1052             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1053             fail("Should have thrown an exception.");
1054         } catch (BuildRetrievalError expected) {
1055             // Expected
1056         }
1057 
1058         verify(mMockBuildProvider).cleanUp(captured.capture());
1059         verify(mMockLogRegistry, times(3)).registerLogger(mMockLogger);
1060         verify(mMockLogRegistry, times(2)).unregisterLogger();
1061         verify(mMockLogger, times(3)).init();
1062         verify(mMockLogger, times(2)).closeLog();
1063 
1064         verifyMockFailureListenersAny(error, true);
1065 
1066         IBuildInfo stubBuild = captured.getValue();
1067         assertEquals(BuildInfo.UNKNOWN_BUILD_ID, stubBuild.getBuildId());
1068         stubBuild.cleanUp();
1069     }
1070 
1071     /**
1072      * Test when the reporting of host_log is returning null, in this case we don't log anything.
1073      */
1074     @Test
testInvoke_noBuild_noHostLog()1075     public void testInvoke_noBuild_noHostLog() throws Throwable {
1076         when(mMockBuildProvider.getBuild()).thenReturn(null);
1077 
1078         Throwable error =
1079                 new BuildRetrievalError(
1080                         "No build found to test.", InfraErrorIdentifier.ARTIFACT_NOT_FOUND);
1081         IRemoteTest test = Mockito.mock(IRemoteTest.class);
1082         mStubConfiguration.setTest(test);
1083 
1084         stubMockFailureListeners(error, true, false);
1085 
1086         when(mMockLogger.getLog()).thenReturn(null);
1087         when(mMockDevice.getLogcat()).thenReturn(mLogcatSetupSource).thenReturn(mLogcatTestSource);
1088         ArgumentCaptor<IBuildInfo> captured = ArgumentCaptor.forClass(IBuildInfo.class);
1089 
1090         try {
1091             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1092             fail("Should have thrown an exception.");
1093         } catch (BuildRetrievalError expected) {
1094             // Expected
1095         }
1096 
1097         verify(mMockBuildProvider).cleanUp(captured.capture());
1098         verify(mMockLogRegistry, times(3)).registerLogger(mMockLogger);
1099         verify(mMockLogRegistry, times(2)).unregisterLogger();
1100         verify(mMockLogger, times(3)).init();
1101         verify(mMockLogger, times(2)).closeLog();
1102 
1103         verifyMockFailureListeners(error, true, false);
1104 
1105         IBuildInfo stubBuild = captured.getValue();
1106         assertEquals(BuildInfo.UNKNOWN_BUILD_ID, stubBuild.getBuildId());
1107         stubBuild.cleanUp();
1108     }
1109 
1110     /** Ensure that we do not take a bugreport if the build throws a runtime exception */
1111     @Test
testInvoke_skipBugreport_buildFailed()1112     public void testInvoke_skipBugreport_buildFailed() throws Throwable {
1113         RuntimeException runtimeException = new RuntimeException("failed to get build.");
1114         when(mMockBuildProvider.getBuild()).thenThrow(runtimeException);
1115 
1116         BuildRetrievalError error =
1117                 new BuildRetrievalError("fake", InfraErrorIdentifier.ARTIFACT_DOWNLOAD_ERROR);
1118         stubMockFailureListenersAny(error, true);
1119 
1120         when(mMockLogger.getLog()).thenReturn(mHostLogSource);
1121         when(mMockDevice.getLogcat())
1122                 .thenReturn(mLogcatSetupSource)
1123                 .thenReturn(mLogcatTeardownSource);
1124         ArgumentCaptor<IBuildInfo> captured = ArgumentCaptor.forClass(IBuildInfo.class);
1125 
1126         mStubConfiguration.setCommandLine(new String[] {"empty", "--build-id", "5"});
1127 
1128         try {
1129             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1130             fail("Should have thrown an exception.");
1131         } catch (RuntimeException expected) {
1132             // Expected
1133         }
1134 
1135         verify(mMockBuildProvider).cleanUp(captured.capture());
1136         verify(mMockLogRegistry, times(3)).registerLogger(mMockLogger);
1137         verify(mMockLogRegistry, times(2)).unregisterLogger();
1138         verify(mMockLogger, times(3)).init();
1139         verify(mMockLogger, times(2)).closeLog();
1140         verify(mMockDevice, times(0))
1141                 .logBugreport(Mockito.anyString(), (ITestInvocationListener) Mockito.any());
1142 
1143         verifyMockFailureListenersAny(error, true);
1144 
1145         IBuildInfo stubBuild = captured.getValue();
1146         assertEquals("5", stubBuild.getBuildId());
1147         stubBuild.cleanUp();
1148     }
1149 
1150     /**
1151      * Test the{@link TestInvocation#invoke(IInvocationContext, IConfiguration, IRescheduler,
1152      * ITestInvocationListener[])} scenario where the test is a {@link IDeviceTest}
1153      */
1154     @Test
testInvoke_deviceTest()1155     public void testInvoke_deviceTest() throws Throwable {
1156         DeviceConfigTest mockDeviceTest = Mockito.mock(DeviceConfigTest.class);
1157         mStubConfiguration.setTest(mockDeviceTest);
1158 
1159         stubMockSuccessListeners();
1160         stubEarlyDeviceReleaseExpectation();
1161         stubNormalInvoke(mockDeviceTest);
1162 
1163         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1164 
1165         verifyMockSuccessListeners();
1166 
1167         verify(mMockPreparer).tearDown(Mockito.any(), Mockito.isNull());
1168         verify(mockDeviceTest).setDevice(mMockDevice);
1169         verify(mockDeviceTest).run(Mockito.any(), Mockito.any());
1170 
1171         verifySummaryListener();
1172     }
1173 
1174     /**
1175      * Test the invoke scenario where test run throws {@link IllegalArgumentException}
1176      *
1177      * @throws Exception if unexpected error occurs
1178      */
1179     @Test
testInvoke_testFail()1180     public void testInvoke_testFail() throws Throwable {
1181         IllegalArgumentException exception = new IllegalArgumentException("testInvoke_testFail");
1182         mExceptedStatus = FailureStatus.UNSET;
1183         IRemoteTest test = Mockito.mock(IRemoteTest.class);
1184 
1185         stubEarlyDeviceReleaseExpectation();
1186         stubMockFailureListeners(exception);
1187         stubNormalInvoke(test);
1188 
1189         doThrow(exception).when(test).run(Mockito.any(), Mockito.any());
1190 
1191         try {
1192             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1193             fail("IllegalArgumentException was not rethrown");
1194         } catch (IllegalArgumentException e) {
1195             // expected
1196         }
1197 
1198         verify(mMockPreparer).tearDown(Mockito.any(), Mockito.eq(exception));
1199         verifyMockFailureListeners(exception);
1200         verifyNormalInvoke(test);
1201 
1202         verifySummaryListener();
1203     }
1204 
1205     /**
1206      * Test that tests were skipped and metrics SHUTDOWN_HARD_LATENCY is collected when the
1207      * invocation is stopped/interrupted before test phase started.
1208      */
1209     @Test
testInvoke_metricsCollectedWhenStopped()1210     public void testInvoke_metricsCollectedWhenStopped() throws Throwable {
1211         IRemoteTest test = Mockito.mock(IRemoteTest.class);
1212 
1213         stubEarlyDeviceReleaseExpectation();
1214         stubMockStoppedListeners();
1215         stubNormalInvoke(test);
1216 
1217         mTestInvocation.notifyInvocationForceStopped(
1218                 "Stopped", InfraErrorIdentifier.INVOCATION_TIMEOUT);
1219         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1220 
1221         verify(test, never()).run(Mockito.any(), Mockito.any());
1222         verify(mMockPreparer).tearDown(Mockito.any(), Mockito.any());
1223 
1224         verifyNormalInvoke(test);
1225         verifyMockStoppedListeners(true);
1226 
1227         assertTrue(
1228                 mStubInvocationMetadata
1229                         .getAttributes()
1230                         .containsKey(InvocationMetricKey.SHUTDOWN_HARD_LATENCY.toString()));
1231         verifySummaryListener();
1232     }
1233 
1234     /**
1235      * Test the invoke scenario where test run throws {@link FatalHostError}
1236      *
1237      * @throws Exception if unexpected error occurs
1238      */
1239     @Test
testInvoke_fatalError()1240     public void testInvoke_fatalError() throws Throwable {
1241         FatalHostError exception = new FatalHostError("testInvoke_fatalError");
1242         mExceptedStatus = FailureStatus.UNSET;
1243         IRemoteTest test = Mockito.mock(IRemoteTest.class);
1244 
1245         stubEarlyDeviceReleaseExpectation();
1246         stubMockFailureListeners(exception);
1247         doThrow(exception).when(test).run(Mockito.any(), Mockito.any());
1248         stubNormalInvoke(test);
1249 
1250         try {
1251             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1252             fail("FatalHostError was not rethrown");
1253         } catch (FatalHostError e) {
1254             // expected
1255         }
1256 
1257         verifyNormalInvoke(test);
1258         verifyMockFailureListeners(exception);
1259 
1260         verifySummaryListener();
1261     }
1262 
1263     /**
1264      * Test the invoke scenario where test run throws {@link DeviceNotAvailableException}
1265      *
1266      * @throws Exception if unexpected error occurs
1267      */
1268     @Test
testInvoke_deviceNotAvail()1269     public void testInvoke_deviceNotAvail() throws Throwable {
1270         DeviceNotAvailableException exception = new DeviceNotAvailableException("ERROR", SERIAL);
1271         IRemoteTest test = Mockito.mock(IRemoteTest.class);
1272 
1273         stubEarlyDeviceReleaseExpectation();
1274         stubMockFailureListeners(exception);
1275         stubNormalInvoke(test);
1276 
1277         doThrow(exception).when(test).run(Mockito.any(), Mockito.any());
1278 
1279         try {
1280             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1281             fail("DeviceNotAvailableException not thrown");
1282         } catch (DeviceNotAvailableException e) {
1283             // expected
1284         }
1285 
1286         verifyNormalInvoke(test);
1287         verifyMockFailureListeners(exception);
1288 
1289         verifySummaryListener();
1290     }
1291 
1292     @Test
testInvoke_setupError()1293     public void testInvoke_setupError() throws Throwable {
1294         TargetSetupError tse = createTargetSetupError("reason");
1295         IRemoteTest test = Mockito.mock(IRemoteTest.class);
1296 
1297         mStubConfiguration.setTest(test);
1298         mStubMultiConfiguration.setTest(test);
1299 
1300         stubEarlyDeviceReleaseExpectation();
1301         stubMockFailureListeners(tse);
1302         stubInvokeWithBuild();
1303 
1304         when(mMockDevice.getRecoveryMode()).thenReturn(RecoveryMode.AVAILABLE);
1305         when(mMockDevice.logBugreport(
1306                         Mockito.startsWith("target_setup_error_bugreport"), Mockito.any()))
1307                 .thenReturn(true);
1308         when(mMockBuildProvider.getBuild()).thenReturn(mMockBuildInfo);
1309         doThrow(tse).when(mMockPreparer).setUp(Mockito.any());
1310         when(mMockDevice.getIDevice()).thenReturn(new StubDevice("stub"));
1311 
1312         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1313 
1314         verifyInvokeWithBuild();
1315         verifyMockFailureListeners(tse);
1316 
1317         verifySummaryListener();
1318     }
1319 
1320     /**
1321      * Test the invoke scenario where preparer throws {@link BuildError}
1322      *
1323      * @throws Exception if unexpected error occurs
1324      */
1325     @Test
testInvoke_buildError()1326     public void testInvoke_buildError() throws Throwable {
1327         BuildError exception =
1328                 new BuildError(
1329                         "error", mFakeDescriptor, InfraErrorIdentifier.ARTIFACT_DOWNLOAD_ERROR);
1330         IRemoteTest test = Mockito.mock(IRemoteTest.class);
1331         mStubConfiguration.setTest(test);
1332 
1333         stubEarlyDeviceReleaseExpectation();
1334         stubMockFailureListeners(exception);
1335         stubInvokeWithBuild();
1336 
1337         when(mMockBuildProvider.getBuild()).thenReturn(mMockBuildInfo);
1338         doThrow(exception).when(mMockPreparer).setUp(Mockito.any());
1339         when(mMockDevice.getRecoveryMode()).thenReturn(RecoveryMode.AVAILABLE);
1340         when(mMockDevice.getBugreport()).thenReturn(new ByteArrayInputStreamSource(new byte[0]));
1341 
1342         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1343 
1344         verifyInvokeWithBuild();
1345         verifyMockFailureListeners(exception);
1346 
1347         verifySummaryListener();
1348     }
1349 
1350     /**
1351      * Test the {@link TestInvocation#invoke(IInvocationContext, IConfiguration, IRescheduler,
1352      * ITestInvocationListener[])} scenario when a {@link ITargetPreparer} is part of the config.
1353      */
1354     @Test
testInvoke_tearDown()1355     public void testInvoke_tearDown() throws Throwable {
1356         IRemoteTest test = Mockito.mock(IRemoteTest.class);
1357         ITargetPreparer mockCleaner = Mockito.mock(ITargetPreparer.class);
1358         mStubConfiguration.getTargetPreparers().add(mockCleaner);
1359 
1360         stubEarlyDeviceReleaseExpectation();
1361         stubMockSuccessListeners();
1362         stubNormalInvoke(test);
1363 
1364         when(mockCleaner.isDisabled()).thenReturn(false);
1365         when(mockCleaner.isTearDownDisabled()).thenReturn(false);
1366 
1367         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1368 
1369         verify(mockCleaner, times(2)).isDisabled();
1370         verify(mockCleaner).isTearDownDisabled();
1371         verify(mockCleaner).setUp(Mockito.any());
1372         verify(mockCleaner).tearDown(Mockito.any(), Mockito.isNull());
1373         verify(mMockPreparer).tearDown(Mockito.any(), Mockito.isNull());
1374 
1375         verifyNormalInvoke(test);
1376         verifyMockSuccessListeners();
1377 
1378         verifySummaryListener();
1379     }
1380 
1381     /**
1382      * Test the {@link TestInvocation#invoke(IInvocationContext, IConfiguration, IRescheduler,
1383      * ITestInvocationListener[])} scenario when a {@link ITargetPreparer} is part of the config,
1384      * and the test throws a {@link DeviceNotAvailableException}.
1385      */
1386     @Test
testInvoke_tearDown_deviceNotAvail()1387     public void testInvoke_tearDown_deviceNotAvail() throws Throwable {
1388         DeviceNotAvailableException exception = new DeviceNotAvailableException("ERROR", SERIAL);
1389         IRemoteTest test = Mockito.mock(IRemoteTest.class);
1390         ITargetPreparer mockCleaner = Mockito.mock(ITargetPreparer.class);
1391         mStubConfiguration.getTargetPreparers().add(mockCleaner);
1392 
1393         stubEarlyDeviceReleaseExpectation();
1394         stubMockFailureListeners(exception);
1395         stubNormalInvoke(test);
1396 
1397         when(mockCleaner.isDisabled()).thenReturn(false);
1398         when(mockCleaner.isTearDownDisabled()).thenReturn(false);
1399         doThrow(exception).when(test).run(Mockito.any(), Mockito.any());
1400 
1401         try {
1402             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1403             fail("DeviceNotAvailableException not thrown");
1404         } catch (DeviceNotAvailableException e) {
1405             // expected
1406         }
1407 
1408         verify(mockCleaner, times(2)).isDisabled();
1409         verify(mockCleaner).isTearDownDisabled();
1410         verify(mockCleaner).setUp(Mockito.any());
1411         verify(mockCleaner).tearDown(Mockito.any(), Mockito.eq(exception));
1412         verify(mMockPreparer).tearDown(Mockito.any(), Mockito.eq(exception));
1413 
1414         verifyNormalInvoke(test);
1415         verifyMockFailureListeners(exception);
1416 
1417         verifySummaryListener();
1418     }
1419 
1420     /**
1421      * Test the {@link TestInvocation#invoke(IInvocationContext, IConfiguration, IRescheduler,
1422      * ITestInvocationListener[])} scenario when a {@link ITargetCleaner} is part of the config, and
1423      * the test throws a {@link RuntimeException}.
1424      */
1425     @Test
testInvoke_tearDown_runtime()1426     public void testInvoke_tearDown_runtime() throws Throwable {
1427         RuntimeException exception = new RuntimeException("testInvoke_tearDown_runtime");
1428         mExceptedStatus = FailureStatus.UNSET;
1429         IRemoteTest test = Mockito.mock(IRemoteTest.class);
1430         ITargetPreparer mockCleaner = Mockito.mock(ITargetPreparer.class);
1431         mStubConfiguration.getTargetPreparers().add(mockCleaner);
1432 
1433         stubEarlyDeviceReleaseExpectation();
1434         stubMockFailureListeners(exception);
1435         stubNormalInvoke(test);
1436 
1437         when(mockCleaner.isDisabled()).thenReturn(false);
1438         when(mockCleaner.isTearDownDisabled()).thenReturn(false);
1439         doThrow(exception).when(test).run(Mockito.any(), Mockito.any());
1440 
1441         try {
1442             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1443             fail("RuntimeException not thrown");
1444         } catch (RuntimeException e) {
1445             // expected
1446         }
1447 
1448         verify(mockCleaner, times(2)).isDisabled();
1449         verify(mockCleaner).isTearDownDisabled();
1450         verify(mockCleaner).setUp(Mockito.any());
1451         verify(mockCleaner).tearDown(Mockito.any(), Mockito.eq(exception));
1452         verify(mMockPreparer).tearDown(Mockito.any(), Mockito.eq(exception));
1453 
1454         verifyNormalInvoke(test);
1455         verifyMockFailureListeners(exception);
1456 
1457         verifySummaryListener();
1458     }
1459 
1460     /**
1461      * Test the {@link TestInvocation#invoke(IInvocationContext, IConfiguration, IRescheduler,
1462      * ITestInvocationListener[])} scenario when there is {@link ITestInvocationListener} which
1463      * implements the {@link ILogSaverListener} interface.
1464      */
1465     @Test
testInvoke_logFileSaved()1466     public void testInvoke_logFileSaved() throws Throwable {
1467         List<ITestInvocationListener> listenerList =
1468                 mStubConfiguration.getTestInvocationListeners();
1469         ILogSaverListener logSaverListener = Mockito.mock(ILogSaverListener.class);
1470         listenerList.add(logSaverListener);
1471         mStubConfiguration.setTestInvocationListeners(listenerList);
1472 
1473         IRemoteTest test = Mockito.mock(IRemoteTest.class);
1474 
1475         stubEarlyDeviceReleaseExpectation();
1476         stubMockSuccessListeners();
1477         stubNormalInvoke(test);
1478 
1479         when(logSaverListener.getSummary()).thenReturn(mSummary);
1480 
1481         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1482 
1483         InOrder io = inOrder(logSaverListener);
1484         io.verify(logSaverListener, calls(1)).setLogSaver(mMockLogSaver);
1485         io.verify(logSaverListener, calls(1)).invocationStarted(mStubInvocationMetadata);
1486         io.verify(logSaverListener, calls(1))
1487                 .testLog(
1488                         Mockito.contains(CONFIG_LOG_NAME),
1489                         Mockito.eq(LogDataType.HARNESS_CONFIG),
1490                         (InputStreamSource) Mockito.any());
1491         io.verify(logSaverListener, calls(1))
1492                 .testLogSaved(
1493                         Mockito.contains(CONFIG_LOG_NAME),
1494                         Mockito.eq(LogDataType.HARNESS_CONFIG),
1495                         (InputStreamSource) Mockito.any(),
1496                         (LogFile) Mockito.any());
1497         io.verify(logSaverListener, calls(1))
1498                 .logAssociation(Mockito.contains(CONFIG_LOG_NAME), Mockito.any());
1499         io.verify(logSaverListener, calls(1))
1500                 .testLog(
1501                         Mockito.startsWith(LOGCAT_NAME_SETUP),
1502                         Mockito.eq(LogDataType.LOGCAT),
1503                         (InputStreamSource) Mockito.any());
1504         io.verify(logSaverListener, calls(1))
1505                 .testLogSaved(
1506                         Mockito.startsWith(LOGCAT_NAME_SETUP),
1507                         Mockito.eq(LogDataType.LOGCAT),
1508                         (InputStreamSource) Mockito.any(),
1509                         (LogFile) Mockito.any());
1510         io.verify(logSaverListener, calls(1))
1511                 .logAssociation(Mockito.startsWith(LOGCAT_NAME_SETUP), Mockito.any());
1512         io.verify(logSaverListener, calls(1))
1513                 .testLog(
1514                         Mockito.startsWith(LOGCAT_NAME_TEST),
1515                         Mockito.eq(LogDataType.LOGCAT),
1516                         (InputStreamSource) Mockito.any());
1517         io.verify(logSaverListener, calls(1))
1518                 .testLogSaved(
1519                         Mockito.startsWith(LOGCAT_NAME_TEST),
1520                         Mockito.eq(LogDataType.LOGCAT),
1521                         (InputStreamSource) Mockito.any(),
1522                         (LogFile) Mockito.any());
1523         io.verify(logSaverListener, calls(1))
1524                 .logAssociation(Mockito.startsWith(LOGCAT_NAME_TEST), Mockito.any());
1525         io.verify(logSaverListener, calls(1))
1526                 .testLog(
1527                         Mockito.startsWith(LOGCAT_NAME_TEARDOWN),
1528                         Mockito.eq(LogDataType.LOGCAT),
1529                         (InputStreamSource) Mockito.any());
1530         io.verify(logSaverListener, calls(1))
1531                 .testLogSaved(
1532                         Mockito.startsWith(LOGCAT_NAME_TEARDOWN),
1533                         Mockito.eq(LogDataType.LOGCAT),
1534                         (InputStreamSource) Mockito.any(),
1535                         (LogFile) Mockito.any());
1536         io.verify(logSaverListener, calls(1))
1537                 .logAssociation(Mockito.startsWith(LOGCAT_NAME_TEARDOWN), Mockito.any());
1538         io.verify(logSaverListener, calls(1))
1539                 .testLog(
1540                         Mockito.eq(TestInvocation.TRADEFED_LOG_NAME),
1541                         Mockito.eq(LogDataType.HOST_LOG),
1542                         (InputStreamSource) Mockito.any());
1543         io.verify(logSaverListener, calls(1))
1544                 .testLogSaved(
1545                         Mockito.eq(TestInvocation.TRADEFED_LOG_NAME),
1546                         Mockito.eq(LogDataType.HOST_LOG),
1547                         (InputStreamSource) Mockito.any(),
1548                         (LogFile) Mockito.any());
1549         io.verify(logSaverListener, calls(1))
1550                 .logAssociation(Mockito.eq(TestInvocation.TRADEFED_LOG_NAME), Mockito.any());
1551         io.verify(logSaverListener, calls(1)).invocationEnded(Mockito.anyLong());
1552 
1553         verify(test).run(Mockito.any(), Mockito.any());
1554         verify(mMockPreparer).tearDown(Mockito.any(), Mockito.isNull());
1555         verify(logSaverListener, times(2)).getSummary();
1556 
1557         verifyNormalInvoke(test);
1558         verifyMockSuccessListeners();
1559 
1560         assertEquals(2, mUriCapture.getValue().size());
1561     }
1562 
1563     @Test
testShouldSkipBugreportError_buildError()1564     public void testShouldSkipBugreportError_buildError() throws Throwable {
1565         Throwable e =
1566                 new BuildError(
1567                         "error", mFakeDescriptor, InfraErrorIdentifier.ARTIFACT_DOWNLOAD_ERROR);
1568         assertTrue(mTestInvocation.shouldSkipBugreportError(e));
1569     }
1570 
1571     @Test
testShouldSkipBugreportError_wifiOptionError()1572     public void testShouldSkipBugreportError_wifiOptionError() throws Throwable {
1573         Throwable e =
1574                 new TargetSetupError(
1575                         "wifi-network not specified",
1576                         InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR);
1577         assertTrue(mTestInvocation.shouldSkipBugreportError(e));
1578     }
1579 
1580     @Test
testShouldSkipBugreportError_wifiConnectionError()1581     public void testShouldSkipBugreportError_wifiConnectionError() throws Throwable {
1582         Throwable e =
1583                 new TargetSetupError("wifi-network not specified", InfraErrorIdentifier.NO_WIFI);
1584         assertFalse(mTestInvocation.shouldSkipBugreportError(e));
1585     }
1586 
1587     @Test
testShouldSkipBugreportError_internalConfigError()1588     public void testShouldSkipBugreportError_internalConfigError() throws Throwable {
1589         Throwable e =
1590                 new HarnessRuntimeException(
1591                         "Invalid Configuration", InfraErrorIdentifier.INTERNAL_CONFIG_ERROR);
1592         assertFalse(mTestInvocation.shouldSkipBugreportError(e));
1593     }
1594 
1595     /** Test the test-tag is set when the IBuildInfo's test-tag is not. */
1596     @Ignore
1597     @Test
testInvoke_testtag()1598     public void testInvoke_testtag() throws Throwable {
1599         /*
1600         String[] commandLine = {"run", "empty"};
1601         mStubConfiguration.setCommandLine(commandLine);
1602         mStubConfiguration.getCommandOptions().setTestTag("not-default");
1603         setEarlyDeviceReleaseExpectation();
1604         setupInvoke();
1605         EasyMock.expect(mMockDevice.getLogcat()).andReturn(mLogcatSetupSource);
1606         EasyMock.expect(mMockDevice.getLogcat()).andReturn(mLogcatTestSource);
1607         EasyMock.expect(mMockDevice.getLogcat()).andReturn(mLogcatTeardownSource);
1608         mMockDevice.clearLogcat();
1609         EasyMock.expectLastCall().times(3);
1610         EasyMock.expect(mMockLogger.getLog()).andReturn(mHostLogSource);
1611         mMockBuildInfo.setDeviceSerial(SERIAL);
1612         mMockBuildProvider.cleanUp(mMockBuildInfo);
1613         setupMockSuccessListeners();
1614         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
1615         mMockBuildInfo.addBuildAttribute("command_line_args", "run empty");
1616         mMockPreparer.setUp(EasyMock.anyObject());
1617         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
1618         // Default build is "stub" so we set the test-tag
1619         mMockBuildInfo.setTestTag(EasyMock.eq("not-default"));
1620         EasyMock.expectLastCall();
1621         EasyMock.expect(mMockBuildInfo.getTestTag()).andStubReturn("stub");
1622         mMockLogRegistry.unregisterLogger();
1623         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
1624         mMockLogger.closeLog();
1625         replayMocks();
1626         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1627         verifyMocks();
1628         */
1629     }
1630 
1631     /**
1632      * Test the test-tag of the IBuildInfo is not modified when the CommandOption default test-tag
1633      * is not modified.
1634      */
1635     @Ignore
1636     @Test
testInvoke_testtag_notset()1637     public void testInvoke_testtag_notset() throws Throwable {
1638         /*
1639         String[] commandLine = {"run", "empty"};
1640         mStubConfiguration.setCommandLine(commandLine);
1641         setEarlyDeviceReleaseExpectation();
1642         setupInvoke();
1643         EasyMock.expect(mMockDevice.getLogcat()).andReturn(mLogcatSetupSource);
1644         EasyMock.expect(mMockDevice.getLogcat()).andReturn(mLogcatTestSource);
1645         EasyMock.expect(mMockDevice.getLogcat()).andReturn(mLogcatTeardownSource);
1646         mMockDevice.clearLogcat();
1647         EasyMock.expectLastCall().times(3);
1648         EasyMock.expect(mMockLogger.getLog()).andReturn(mHostLogSource);
1649         mMockBuildInfo.setDeviceSerial(SERIAL);
1650         mMockBuildProvider.cleanUp(mMockBuildInfo);
1651         setupMockSuccessListeners();
1652         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
1653         mMockBuildInfo.addBuildAttribute("command_line_args", "run empty");
1654         mMockPreparer.setUp(EasyMock.anyObject());
1655         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
1656         EasyMock.expect(mMockBuildInfo.getTestTag()).andStubReturn("buildprovidertesttag");
1657         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
1658         mMockLogRegistry.unregisterLogger();
1659         mMockLogger.closeLog();
1660         replayMocks();
1661         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1662         verifyMocks();
1663         */
1664     }
1665 
1666     /**
1667      * Test the test-tag of the IBuildInfo is not set and Command Option is not set either. A
1668      * default 'stub' test-tag is set to ensure reporting is done.
1669      */
1670     @Ignore
1671     @Test
testInvoke_notesttag()1672     public void testInvoke_notesttag() throws Throwable {
1673         /*
1674         String[] commandLine = {"run", "empty"};
1675         mStubConfiguration.setCommandLine(commandLine);
1676         setEarlyDeviceReleaseExpectation();
1677         setupInvoke();
1678         EasyMock.expect(mMockDevice.getLogcat()).andReturn(mLogcatSetupSource);
1679         EasyMock.expect(mMockDevice.getLogcat()).andReturn(mLogcatTestSource);
1680         EasyMock.expect(mMockDevice.getLogcat()).andReturn(mLogcatTeardownSource);
1681         mMockDevice.clearLogcat();
1682         EasyMock.expectLastCall().times(3);
1683         EasyMock.expect(mMockLogger.getLog()).andReturn(mHostLogSource);
1684         mMockBuildInfo.setDeviceSerial(SERIAL);
1685         mMockBuildProvider.cleanUp(mMockBuildInfo);
1686         setupMockSuccessListeners();
1687         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
1688         mMockBuildInfo.addBuildAttribute("command_line_args", "run empty");
1689         mMockPreparer.setUp(EasyMock.anyObject());
1690         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
1691         EasyMock.expect(mMockBuildInfo.getTestTag()).andStubReturn(null);
1692         mMockBuildInfo.setTestTag(EasyMock.eq("stub"));
1693         EasyMock.expectLastCall();
1694         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
1695         mMockLogRegistry.unregisterLogger();
1696         mMockLogger.closeLog();
1697         replayMocks();
1698         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1699         verifyMocks();
1700         */
1701     }
1702 
1703     /**
1704      * Test the injection of test-tag from TestInvocation to the build provider via the {@link
1705      * IInvocationContextReceiver}.
1706      */
1707     @Ignore
1708     @Test
testInvoke_buildProviderNeedTestTag()1709     public void testInvoke_buildProviderNeedTestTag() throws Throwable {
1710         /*
1711         final String testTag = "THISISTHETAG";
1712         String[] commandLine = {"run", "empty"};
1713         mStubConfiguration.setCommandLine(commandLine);
1714         ICommandOptions commandOption = new CommandOptions();
1715         commandOption.setTestTag(testTag);
1716         IFakeBuildProvider mockProvider = EasyMock.createMock(IFakeBuildProvider.class);
1717         mStubConfiguration.setBuildProvider(mockProvider);
1718         mStubConfiguration.setCommandOptions(commandOption);
1719         setEarlyDeviceReleaseExpectation();
1720         setupInvoke();
1721         EasyMock.expect(mMockDevice.getLogcat()).andReturn(mLogcatSetupSource);
1722         EasyMock.expect(mMockDevice.getLogcat()).andReturn(mLogcatTestSource);
1723         EasyMock.expect(mMockDevice.getLogcat()).andReturn(mLogcatTeardownSource);
1724         mMockDevice.clearLogcat();
1725         EasyMock.expectLastCall().times(3);
1726         EasyMock.expect(mMockLogger.getLog()).andReturn(mHostLogSource);
1727         mMockBuildInfo.setDeviceSerial(SERIAL);
1728         setupMockSuccessListeners();
1729         mMockBuildInfo.addBuildAttribute("command_line_args", "run empty");
1730         mMockPreparer.setUp(EasyMock.anyObject());
1731         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
1732         EasyMock.expect(mMockBuildInfo.getTestTag()).andStubReturn(null);
1733         // Validate proper tag is set on the build.
1734         mMockBuildInfo.setTestTag(EasyMock.eq(testTag));
1735         mockProvider.setInvocationContext((IInvocationContext) EasyMock.anyObject());
1736         EasyMock.expect(mockProvider.getBuild(mMockDevice)).andReturn(mMockBuildInfo);
1737         EasyMock.expectLastCall().anyTimes();
1738         mockProvider.cleanUp(mMockBuildInfo);
1739         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
1740         mMockLogRegistry.unregisterLogger();
1741         mMockLogger.closeLog();
1742 
1743         replayMocks(mockProvider);
1744         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1745         verifyMocks(mockProvider);
1746         */
1747     }
1748 
1749     /**
1750      * Test the {@link TestInvocation#invoke(IInvocationContext, IConfiguration, IRescheduler,
1751      * ITestInvocationListener[])} scenario with {@link IShardableTest}.
1752      */
1753     @Ignore
1754     @Test
testInvoke_shardableTest_legacy()1755     public void testInvoke_shardableTest_legacy() throws Throwable {
1756         /*
1757         TestInformation info =
1758                 TestInformation.newBuilder().setInvocationContext(mStubInvocationMetadata).build();
1759         mStubConfiguration.setConfigurationObject(ShardHelper.SHARED_TEST_INFORMATION, info);
1760         String command = "empty --test-tag t";
1761         String[] commandLine = {"empty", "--test-tag", "t"};
1762         int shardCount = 2;
1763         IShardableTest test = EasyMock.createMock(IShardableTest.class);
1764         List<IRemoteTest> shards = new ArrayList<>();
1765         IRemoteTest shard1 = EasyMock.createMock(IRemoteTest.class);
1766         IRemoteTest shard2 = EasyMock.createMock(IRemoteTest.class);
1767         shards.add(shard1);
1768         shards.add(shard2);
1769         EasyMock.expect(test.split(EasyMock.isNull(), EasyMock.anyObject())).andReturn(shards);
1770         mStubConfiguration.setTest(test);
1771         mStubConfiguration.setCommandLine(commandLine);
1772         mMockBuildProvider.cleanUp(mMockBuildInfo);
1773         // The keystore is cloned for each shard.
1774         EasyMock.expect(mGlobalConfiguration.getKeyStoreFactory())
1775                 .andReturn(new StubKeyStoreFactory())
1776                 .times(2);
1777         setupInvoke();
1778         EasyMock.reset(mMockLogger, mMockLogRegistry);
1779         mMockLogRegistry.registerLogger(mMockLogger);
1780         mMockLogger.init();
1781         mMockLogger.closeLog();
1782         mMockLogRegistry.unregisterLogger();
1783         mMockLogSaver.invocationStarted(mStubInvocationMetadata);
1784         mMockLogSaver.invocationEnded(0L);
1785         setupNShardInvocation(shardCount, command);
1786         // Ensure that the host_log gets logged after sharding.
1787         EasyMock.expect(mMockLogger.getLog()).andReturn(mHostLogSource);
1788         String logName = "host_log_before_sharding";
1789         mMockTestListener.testLog(
1790                 EasyMock.eq(logName), EasyMock.eq(LogDataType.HOST_LOG), EasyMock.anyObject());
1791         mMockSummaryListener.testLog(
1792                 EasyMock.eq(logName), EasyMock.eq(LogDataType.HOST_LOG), EasyMock.anyObject());
1793         EasyMock.expect(
1794                         mMockLogSaver.saveLogData(
1795                                 EasyMock.eq(logName),
1796                                 EasyMock.eq(LogDataType.HOST_LOG),
1797                                 EasyMock.anyObject()))
1798                 .andReturn(new LogFile(PATH, URL, LogDataType.HOST_LOG));
1799         mMockLogRegistry.unregisterLogger();
1800         EasyMock.expectLastCall();
1801         mMockLogger.closeLog();
1802         EasyMock.expectLastCall();
1803 
1804         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
1805         replayMocks(test, mockRescheduler, shard1, shard2, mGlobalConfiguration);
1806         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1807         verifyMocks(test, mockRescheduler, shard1, shard2, mGlobalConfiguration);
1808         */
1809     }
1810 
1811     /** Test that the before sharding log is properly carried even with auto-retry. */
1812     @Ignore
1813     @Test
testInvoke_shardableTest_autoRetry()1814     public void testInvoke_shardableTest_autoRetry() throws Throwable {
1815         /*
1816         TestInformation info =
1817                 TestInformation.newBuilder().setInvocationContext(mStubInvocationMetadata).build();
1818         mStubConfiguration.setConfigurationObject(ShardHelper.SHARED_TEST_INFORMATION, info);
1819         List<ITestInvocationListener> listenerList =
1820                 mStubConfiguration.getTestInvocationListeners();
1821         ILogSaverListener logSaverListener = EasyMock.createMock(ILogSaverListener.class);
1822         listenerList.add(logSaverListener);
1823         mStubConfiguration.setTestInvocationListeners(listenerList);
1824 
1825         logSaverListener.setLogSaver(mMockLogSaver);
1826         logSaverListener.invocationStarted(mStubInvocationMetadata);
1827 
1828         String command = "empty --test-tag t";
1829         String[] commandLine = {"empty", "--test-tag", "t"};
1830         int shardCount = 2;
1831         IShardableTest test = EasyMock.createMock(IShardableTest.class);
1832         List<IRemoteTest> shards = new ArrayList<>();
1833         IRemoteTest shard1 = EasyMock.createMock(IRemoteTest.class);
1834         IRemoteTest shard2 = EasyMock.createMock(IRemoteTest.class);
1835         shards.add(shard1);
1836         shards.add(shard2);
1837         EasyMock.expect(test.split(EasyMock.isNull(), EasyMock.anyObject())).andReturn(shards);
1838         mStubConfiguration.setTest(test);
1839         mStubConfiguration.setCommandLine(commandLine);
1840 
1841         IRetryDecision decision = mStubConfiguration.getRetryDecision();
1842         OptionSetter decisionSetter = new OptionSetter(decision);
1843         decisionSetter.setOptionValue("auto-retry", "true");
1844         decisionSetter.setOptionValue("max-testcase-run-count", "2");
1845         decisionSetter.setOptionValue("retry-strategy", "RETRY_ANY_FAILURE");
1846         mMockBuildProvider.cleanUp(mMockBuildInfo);
1847         // The keystore is cloned for each shard.
1848         EasyMock.expect(mGlobalConfiguration.getKeyStoreFactory())
1849                 .andReturn(new StubKeyStoreFactory())
1850                 .times(2);
1851         setupInvoke();
1852         EasyMock.reset(mMockLogger, mMockLogRegistry);
1853         mMockLogRegistry.registerLogger(mMockLogger);
1854         mMockLogger.init();
1855         mMockLogger.closeLog();
1856         mMockLogRegistry.unregisterLogger();
1857         mMockLogSaver.invocationStarted(mStubInvocationMetadata);
1858         mMockLogSaver.invocationEnded(0L);
1859         setupNShardInvocation(shardCount, command);
1860         // Ensure that the host_log gets logged after sharding.
1861         EasyMock.expect(mMockLogger.getLog()).andReturn(mHostLogSource);
1862         String logName = "host_log_before_sharding";
1863         LogFile loggedFile = new LogFile(PATH, URL, LogDataType.HOST_LOG);
1864         for (ITestInvocationListener listener : listenerList) {
1865             listener.testLog(
1866                     EasyMock.eq(logName), EasyMock.eq(LogDataType.HOST_LOG), EasyMock.anyObject());
1867         }
1868         EasyMock.expect(
1869                         mMockLogSaver.saveLogData(
1870                                 EasyMock.eq(logName),
1871                                 EasyMock.eq(LogDataType.HOST_LOG),
1872                                 EasyMock.anyObject()))
1873                 .andReturn(loggedFile);
1874         logSaverListener.logAssociation(logName, loggedFile);
1875         mMockLogRegistry.unregisterLogger();
1876         EasyMock.expectLastCall();
1877         mMockLogger.closeLog();
1878         EasyMock.expectLastCall();
1879 
1880         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
1881         replayMocks(test, mockRescheduler, shard1, shard2, mGlobalConfiguration, logSaverListener);
1882         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1883         verifyMocks(test, mockRescheduler, shard1, shard2, mGlobalConfiguration, logSaverListener);
1884         */
1885     }
1886 
1887     /**
1888      * Test that {@link TestInvocation#logDeviceBatteryLevel(IInvocationContext, String)} is not
1889      * adding battery information for placeholder device.
1890      */
1891     @Ignore
1892     @Test
testLogDeviceBatteryLevel_placeholderDevice()1893     public void testLogDeviceBatteryLevel_placeholderDevice() {
1894         /*
1895         final String fakeEvent = "event";
1896         IInvocationContext context = new InvocationContext();
1897         ITestDevice device1 = EasyMock.createMock(ITestDevice.class);
1898         EasyMock.expect(device1.getIDevice()).andReturn(new StubDevice("stub"));
1899         context.addAllocatedDevice("device1", device1);
1900         EasyMock.replay(device1);
1901         mTestInvocation.logDeviceBatteryLevel(context, fakeEvent);
1902         EasyMock.verify(device1);
1903         assertEquals(0, context.getAttributes().size());
1904         */
1905     }
1906 
1907     /**
1908      * Test that {@link TestInvocation#logDeviceBatteryLevel(IInvocationContext, String)} is adding
1909      * battery information for physical real device.
1910      */
1911     @Ignore
1912     @Test
testLogDeviceBatteryLevel_physicalDevice()1913     public void testLogDeviceBatteryLevel_physicalDevice() {
1914         /*
1915         final String fakeEvent = "event";
1916         IInvocationContext context = new InvocationContext();
1917         ITestDevice device1 = EasyMock.createMock(ITestDevice.class);
1918         EasyMock.expect(device1.getIDevice()).andReturn(EasyMock.createMock(IDevice.class));
1919         EasyMock.expect(device1.getSerialNumber()).andReturn("serial1");
1920         EasyMock.expect(device1.getBattery()).andReturn(50);
1921         context.addAllocatedDevice("device1", device1);
1922         context.addDeviceBuildInfo("device1", new BuildInfo());
1923         EasyMock.replay(device1);
1924         mTestInvocation.logDeviceBatteryLevel(context, fakeEvent);
1925         EasyMock.verify(device1);
1926         assertEquals(1, context.getBuildInfo("device1").getBuildAttributes().size());
1927         assertEquals(
1928                 "50",
1929                 context.getBuildInfo("device1")
1930                         .getBuildAttributes()
1931                         .get("serial1-battery-" + fakeEvent));
1932         */
1933     }
1934 
1935     /**
1936      * Test that {@link TestInvocation#logDeviceBatteryLevel(IInvocationContext, String)} is adding
1937      * battery information for multiple physical real device.
1938      */
1939     @Ignore
1940     @Test
testLogDeviceBatteryLevel_physicalDevice_multi()1941     public void testLogDeviceBatteryLevel_physicalDevice_multi() {
1942         /*
1943         final String fakeEvent = "event";
1944         IInvocationContext context = new InvocationContext();
1945         ITestDevice device1 = EasyMock.createMock(ITestDevice.class);
1946         EasyMock.expect(device1.getIDevice()).andReturn(EasyMock.createMock(IDevice.class));
1947         EasyMock.expect(device1.getSerialNumber()).andReturn("serial1");
1948         EasyMock.expect(device1.getBattery()).andReturn(50);
1949 
1950         ITestDevice device2 = EasyMock.createMock(ITestDevice.class);
1951         EasyMock.expect(device2.getIDevice()).andReturn(EasyMock.createMock(IDevice.class));
1952         EasyMock.expect(device2.getSerialNumber()).andReturn("serial2");
1953         EasyMock.expect(device2.getBattery()).andReturn(55);
1954 
1955         context.addAllocatedDevice("device1", device1);
1956         context.addDeviceBuildInfo("device1", new BuildInfo());
1957         context.addAllocatedDevice("device2", device2);
1958         context.addDeviceBuildInfo("device2", new BuildInfo());
1959         EasyMock.replay(device1, device2);
1960         mTestInvocation.logDeviceBatteryLevel(context, fakeEvent);
1961         EasyMock.verify(device1, device2);
1962         assertEquals(1, context.getBuildInfo("device1").getBuildAttributes().size());
1963         assertEquals(1, context.getBuildInfo("device2").getBuildAttributes().size());
1964         assertEquals(
1965                 "50",
1966                 context.getBuildInfo("device1")
1967                         .getBuildAttributes()
1968                         .get("serial1-battery-" + fakeEvent));
1969         assertEquals(
1970                 "55",
1971                 context.getBuildInfo("device2")
1972                         .getBuildAttributes()
1973                         .get("serial2-battery-" + fakeEvent));
1974         */
1975     }
1976 
1977     /**
1978      * Test that {@link TestInvocation#logDeviceBatteryLevel(IInvocationContext, String)} is adding
1979      * battery information for multiple physical real device, and ignore stub device if any.
1980      */
1981     @Test
testLogDeviceBatteryLevel_physicalDevice_stub_multi()1982     public void testLogDeviceBatteryLevel_physicalDevice_stub_multi() {
1983         /*
1984         final String fakeEvent = "event";
1985         IInvocationContext context = new InvocationContext();
1986         ITestDevice device1 = EasyMock.createMock(ITestDevice.class);
1987         EasyMock.expect(device1.getIDevice()).andReturn(EasyMock.createMock(IDevice.class));
1988         EasyMock.expect(device1.getSerialNumber()).andReturn("serial1");
1989         EasyMock.expect(device1.getBattery()).andReturn(50);
1990 
1991         ITestDevice device2 = EasyMock.createMock(ITestDevice.class);
1992         EasyMock.expect(device2.getIDevice()).andReturn(EasyMock.createMock(IDevice.class));
1993         EasyMock.expect(device2.getSerialNumber()).andReturn("serial2");
1994         EasyMock.expect(device2.getBattery()).andReturn(50);
1995 
1996         ITestDevice device3 = EasyMock.createMock(ITestDevice.class);
1997         EasyMock.expect(device3.getIDevice()).andReturn(new StubDevice("stub-3"));
1998 
1999         ITestDevice device4 = EasyMock.createMock(ITestDevice.class);
2000         EasyMock.expect(device4.getIDevice()).andReturn(new TcpDevice("tcp-4"));
2001 
2002         context.addAllocatedDevice("device1", device1);
2003         context.addDeviceBuildInfo("device1", new BuildInfo());
2004         context.addAllocatedDevice("device2", device2);
2005         context.addDeviceBuildInfo("device2", new BuildInfo());
2006         context.addAllocatedDevice("device3", device3);
2007         context.addAllocatedDevice("device4", device4);
2008         EasyMock.replay(device1, device2, device3, device4);
2009         mTestInvocation.logDeviceBatteryLevel(context, fakeEvent);
2010         EasyMock.verify(device1, device2, device3, device4);
2011         assertEquals(1, context.getBuildInfo("device1").getBuildAttributes().size());
2012         assertEquals(1, context.getBuildInfo("device2").getBuildAttributes().size());
2013         assertEquals(
2014                 "50",
2015                 context.getBuildInfo("device1")
2016                         .getBuildAttributes()
2017                         .get("serial1-battery-" + fakeEvent));
2018         assertEquals(
2019                 "50",
2020                 context.getBuildInfo("device2")
2021                         .getBuildAttributes()
2022                         .get("serial2-battery-" + fakeEvent));
2023         */
2024     }
2025 
2026     /**
2027      * Test when a {@link IDeviceBuildInfo} is passing through we do not attempt to add any external
2028      * directories when there is none coming from environment.
2029      */
2030     @Ignore
2031     @Test
testInvoke_deviceInfoBuild_noEnv()2032     public void testInvoke_deviceInfoBuild_noEnv() throws Throwable {
2033         /*
2034         mTestInvocation =
2035                 new TestInvocation() {
2036                     @Override
2037                     ILogRegistry getLogRegistry() {
2038                         return mMockLogRegistry;
2039                     }
2040 
2041                     @Override
2042                     public IInvocationExecution createInvocationExec(RunMode mode) {
2043                         return new InvocationExecution() {
2044                             @Override
2045                             protected IShardHelper createShardHelper() {
2046                                 return new ShardHelper();
2047                             }
2048 
2049                             @Override
2050                             File getExternalTestCasesDirs(EnvVariable envVar) {
2051                                 // Return empty list to ensure we do not have any environment loaded
2052                                 return null;
2053                             }
2054 
2055                             @Override
2056                             protected String getAdbVersion() {
2057                                 return null;
2058                             }
2059 
2060                             @Override
2061                             protected void logHostAdb(IConfiguration config, ITestLogger logger) {
2062                                 // inop for the common test case.
2063                             }
2064 
2065                             @Override
2066                             protected void collectAutoInfo(
2067                                     IConfiguration config, TestInformation info)
2068                                     throws DeviceNotAvailableException {
2069                                 // Inop for the command test case.
2070                             }
2071                         };
2072                     }
2073 
2074                     @Override
2075                     protected void setExitCode(ExitCode code, Throwable stack) {
2076                         // empty on purpose
2077                     }
2078 
2079                     @Override
2080                     protected void applyAutomatedReporters(IConfiguration config) {
2081                         // Empty on purpose
2082                     }
2083 
2084                     @Override
2085                     protected void addInvocationMetric(InvocationMetricKey key, long value) {}
2086 
2087                     @Override
2088                     protected void addInvocationMetric(InvocationMetricKey key, String value) {}
2089                 };
2090         mMockBuildInfo = EasyMock.createMock(IDeviceBuildInfo.class);
2091         EasyMock.expect(mMockBuildInfo.getProperties()).andStubReturn(new HashSet<>());
2092 
2093         IRemoteTest test = EasyMock.createNiceMock(IRemoteTest.class);
2094         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
2095 
2096         File tmpTestsDir = FileUtil.createTempDir("invocation-tests-dir");
2097         try {
2098             EasyMock.expect(((IDeviceBuildInfo) mMockBuildInfo).getTestsDir())
2099                     .andReturn(tmpTestsDir);
2100             setupMockSuccessListeners();
2101             setEarlyDeviceReleaseExpectation();
2102             setupNormalInvoke(test);
2103             EasyMock.replay(mockRescheduler);
2104             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
2105             verifyMocks(mockRescheduler);
2106             verifySummaryListener();
2107         } finally {
2108             FileUtil.recursiveDelete(tmpTestsDir);
2109         }
2110         */
2111     }
2112 
2113     /**
2114      * Test when a {@link IDeviceBuildInfo} is passing through we attempt to add the external
2115      * directories to it when they are available.
2116      */
2117     @Ignore
2118     @Test
testInvoke_deviceInfoBuild_withEnv()2119     public void testInvoke_deviceInfoBuild_withEnv() throws Throwable {
2120         /*
2121         File tmpTestsDir = FileUtil.createTempDir("invocation-tests-dir");
2122         File tmpExternalTestsDir = FileUtil.createTempDir("external-tf-dir");
2123         File tmpTestsFile = FileUtil.createTempFile("testsfile", "txt", tmpExternalTestsDir);
2124         try {
2125             mTestInvocation =
2126                     new TestInvocation() {
2127                         @Override
2128                         ILogRegistry getLogRegistry() {
2129                             return mMockLogRegistry;
2130                         }
2131 
2132                         @Override
2133                         public IInvocationExecution createInvocationExec(RunMode mode) {
2134                             return new InvocationExecution() {
2135                                 @Override
2136                                 protected IShardHelper createShardHelper() {
2137                                     return new ShardHelper();
2138                                 }
2139 
2140                                 @Override
2141                                 File getExternalTestCasesDirs(EnvVariable envVar) {
2142                                     if (EnvVariable.ANDROID_TARGET_OUT_TESTCASES.equals(envVar)) {
2143                                         return tmpExternalTestsDir;
2144                                     }
2145                                     return null;
2146                                 }
2147 
2148                                 @Override
2149                                 protected String getAdbVersion() {
2150                                     return null;
2151                                 }
2152 
2153                                 @Override
2154                                 protected void logHostAdb(
2155                                         IConfiguration config, ITestLogger logger) {
2156                                     // inop for the common test case.
2157                                 }
2158 
2159                                 @Override
2160                                 protected void collectAutoInfo(
2161                                         IConfiguration config, TestInformation info)
2162                                         throws DeviceNotAvailableException {
2163                                     // Inop for the command test case.
2164                                 }
2165                             };
2166                         }
2167 
2168                         @Override
2169                         protected void applyAutomatedReporters(IConfiguration config) {
2170                             // Empty on purpose
2171                         }
2172 
2173                         @Override
2174                         protected void setExitCode(ExitCode code, Throwable stack) {
2175                             // empty on purpose
2176                         }
2177 
2178                         @Override
2179                         protected void addInvocationMetric(InvocationMetricKey key, long value) {}
2180 
2181                         @Override
2182                         protected void addInvocationMetric(InvocationMetricKey key, String value) {}
2183                     };
2184             mMockBuildInfo = EasyMock.createMock(IDeviceBuildInfo.class);
2185             IRemoteTest test = EasyMock.createNiceMock(IRemoteTest.class);
2186             ITargetPreparer mockCleaner = EasyMock.createMock(ITargetPreparer.class);
2187             EasyMock.expect(mockCleaner.isDisabled()).andReturn(false).times(2);
2188             EasyMock.expect(mockCleaner.isTearDownDisabled()).andReturn(false);
2189             mockCleaner.setUp(EasyMock.anyObject());
2190             mockCleaner.tearDown(EasyMock.anyObject(), EasyMock.isNull());
2191             mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
2192             mStubConfiguration.getTargetPreparers().add(mockCleaner);
2193 
2194             mMockBuildInfo.setFile(
2195                     EasyMock.contains(BuildInfoFileKey.TARGET_LINKED_DIR.getFileKey()),
2196                     EasyMock.anyObject(),
2197                     EasyMock.eq("v1"));
2198             EasyMock.expect(((IDeviceBuildInfo) mMockBuildInfo).getTestsDir())
2199                     .andReturn(tmpTestsDir);
2200             EasyMock.expect(mMockBuildInfo.getProperties()).andStubReturn(new HashSet<>());
2201 
2202             setupMockSuccessListeners();
2203             setEarlyDeviceReleaseExpectation();
2204             setupNormalInvoke(test);
2205             EasyMock.replay(mockCleaner, mockRescheduler);
2206             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
2207             verifyMocks(mockCleaner, mockRescheduler);
2208             verifySummaryListener();
2209             // Check that the external directory was copied in the testsDir.
2210             assertTrue(tmpTestsDir.listFiles().length == 1);
2211             // external-tf-dir - the symlink is the original file name + randomized sequence
2212             assertTrue(
2213                     tmpTestsDir
2214                             .listFiles()[0]
2215                             .getName()
2216                             .startsWith(BuildInfoFileKey.TARGET_LINKED_DIR.getFileKey()));
2217             // testsfile.txt
2218             assertTrue(tmpTestsDir.listFiles()[0].listFiles().length == 1);
2219             assertEquals(
2220                     tmpTestsFile.getName(), tmpTestsDir.listFiles()[0].listFiles()[0].getName());
2221         } finally {
2222             FileUtil.recursiveDelete(tmpTestsDir);
2223             FileUtil.recursiveDelete(tmpExternalTestsDir);
2224         }
2225         */
2226     }
2227 
2228     /**
2229      * Test when a {@link IDeviceBuildInfo} is passing through we do not attempt to add the external
2230      * directories to it, since {@link BuildInfoProperties} is set to skip the linking.
2231      */
2232     @Ignore
2233     @Test
testInvoke_deviceInfoBuild_withEnv_andSkipProperty()2234     public void testInvoke_deviceInfoBuild_withEnv_andSkipProperty() throws Throwable {
2235         /*
2236         File tmpTestsDir = FileUtil.createTempDir("invocation-tests-dir");
2237         File tmpExternalTestsDir = FileUtil.createTempDir("external-tf-dir");
2238         FileUtil.createTempFile("testsfile", "txt", tmpExternalTestsDir);
2239         try {
2240             mTestInvocation =
2241                     new TestInvocation() {
2242                         @Override
2243                         ILogRegistry getLogRegistry() {
2244                             return mMockLogRegistry;
2245                         }
2246 
2247                         @Override
2248                         public IInvocationExecution createInvocationExec(RunMode mode) {
2249                             return new InvocationExecution() {
2250                                 @Override
2251                                 protected IShardHelper createShardHelper() {
2252                                     return new ShardHelper();
2253                                 }
2254 
2255                                 @Override
2256                                 File getExternalTestCasesDirs(EnvVariable envVar) {
2257                                     return tmpExternalTestsDir;
2258                                 }
2259 
2260                                 @Override
2261                                 protected String getAdbVersion() {
2262                                     return null;
2263                                 }
2264 
2265                                 @Override
2266                                 protected void logHostAdb(
2267                                         IConfiguration config, ITestLogger logger) {
2268                                     // inop for the common test case.
2269                                 }
2270 
2271                                 @Override
2272                                 protected void collectAutoInfo(
2273                                         IConfiguration config, TestInformation info)
2274                                         throws DeviceNotAvailableException {
2275                                     // Inop for the command test case.
2276                                 }
2277                             };
2278                         }
2279 
2280                         @Override
2281                         protected void setExitCode(ExitCode code, Throwable stack) {
2282                             // empty on purpose
2283                         }
2284 
2285                         @Override
2286                         protected void applyAutomatedReporters(IConfiguration config) {
2287                             // Empty on purpose
2288                         }
2289 
2290                         @Override
2291                         protected void addInvocationMetric(InvocationMetricKey key, long value) {}
2292 
2293                         @Override
2294                         protected void addInvocationMetric(InvocationMetricKey key, String value) {}
2295                     };
2296             mMockBuildInfo = EasyMock.createMock(IDeviceBuildInfo.class);
2297             IRemoteTest test = EasyMock.createNiceMock(IRemoteTest.class);
2298             mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
2299 
2300             Set<BuildInfoProperties> prop = new HashSet<>();
2301             prop.add(BuildInfoProperties.DO_NOT_LINK_TESTS_DIR);
2302             EasyMock.expect(mMockBuildInfo.getProperties()).andStubReturn(prop);
2303             setEarlyDeviceReleaseExpectation();
2304             setupMockSuccessListeners();
2305             setupNormalInvoke(test);
2306             EasyMock.replay(mockRescheduler);
2307             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
2308             verifyMocks(mockRescheduler);
2309             verifySummaryListener();
2310             // Check that the external directory was NOT copied in the testsDir.
2311             assertTrue(tmpTestsDir.listFiles().length == 0);
2312         } finally {
2313             FileUtil.recursiveDelete(tmpTestsDir);
2314             FileUtil.recursiveDelete(tmpExternalTestsDir);
2315         }
2316         */
2317     }
2318 
2319     /**
2320      * Test that when {@link IMetricCollector} are used, they wrap and call in sequence the listener
2321      * so all metrics end up on the final receiver.
2322      */
2323     @Ignore
2324     @Test
testMetricCollectionChain()2325     public void testMetricCollectionChain() throws Throwable {
2326         /*
2327         TestInformation info =
2328                 TestInformation.newBuilder().setInvocationContext(mStubInvocationMetadata).build();
2329         IConfiguration configuration = new Configuration("test", "description");
2330         StubTest test = new StubTest();
2331         OptionSetter setter = new OptionSetter(test);
2332         setter.setOptionValue("run-a-test", "true");
2333         configuration.setTest(test);
2334 
2335         List<IMetricCollector> collectors = new ArrayList<>();
2336         collectors.add(new TestableCollector("collector1"));
2337         collectors.add(new TestableCollector("collector2"));
2338         collectors.add(new TestableCollector("collector3"));
2339         collectors.add(new TestableCollector("collector4"));
2340         configuration.setDeviceMetricCollectors(collectors);
2341 
2342         mMockTestListener.testRunStarted(
2343                 EasyMock.eq("TestStub"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
2344         TestDescription testId = new TestDescription("StubTest", "StubMethod");
2345         mMockTestListener.testStarted(EasyMock.eq(testId), EasyMock.anyLong());
2346         mMockTestListener.testEnded(
2347                 EasyMock.eq(testId),
2348                 EasyMock.anyLong(),
2349                 EasyMock.eq(new HashMap<String, Metric>()));
2350         Capture<HashMap<String, Metric>> captured = new Capture<>();
2351         mMockTestListener.testRunEnded(EasyMock.anyLong(), EasyMock.capture(captured));
2352         EasyMock.replay(mMockTestListener);
2353         new InvocationExecution().runTests(info, configuration, mMockTestListener);
2354         EasyMock.verify(mMockTestListener);
2355         // The collectors are called in sequence
2356         List<String> listKeys = new ArrayList<>(captured.getValue().keySet());
2357         assertEquals(4, listKeys.size());
2358         assertEquals("collector4", listKeys.get(0));
2359         assertEquals("collector3", listKeys.get(1));
2360         assertEquals("collector2", listKeys.get(2));
2361         assertEquals("collector1", listKeys.get(3));
2362         */
2363     }
2364 
2365     /**
2366      * Test that when a device collector is disabled, it will not be part of the initialization and
2367      * the collector chain.
2368      */
2369     @Ignore
2370     @Test
testMetricCollectionChain_disabled()2371     public void testMetricCollectionChain_disabled() throws Throwable {
2372         /*
2373         TestInformation info =
2374                 TestInformation.newBuilder().setInvocationContext(mStubInvocationMetadata).build();
2375         IConfiguration configuration = new Configuration("test", "description");
2376         StubTest test = new StubTest();
2377         OptionSetter setter = new OptionSetter(test);
2378         setter.setOptionValue("run-a-test", "true");
2379         configuration.setTest(test);
2380 
2381         List<IMetricCollector> collectors = new ArrayList<>();
2382         collectors.add(new TestableCollector("collector1"));
2383         collectors.add(new TestableCollector("collector2"));
2384         // Collector 3 is disabled
2385         TestableCollector col3 = new TestableCollector("collector3");
2386         col3.setDisable(true);
2387         collectors.add(col3);
2388         collectors.add(new TestableCollector("collector4"));
2389         configuration.setDeviceMetricCollectors(collectors);
2390 
2391         mMockTestListener.testRunStarted(
2392                 EasyMock.eq("TestStub"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
2393         TestDescription testId = new TestDescription("StubTest", "StubMethod");
2394         mMockTestListener.testStarted(EasyMock.eq(testId), EasyMock.anyLong());
2395         mMockTestListener.testEnded(
2396                 EasyMock.eq(testId),
2397                 EasyMock.anyLong(),
2398                 EasyMock.eq(new HashMap<String, Metric>()));
2399         Capture<HashMap<String, Metric>> captured = new Capture<>();
2400         mMockTestListener.testRunEnded(EasyMock.anyLong(), EasyMock.capture(captured));
2401         EasyMock.replay(mMockTestListener);
2402         new InvocationExecution().runTests(info, configuration, mMockTestListener);
2403         EasyMock.verify(mMockTestListener);
2404         // The collectors are called in sequence
2405         List<String> listKeys = new ArrayList<>(captured.getValue().keySet());
2406         assertEquals(3, listKeys.size());
2407         assertEquals("collector4", listKeys.get(0));
2408         assertEquals("collector2", listKeys.get(1));
2409         assertEquals("collector1", listKeys.get(2));
2410         */
2411     }
2412 
2413     /** Ensure post processors are called in order. */
2414     @Ignore
2415     @Test
testProcessorCollectionChain()2416     public void testProcessorCollectionChain() throws Throwable {
2417         /*
2418         mMockTestListener = EasyMock.createMock(ITestInvocationListener.class);
2419         List<ITestInvocationListener> listenerList = new ArrayList<ITestInvocationListener>(1);
2420         listenerList.add(mMockTestListener);
2421         mStubConfiguration.setTestInvocationListeners(listenerList);
2422 
2423         List<IPostProcessor> processors = new ArrayList<>();
2424         processors.add(new TestableProcessor("processor1"));
2425         processors.add(new TestableProcessor("processor2"));
2426         processors.add(new TestableProcessor("processor3"));
2427         processors.add(new TestableProcessor("processor4"));
2428         mStubConfiguration.setPostProcessors(processors);
2429 
2430         mMockTestListener.testRunStarted("TestStub", 1);
2431         TestDescription testId = new TestDescription("StubTest", "StubMethod");
2432         mMockTestListener.testStarted(EasyMock.eq(testId), EasyMock.anyLong());
2433         mMockTestListener.testEnded(
2434                 EasyMock.eq(testId),
2435                 EasyMock.anyLong(),
2436                 EasyMock.eq(new HashMap<String, Metric>()));
2437         Capture<HashMap<String, Metric>> captured = new Capture<>();
2438         mMockTestListener.testRunEnded(EasyMock.anyLong(), EasyMock.capture(captured));
2439 
2440         setupMockSuccessListeners();
2441         setEarlyDeviceReleaseExpectation();
2442         setupInvokeWithBuild();
2443         StubTest test = new StubTest();
2444         OptionSetter setter = new OptionSetter(test);
2445         setter.setOptionValue("run-a-test", "true");
2446         mStubConfiguration.setTest(test);
2447         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
2448 
2449         mMockPreparer.setUp(EasyMock.anyObject());
2450         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
2451 
2452         EasyMock.reset(mMockSummaryListener);
2453         replayMocks();
2454         EasyMock.replay(mockRescheduler);
2455         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
2456         verifyMocks(mockRescheduler);
2457 
2458         // The post processors are called in sequence
2459         List<String> listKeys = new ArrayList<>(captured.getValue().keySet());
2460         assertEquals(4, listKeys.size());
2461         assertEquals("processor4", listKeys.get(0));
2462         assertEquals("processor3", listKeys.get(1));
2463         assertEquals("processor2", listKeys.get(2));
2464         assertEquals("processor1", listKeys.get(3));
2465         */
2466     }
2467 
2468     /** Ensure post processors are called in order and that ILogSaver is set properly */
2469     @Ignore
2470     @Test
testProcessorCollectionChain_logSaver()2471     public void testProcessorCollectionChain_logSaver() throws Throwable {
2472         /*
2473         TestLogSaverListener mockLogSaverListener = new TestLogSaverListener();
2474         mMockTestListener = EasyMock.createMock(ITestInvocationListener.class);
2475         List<ITestInvocationListener> listenerList = new ArrayList<ITestInvocationListener>(1);
2476         listenerList.add(mMockTestListener);
2477         listenerList.add(mockLogSaverListener);
2478         mStubConfiguration.setTestInvocationListeners(listenerList);
2479 
2480         List<IPostProcessor> processors = new ArrayList<>();
2481         processors.add(new TestableProcessor("processor1"));
2482         processors.add(new TestableProcessor("processor2"));
2483         processors.add(new TestableProcessor("processor3"));
2484         processors.add(new TestableProcessor("processor4"));
2485         mStubConfiguration.setPostProcessors(processors);
2486 
2487         mMockTestListener.testRunStarted("TestStub", 1);
2488         TestDescription testId = new TestDescription("StubTest", "StubMethod");
2489         mMockTestListener.testStarted(EasyMock.eq(testId), EasyMock.anyLong());
2490         mMockTestListener.testEnded(
2491                 EasyMock.eq(testId),
2492                 EasyMock.anyLong(),
2493                 EasyMock.eq(new HashMap<String, Metric>()));
2494         Capture<HashMap<String, Metric>> captured = new Capture<>();
2495         mMockTestListener.testRunEnded(EasyMock.anyLong(), EasyMock.capture(captured));
2496 
2497         setupMockSuccessListeners();
2498         setEarlyDeviceReleaseExpectation();
2499         setupInvokeWithBuild();
2500         StubTest test = new StubTest();
2501         OptionSetter setter = new OptionSetter(test);
2502         setter.setOptionValue("run-a-test", "true");
2503         mStubConfiguration.setTest(test);
2504         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
2505 
2506         mMockPreparer.setUp(EasyMock.anyObject());
2507         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
2508 
2509         EasyMock.reset(mMockSummaryListener);
2510         replayMocks();
2511         EasyMock.replay(mockRescheduler);
2512         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
2513         verifyMocks(mockRescheduler);
2514 
2515         // The post processors are called in sequence
2516         List<String> listKeys = new ArrayList<>(captured.getValue().keySet());
2517         assertEquals(4, listKeys.size());
2518         assertEquals("processor4", listKeys.get(0));
2519         assertEquals("processor3", listKeys.get(1));
2520         assertEquals("processor2", listKeys.get(2));
2521         assertEquals("processor1", listKeys.get(3));
2522 
2523         assertTrue(mockLogSaverListener.mWasLoggerSet);
2524         */
2525     }
2526 
2527     @Ignore
2528     @Test
testInvoke_shardingException()2529     public void testInvoke_shardingException() throws Throwable {
2530         /*
2531         mShardingEarlyFailure = true;
2532         RuntimeException failedShard =
2533                 new HarnessRuntimeException(
2534                         "Failed to shard", InfraErrorIdentifier.FAIL_TO_CREATE_FILE);
2535         mTestInvocation =
2536                 new TestInvocation() {
2537                     @Override
2538                     ILogRegistry getLogRegistry() {
2539                         return mMockLogRegistry;
2540                     }
2541 
2542                     @Override
2543                     public IInvocationExecution createInvocationExec(RunMode mode) {
2544                         return new InvocationExecution() {
2545                             @Override
2546                             public boolean shardConfig(
2547                                     IConfiguration config,
2548                                     TestInformation testInfo,
2549                                     IRescheduler rescheduler,
2550                                     ITestLogger logger) {
2551                                 throw failedShard;
2552                             }
2553 
2554                             @Override
2555                             protected String getAdbVersion() {
2556                                 return null;
2557                             }
2558 
2559                             @Override
2560                             protected void logHostAdb(IConfiguration config, ITestLogger logger) {
2561                                 // inop for the common test case.
2562                             }
2563                         };
2564                     }
2565 
2566                     @Override
2567                     protected void setExitCode(ExitCode code, Throwable stack) {
2568                         // Empty on purpose
2569                     }
2570 
2571                     @Override
2572                     public void registerExecutionFiles(ExecutionFiles executionFiles) {
2573                         // Empty on purpose
2574                     }
2575 
2576                     @Override
2577                     protected void applyAutomatedReporters(IConfiguration config) {
2578                         // Empty on purpose
2579                     }
2580 
2581                     @Override
2582                     protected void addInvocationMetric(InvocationMetricKey key, long value) {}
2583 
2584                     @Override
2585                     protected void addInvocationMetric(InvocationMetricKey key, String value) {}
2586                 };
2587         mStubConfiguration.getCommandOptions().setShardCount(5);
2588         mStubConfiguration.getCommandOptions().setShardIndex(2);
2589         mMockBuildInfo.addBuildAttribute("shard_count", "5");
2590         mMockBuildInfo.addBuildAttribute("shard_index", "2");
2591         mMockDevice.postInvocationTearDown(EasyMock.anyObject());
2592 
2593         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
2594         setupMockFailureListeners(failedShard);
2595 
2596         setEarlyDeviceReleaseExpectation();
2597         setupInvokeWithBuild();
2598         mStubConfiguration.setTest(test);
2599         mStubMultiConfiguration.setTest(test);
2600         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
2601         replayMocks(test);
2602         EasyMock.reset(mMockLogger, mMockLogRegistry);
2603         mMockLogRegistry.registerLogger(mMockLogger);
2604 
2605         mMockLogger.init();
2606         mMockLogger.closeLog();
2607         EasyMock.expectLastCall().times(2);
2608         mMockLogRegistry.unregisterLogger();
2609         EasyMock.expectLastCall().times(2);
2610         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
2611         EasyMock.expect(mMockLogger.getLog()).andReturn(mHostLogSource);
2612         EasyMock.replay(mockRescheduler, mMockLogger, mMockLogRegistry);
2613         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
2614         verifyMocks(test, mockRescheduler);
2615         verifySummaryListener();
2616         */
2617     }
2618 }
2619