• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.bazel;
17 
18 import static com.google.common.truth.Truth.assertThat;
19 
20 import static org.mockito.ArgumentMatchers.any;
21 import static org.mockito.ArgumentMatchers.eq;
22 import static org.mockito.Mockito.anyLong;
23 import static org.mockito.Mockito.anyMap;
24 import static org.mockito.Mockito.argThat;
25 import static org.mockito.Mockito.contains;
26 import static org.mockito.Mockito.inOrder;
27 import static org.mockito.Mockito.mock;
28 import static org.mockito.Mockito.never;
29 import static org.mockito.Mockito.times;
30 import static org.mockito.Mockito.verify;
31 
32 import com.android.tradefed.config.ConfigurationException;
33 import com.android.tradefed.config.OptionSetter;
34 import com.android.tradefed.invoker.IInvocationContext;
35 import com.android.tradefed.invoker.InvocationContext;
36 import com.android.tradefed.invoker.IInvocationContext;
37 import com.android.tradefed.invoker.TestInformation;
38 import com.android.tradefed.log.LogUtil.CLog;
39 import com.android.tradefed.result.FailureDescription;
40 import com.android.tradefed.result.ILogSaverListener;
41 import com.android.tradefed.result.LogDataType;
42 import com.android.tradefed.result.LogFile;
43 import com.android.tradefed.result.TestDescription;
44 import com.android.tradefed.result.error.ErrorIdentifier;
45 import com.android.tradefed.result.error.TestErrorIdentifier;
46 import com.android.tradefed.result.proto.FileProtoResultReporter;
47 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
48 import com.android.tradefed.util.ZipUtil;
49 
50 import com.google.common.base.Splitter;
51 import com.google.common.collect.ImmutableMap;
52 import com.google.common.io.MoreFiles;
53 import com.google.common.util.concurrent.Uninterruptibles;
54 import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos;
55 
56 import org.junit.Before;
57 import org.junit.Ignore;
58 import org.junit.Rule;
59 import org.junit.Test;
60 import org.junit.rules.TemporaryFolder;
61 import org.junit.runner.RunWith;
62 import org.junit.runners.JUnit4;
63 import org.mockito.ArgumentMatcher;
64 import org.mockito.InOrder;
65 
66 import java.io.ByteArrayInputStream;
67 import java.io.ByteArrayOutputStream;
68 import java.io.File;
69 import java.io.FileOutputStream;
70 import java.io.IOException;
71 import java.io.InputStream;
72 import java.io.OutputStream;
73 import java.nio.file.Files;
74 import java.nio.file.Path;
75 import java.nio.file.Paths;
76 import java.time.Duration;
77 import java.util.ArrayList;
78 import java.util.Collections;
79 import java.util.HashMap;
80 import java.util.List;
81 import java.util.Map;
82 import java.util.Map.Entry;
83 import java.util.Properties;
84 import java.util.Random;
85 import java.util.concurrent.TimeUnit;
86 import java.util.concurrent.atomic.AtomicLong;
87 import java.util.function.Function;
88 import java.util.stream.Collectors;
89 import java.util.stream.Stream;
90 
91 @RunWith(JUnit4.class)
92 public final class BazelTestTest {
93 
94     private ILogSaverListener mMockListener;
95     private TestInformation mTestInfo;
96     private Path mBazelTempPath;
97     private Path mWorkspaceArchivePath;
98 
99     private static final String BAZEL_TEST_TARGETS_OPTION = "bazel-test-target-patterns";
100     private static final String BEP_FILE_OPTION_NAME = "--build_event_binary_file";
101     private static final String REPORT_CACHED_TEST_RESULTS_OPTION = "report-cached-test-results";
102     private static final String REPORT_CACHED_MODULES_SPARSELY_OPTION =
103             "report-cached-modules-sparsely";
104     private static final String BAZEL_TEST_MODULE_ID = "bazel-test-module-id";
105     private static final String TEST_MODULE_MODULE_ID = "single-tradefed-test-module-id";
106     private static final long RANDOM_SEED = 1234567890L;
107 
108     @Rule public final TemporaryFolder tempDir = new TemporaryFolder();
109 
110     @Before
setUp()111     public void setUp() throws Exception {
112         mMockListener = mock(ILogSaverListener.class);
113         InvocationContext context = new InvocationContext();
114         context.addInvocationAttribute("module-id", BAZEL_TEST_MODULE_ID);
115         context.lockAttributes();
116         mTestInfo = TestInformation.newBuilder().setInvocationContext(context).build();
117         mBazelTempPath =
118                 Files.createDirectory(tempDir.getRoot().toPath().resolve("bazel_temp_dir"));
119         Files.createDirectories(
120                 tempDir.getRoot()
121                         .toPath()
122                         .resolve("bazel_suite_root/android-bazel-suite/out/atest_bazel_workspace"));
123         mWorkspaceArchivePath = tempDir.getRoot().toPath().resolve("bazel_suite_root");
124     }
125 
126     @Test
runSucceeds_invokesListenerEvents()127     public void runSucceeds_invokesListenerEvents() throws Exception {
128         BazelTest bazelTest = newBazelTest();
129 
130         bazelTest.run(mTestInfo, mMockListener);
131 
132         verify(mMockListener).testRunStarted(eq(BazelTest.class.getName()), eq(0));
133         verify(mMockListener).testRunEnded(anyLong(), anyMap());
134     }
135 
136     @Test
runSucceeds_noFailuresReported()137     public void runSucceeds_noFailuresReported() throws Exception {
138         BazelTest bazelTest = newBazelTest();
139 
140         bazelTest.run(mTestInfo, mMockListener);
141 
142         verify(mMockListener, never()).testRunFailed(any(FailureDescription.class));
143     }
144 
145     @Test
runSucceeds_tempDirEmptied()146     public void runSucceeds_tempDirEmptied() throws Exception {
147         BazelTest bazelTest = newBazelTest();
148 
149         bazelTest.run(mTestInfo, mMockListener);
150 
151         assertThat(listDirContents(mBazelTempPath)).isEmpty();
152     }
153 
154     @Test
runSucceeds_logsSaved()155     public void runSucceeds_logsSaved() throws Exception {
156         BazelTest bazelTest = newBazelTest();
157 
158         bazelTest.run(mTestInfo, mMockListener);
159 
160         verify(mMockListener)
161                 .testLog(
162                         contains(String.format("%s-log", BazelTest.QUERY_ALL_TARGETS)),
163                         any(),
164                         any());
165         verify(mMockListener)
166                 .testLog(
167                         contains(String.format("%s-log", BazelTest.QUERY_MAP_MODULES_TO_TARGETS)),
168                         any(),
169                         any());
170         verify(mMockListener)
171                 .testLog(contains(String.format("%s-log", BazelTest.RUN_TESTS)), any(), any());
172     }
173 
174     @Test
runSucceeds_testLogsReportedUnderModule()175     public void runSucceeds_testLogsReportedUnderModule() throws Exception {
176         BazelTest bazelTest = newBazelTest();
177 
178         bazelTest.run(mTestInfo, mMockListener);
179 
180         InOrder inOrder = inOrder(mMockListener);
181         inOrder.verify(mMockListener).testModuleStarted(any());
182         inOrder.verify(mMockListener)
183                 .testLog(eq("tf-test-process-module-log"), eq(LogDataType.TAR_GZ), any());
184         inOrder.verify(mMockListener)
185                 .testLog(eq("tf-test-process-invocation-log"), eq(LogDataType.XML), any());
186         inOrder.verify(mMockListener).testModuleEnded();
187     }
188 
189     @Test
traceFileWritten_traceFileReported()190     public void traceFileWritten_traceFileReported() throws Exception {
191         FakeProcessStarter processStarter = newFakeProcessStarter();
192         processStarter.put(
193                 BazelTest.RUN_TESTS,
194                 builder -> {
195                     return new FakeBazelTestProcess(builder, mBazelTempPath) {
196                         @Override
197                         public void writeSingleTestOutputs(Path outputsDir, String testName)
198                                 throws IOException, ConfigurationException {
199 
200                             defaultWriteSingleTestOutputs(outputsDir, testName, true);
201                         }
202                     };
203                 });
204         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
205 
206         bazelTest.run(mTestInfo, mMockListener);
207 
208         verify(mMockListener)
209                 .testLog(
210                         eq("tf-test-process-fake-invocation-trace.perfetto-trace"),
211                         eq(LogDataType.TEXT),
212                         any());
213     }
214 
215     @Test
malformedProtoResults_runFails()216     public void malformedProtoResults_runFails() throws Exception {
217         FakeProcessStarter processStarter = newFakeProcessStarter();
218         processStarter.put(
219                 BazelTest.RUN_TESTS,
220                 builder -> {
221                     return new FakeBazelTestProcess(builder, mBazelTempPath) {
222                         @Override
223                         public void writeSingleTestOutputs(Path outputsDir, String testName)
224                                 throws IOException, ConfigurationException {
225 
226                             defaultWriteSingleTestOutputs(outputsDir, testName, false);
227 
228                             Path outputFile = outputsDir.resolve("proto-results");
229                             Files.write(outputFile, "Malformed Proto File".getBytes());
230                         }
231                     };
232                 });
233         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
234 
235         bazelTest.run(mTestInfo, mMockListener);
236 
237         verify(mMockListener).testRunFailed(hasFailureStatus(FailureStatus.INFRA_FAILURE));
238     }
239 
240     @Test
malformedBepFile_runFails()241     public void malformedBepFile_runFails() throws Exception {
242         FakeProcessStarter processStarter = newFakeProcessStarter();
243         processStarter.put(
244                 BazelTest.RUN_TESTS,
245                 builder -> {
246                     return new FakeBazelTestProcess(builder, mBazelTempPath) {
247                         @Override
248                         public void writeSingleTestResultEvent(File outputsZipFile, Path bepFile)
249                                 throws IOException {
250 
251                             Files.write(bepFile, "Malformed BEP File".getBytes());
252                         }
253                     };
254                 });
255         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
256 
257         bazelTest.run(mTestInfo, mMockListener);
258 
259         verify(mMockListener).testRunFailed(hasFailureStatus(FailureStatus.TEST_FAILURE));
260     }
261 
262     @Test
bepFileMissingLastMessage_runFails()263     public void bepFileMissingLastMessage_runFails() throws Exception {
264         FakeProcessStarter processStarter = newFakeProcessStarter();
265         processStarter.put(
266                 BazelTest.RUN_TESTS,
267                 builder -> {
268                     return new FakeBazelTestProcess(builder, mBazelTempPath) {
269                         @Override
270                         public void writeLastEvent() throws IOException {
271                             // Do nothing.
272                         }
273                     };
274                 });
275         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
276 
277         bazelTest.run(mTestInfo, mMockListener);
278 
279         verify(mMockListener).testRunFailed(hasFailureStatus(FailureStatus.INFRA_FAILURE));
280     }
281 
282     @Test
targetsNotSet_testsAllTargets()283     public void targetsNotSet_testsAllTargets() throws Exception {
284         List<String> command = new ArrayList<>();
285         FakeProcessStarter processStarter = newFakeProcessStarter();
286         processStarter.put(
287                 BazelTest.QUERY_ALL_TARGETS,
288                 newPassingProcessWithStdout("//bazel/target:default_target_host"));
289         processStarter.put(
290                 BazelTest.QUERY_MAP_MODULES_TO_TARGETS,
291                 newPassingProcessWithStdout("default_target //bazel/target:default_target_host"));
292         processStarter.put(
293                 BazelTest.RUN_TESTS,
294                 builder -> {
295                     command.addAll(builder.command());
296                     return new FakeBazelTestProcess(builder, mBazelTempPath);
297                 });
298         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
299 
300         bazelTest.run(mTestInfo, mMockListener);
301 
302         assertThat(command).contains("//bazel/target:default_target_host");
303     }
304 
305     @Test
archiveRootPathNotSet_runAborted()306     public void archiveRootPathNotSet_runAborted() throws Exception {
307         Properties properties = bazelTestProperties();
308         properties.remove("BAZEL_SUITE_ROOT");
309         BazelTest bazelTest = newBazelTestWithProperties(properties);
310 
311         bazelTest.run(mTestInfo, mMockListener);
312 
313         verify(mMockListener).testRunFailed(hasFailureStatus(FailureStatus.DEPENDENCY_ISSUE));
314     }
315 
316     @Test
archiveRootPathEmptyString_runAborted()317     public void archiveRootPathEmptyString_runAborted() throws Exception {
318         Properties properties = bazelTestProperties();
319         properties.put("BAZEL_SUITE_ROOT", "");
320         BazelTest bazelTest = newBazelTestWithProperties(properties);
321 
322         bazelTest.run(mTestInfo, mMockListener);
323 
324         verify(mMockListener).testRunFailed(hasFailureStatus(FailureStatus.DEPENDENCY_ISSUE));
325     }
326 
327     @Test
bazelQueryAllTargetsFails_runAborted()328     public void bazelQueryAllTargetsFails_runAborted() throws Exception {
329         FakeProcessStarter processStarter = newFakeProcessStarter();
330         processStarter.put(BazelTest.QUERY_ALL_TARGETS, newFailingProcess());
331         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
332 
333         bazelTest.run(mTestInfo, mMockListener);
334 
335         verify(mMockListener).testRunFailed(hasErrorIdentifier(TestErrorIdentifier.TEST_ABORTED));
336     }
337 
338     @Test
bazelQueryMapModuleToTargetsFails_runAborted()339     public void bazelQueryMapModuleToTargetsFails_runAborted() throws Exception {
340         FakeProcessStarter processStarter = newFakeProcessStarter();
341         processStarter.put(BazelTest.QUERY_MAP_MODULES_TO_TARGETS, newFailingProcess());
342         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
343 
344         bazelTest.run(mTestInfo, mMockListener);
345 
346         verify(mMockListener).testRunFailed(hasErrorIdentifier(TestErrorIdentifier.TEST_ABORTED));
347     }
348 
349     @Test
bazelReturnsTestFailureCode_noFailureReported()350     public void bazelReturnsTestFailureCode_noFailureReported() throws Exception {
351         FakeProcessStarter processStarter = newFakeProcessStarter();
352         processStarter.put(
353                 BazelTest.RUN_TESTS,
354                 builder -> {
355                     return new FakeBazelTestProcess(builder, mBazelTempPath) {
356                         @Override
357                         public int exitValue() {
358                             return BazelTest.BAZEL_TESTS_FAILED_RETURN_CODE;
359                         }
360                     };
361                 });
362         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
363 
364         bazelTest.run(mTestInfo, mMockListener);
365 
366         verify(mMockListener, never()).testRunFailed(any(FailureDescription.class));
367     }
368 
369     @Test
370     @Ignore("b/281805276: Flaky")
testTimeout_causesTestFailure()371     public void testTimeout_causesTestFailure() throws Exception {
372         FakeProcessStarter processStarter = newFakeProcessStarter();
373         processStarter.put(
374                 BazelTest.RUN_TESTS,
375                 builder -> {
376                     return new FakeBazelTestProcess(builder, mBazelTempPath) {
377                         @Override
378                         public boolean waitFor(long timeout, TimeUnit unit) {
379                             return false;
380                         }
381                     };
382                 });
383         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
384 
385         bazelTest.run(mTestInfo, mMockListener);
386 
387         verify(mMockListener).testRunFailed(hasFailureStatus(FailureStatus.DEPENDENCY_ISSUE));
388     }
389 
390     @Test
testModuleTimesOut_testReported()391     public void testModuleTimesOut_testReported() throws Exception {
392         FakeProcessStarter processStarter = newFakeProcessStarter();
393         processStarter.put(
394                 BazelTest.RUN_TESTS,
395                 builder -> {
396                     return new FakeBazelTestProcess(builder, mBazelTempPath) {
397                         @Override
398                         public void writeSingleTestResultEvent(File outputsZipFile, Path bepFile)
399                                 throws IOException {
400 
401                             writeSingleTestResultEvent(
402                                     outputsZipFile,
403                                     bepFile, /* status */
404                                     BuildEventStreamProtos.TestStatus.TIMEOUT);
405                         }
406                     };
407                 });
408         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
409 
410         bazelTest.run(mTestInfo, mMockListener);
411 
412         verify(mMockListener).testRunFailed(hasFailureStatus(FailureStatus.TIMED_OUT));
413     }
414 
415     @Test
includeTestModule_runsOnlyThatModule()416     public void includeTestModule_runsOnlyThatModule() throws Exception {
417         String moduleInclude = "custom_module";
418         List<String> command = new ArrayList<>();
419         FakeProcessStarter processStarter = newFakeProcessStarter();
420         processStarter.put(
421                 BazelTest.QUERY_ALL_TARGETS,
422                 newPassingProcessWithStdout(
423                         "//bazel/target:default_target_host\n//bazel/target:custom_module_host"));
424         processStarter.put(
425                 BazelTest.QUERY_MAP_MODULES_TO_TARGETS,
426                 newPassingProcessWithStdout(
427                         "default_target //bazel/target:default_target_host\n"
428                                 + "custom_module //bazel/target:custom_module_host"));
429         processStarter.put(
430                 BazelTest.RUN_TESTS,
431                 builder -> {
432                     command.addAll(builder.command());
433                     return new FakeBazelTestProcess(builder, mBazelTempPath);
434                 });
435         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
436         OptionSetter setter = new OptionSetter(bazelTest);
437         setter.setOptionValue("include-filter", moduleInclude);
438 
439         bazelTest.run(mTestInfo, mMockListener);
440 
441         assertThat(command).contains("//bazel/target:custom_module_host");
442         assertThat(command).doesNotContain("//bazel/target:default_target_host");
443     }
444 
445     @Test
excludeTestModule_doesNotRunTestModule()446     public void excludeTestModule_doesNotRunTestModule() throws Exception {
447         String moduleExclude = "custom_module";
448         List<String> command = new ArrayList<>();
449         FakeProcessStarter processStarter = newFakeProcessStarter();
450         processStarter.put(
451                 BazelTest.QUERY_ALL_TARGETS,
452                 newPassingProcessWithStdout(
453                         "//bazel/target:default_target_host\n//bazel/target:custom_module_host"));
454         processStarter.put(
455                 BazelTest.QUERY_MAP_MODULES_TO_TARGETS,
456                 newPassingProcessWithStdout(
457                         "default_target //bazel/target:default_target_host\n"
458                                 + "custom_module //bazel/target:custom_module_host"));
459         processStarter.put(
460                 BazelTest.RUN_TESTS,
461                 builder -> {
462                     command.addAll(builder.command());
463                     return new FakeBazelTestProcess(builder, mBazelTempPath);
464                 });
465         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
466         OptionSetter setter = new OptionSetter(bazelTest);
467         setter.setOptionValue("exclude-filter", moduleExclude);
468 
469         bazelTest.run(mTestInfo, mMockListener);
470 
471         assertThat(command).doesNotContain("//bazel/target:custom_module_host");
472         assertThat(command).contains("//bazel/target:default_target_host");
473     }
474 
475     @Test
excludeTestFunction_generatesExcludeFilter()476     public void excludeTestFunction_generatesExcludeFilter() throws Exception {
477         String functionExclude = "custom_module custom_module.customClass#customFunction";
478         List<String> command = new ArrayList<>();
479         FakeProcessStarter processStarter = newFakeProcessStarter();
480         processStarter.put(
481                 BazelTest.RUN_TESTS,
482                 builder -> {
483                     command.addAll(builder.command());
484                     return new FakeBazelTestProcess(builder, mBazelTempPath);
485                 });
486         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
487         OptionSetter setter = new OptionSetter(bazelTest);
488         setter.setOptionValue("exclude-filter", functionExclude);
489 
490         bazelTest.run(mTestInfo, mMockListener);
491 
492         assertThat(command)
493                 .contains(
494                         "--test_arg=--global-filters:exclude-filter=custom_module"
495                                 + " custom_module.customClass#customFunction");
496     }
497 
498     @Test
excludeAndIncludeFiltersSet_testRunAborted()499     public void excludeAndIncludeFiltersSet_testRunAborted() throws Exception {
500         String moduleExclude = "custom_module";
501         BazelTest bazelTest = newBazelTest();
502         OptionSetter setter = new OptionSetter(bazelTest);
503         setter.setOptionValue("exclude-filter", moduleExclude);
504         setter.setOptionValue("include-filter", moduleExclude);
505 
506         bazelTest.run(mTestInfo, mMockListener);
507 
508         verify(mMockListener).testRunFailed(hasErrorIdentifier(TestErrorIdentifier.TEST_ABORTED));
509     }
510 
511     @Test
queryMapModulesToTargetsEmpty_abortsRun()512     public void queryMapModulesToTargetsEmpty_abortsRun() throws Exception {
513         FakeProcessStarter processStarter = newFakeProcessStarter();
514         processStarter.put(BazelTest.QUERY_MAP_MODULES_TO_TARGETS, newPassingProcessWithStdout(""));
515         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
516 
517         bazelTest.run(mTestInfo, mMockListener);
518 
519         verify(mMockListener).testRunFailed(hasErrorIdentifier(TestErrorIdentifier.TEST_ABORTED));
520     }
521 
522     @Test
multipleTargetsMappedToSingleModule_abortsRun()523     public void multipleTargetsMappedToSingleModule_abortsRun() throws Exception {
524         FakeProcessStarter processStarter = newFakeProcessStarter();
525         processStarter.put(
526                 BazelTest.QUERY_MAP_MODULES_TO_TARGETS,
527                 newPassingProcessWithStdout(
528                         "default_target //bazel/target:default_target_1\n"
529                                 + "default_target //bazel/target:default_target_2"));
530         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
531 
532         bazelTest.run(mTestInfo, mMockListener);
533 
534         verify(mMockListener).testRunFailed(hasErrorIdentifier(TestErrorIdentifier.TEST_ABORTED));
535     }
536 
537     @Test
queryMapModulesToTargetsBadOutput_abortsRun()538     public void queryMapModulesToTargetsBadOutput_abortsRun() throws Exception {
539         FakeProcessStarter processStarter = newFakeProcessStarter();
540         processStarter.put(
541                 BazelTest.QUERY_MAP_MODULES_TO_TARGETS,
542                 newPassingProcessWithStdout(
543                         "default_target //bazel/target:default_target incorrect_field"));
544         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
545 
546         bazelTest.run(mTestInfo, mMockListener);
547 
548         verify(mMockListener).testRunFailed(hasErrorIdentifier(TestErrorIdentifier.TEST_ABORTED));
549     }
550 
551     @Test
multipleTestsRun_reportsAllResults()552     public void multipleTestsRun_reportsAllResults() throws Exception {
553         int testCount = 3;
554         Duration testDelay = Duration.ofMillis(10);
555         final AtomicLong testTime = new AtomicLong();
556         FakeProcessStarter processStarter = newFakeProcessStarter();
557         byte[] bytes = logFileContents();
558 
559         processStarter.put(
560                 BazelTest.RUN_TESTS,
561                 builder -> {
562                     return new FakeBazelTestProcess(builder, mBazelTempPath) {
563                         @Override
564                         public Path createLogFile(String testName, Path logDir) throws IOException {
565                             Path logFile = logDir.resolve(testName);
566                             Files.write(logFile, bytes);
567                             return logFile;
568                         }
569 
570                         @Override
571                         public void runTests() throws IOException, ConfigurationException {
572                             long start = System.nanoTime();
573                             for (int i = 0; i < testCount; i++) {
574                                 runSingleTest("test-" + i);
575                             }
576                             testTime.set((System.nanoTime() - start) / 1000000);
577                         }
578 
579                         @Override
580                         void singleTestBody() {
581                             Uninterruptibles.sleepUninterruptibly(
582                                     testDelay.toMillis(), TimeUnit.MILLISECONDS);
583                         }
584                     };
585                 });
586         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
587 
588         long start = System.nanoTime();
589         bazelTest.run(mTestInfo, mMockListener);
590         long totalTime = ((System.nanoTime() - start) / 1000000);
591 
592         // TODO(b/267378279): Consider converting this test to a proper benchmark instead of using
593         // logging.
594         CLog.i("Total runtime: " + totalTime + "ms, test time: " + testTime.get() + "ms.");
595 
596         verify(mMockListener, times(testCount)).testStarted(any(), anyLong());
597     }
598 
599     @Test
reportCachedTestResultsDisabled_cachedTestResultNotReported()600     public void reportCachedTestResultsDisabled_cachedTestResultNotReported() throws Exception {
601         FakeProcessStarter processStarter = newFakeProcessStarter();
602         processStarter.put(
603                 BazelTest.RUN_TESTS,
604                 builder -> {
605                     return new FakeBazelTestProcess(builder, mBazelTempPath) {
606                         @Override
607                         public void writeSingleTestResultEvent(File outputsZipFile, Path bepFile)
608                                 throws IOException {
609 
610                             writeSingleTestResultEvent(outputsZipFile, bepFile, /* cached */ true);
611                         }
612                     };
613                 });
614         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
615         OptionSetter setter = new OptionSetter(bazelTest);
616         setter.setOptionValue(REPORT_CACHED_TEST_RESULTS_OPTION, "false");
617 
618         bazelTest.run(mTestInfo, mMockListener);
619 
620         verify(mMockListener, never()).testStarted(any(), anyLong());
621     }
622 
623     @Test
bazelQuery_default()624     public void bazelQuery_default() throws Exception {
625         List<String> command = new ArrayList<>();
626         FakeProcessStarter processStarter = newFakeProcessStarter();
627         processStarter.put(
628                 BazelTest.QUERY_ALL_TARGETS,
629                 builder -> {
630                     command.addAll(builder.command());
631                     return newPassingProcessWithStdout("unused");
632                 });
633 
634         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
635 
636         bazelTest.run(mTestInfo, mMockListener);
637         assertThat(command).contains("kind(tradefed_deviceless_test, tests(//...))");
638     }
639 
640     @Test
bazelQuery_optionOverride()641     public void bazelQuery_optionOverride() throws Exception {
642         List<String> command = new ArrayList<>();
643         FakeProcessStarter processStarter = newFakeProcessStarter();
644         processStarter.put(
645                 BazelTest.QUERY_ALL_TARGETS,
646                 builder -> {
647                     command.addAll(builder.command());
648                     return newPassingProcessWithStdout("unused");
649                 });
650 
651         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
652         OptionSetter setter = new OptionSetter(bazelTest);
653         setter.setOptionValue("bazel-query", "tests(//vendor/...)");
654 
655         bazelTest.run(mTestInfo, mMockListener);
656         assertThat(command).contains("tests(//vendor/...)");
657         // Default should be overridden and not appear in command
658         assertThat(command).doesNotContain("kind(tradefed_deviceless_test");
659     }
660 
661     @Test
badLogFilePaths_failureReported()662     public void badLogFilePaths_failureReported() throws Exception {
663         FakeProcessStarter processStarter = newFakeProcessStarter();
664         processStarter.put(
665                 BazelTest.RUN_TESTS,
666                 builder -> {
667                     return new FakeBazelTestProcess(builder, mBazelTempPath) {
668                         @Override
669                         public void writeSingleTestOutputs(Path outputsDir, String testName)
670                                 throws IOException, ConfigurationException {
671 
672                             defaultWriteSingleTestOutputs(
673                                     outputsDir.resolve(Paths.get("bad-dir")), testName, false);
674                         }
675                     };
676                 });
677         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
678 
679         bazelTest.run(mTestInfo, mMockListener);
680 
681         verify(mMockListener)
682                 .testRunFailed(hasErrorIdentifier(TestErrorIdentifier.OUTPUT_PARSER_ERROR));
683     }
684 
685     @Test
reportCachedModulesSparsely_reportsOnlyModuleLevelEvents()686     public void reportCachedModulesSparsely_reportsOnlyModuleLevelEvents() throws Exception {
687         FakeProcessStarter processStarter = newFakeProcessStarter();
688         processStarter.put(
689                 BazelTest.RUN_TESTS,
690                 builder -> {
691                     return new FakeBazelTestProcess(builder, mBazelTempPath) {
692                         @Override
693                         public void writeSingleTestResultEvent(File outputsZipFile, Path bepFile)
694                                 throws IOException {
695 
696                             writeSingleTestResultEvent(outputsZipFile, bepFile, /* cached */ true);
697                         }
698                     };
699                 });
700         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
701         OptionSetter setter = new OptionSetter(bazelTest);
702         setter.setOptionValue(REPORT_CACHED_MODULES_SPARSELY_OPTION, "true");
703 
704         bazelTest.run(mTestInfo, mMockListener);
705 
706         // Verify that the test module calls happened.
707         InOrder inOrder = inOrder(mMockListener);
708         inOrder.verify(mMockListener)
709                 .testModuleStarted(
710                         contextHasAttributes(
711                                 ImmutableMap.of(
712                                         "module-id",
713                                         TEST_MODULE_MODULE_ID,
714                                         "sparse-module",
715                                         "true")));
716         inOrder.verify(mMockListener).testModuleEnded();
717 
718         // Verify that no tests were reported.
719         verify(mMockListener, never()).testStarted(any(), anyLong());
720     }
721 
722     @Test
testModuleCached_cachedPropertyReported()723     public void testModuleCached_cachedPropertyReported() throws Exception {
724         FakeProcessStarter processStarter = newFakeProcessStarter();
725         processStarter.put(
726                 BazelTest.RUN_TESTS,
727                 builder -> {
728                     return new FakeBazelTestProcess(builder, mBazelTempPath) {
729                         @Override
730                         public void writeSingleTestResultEvent(File outputsZipFile, Path bepFile)
731                                 throws IOException {
732 
733                             writeSingleTestResultEvent(outputsZipFile, bepFile, /* cached */ true);
734                         }
735                     };
736                 });
737         BazelTest bazelTest = newBazelTestWithProcessStarter(processStarter);
738 
739         bazelTest.run(mTestInfo, mMockListener);
740 
741         verify(mMockListener).testModuleStarted(hasInvocationAttribute("module-cached", "true"));
742     }
743 
logFileContents()744     private static byte[] logFileContents() {
745         // Seed Random to always get the same sequence of values.
746         Random rand = new Random(RANDOM_SEED);
747         byte[] bytes = new byte[1024 * 1024];
748         rand.nextBytes(bytes);
749         return bytes;
750     }
751 
newPassingProcess()752     private static FakeProcess newPassingProcess() {
753         return new FakeProcess() {
754             @Override
755             public int exitValue() {
756                 return 0;
757             }
758         };
759     }
760 
761     private static FakeProcess newFailingProcess() {
762         return new FakeProcess() {
763             @Override
764             public int exitValue() {
765                 return -1;
766             }
767         };
768     }
769 
770     private static FakeProcess newPassingProcessWithStdout(String stdOut) {
771         return new FakeProcess() {
772             @Override
773             public int exitValue() {
774                 return 0;
775             }
776 
777             @Override
778             public InputStream getInputStream() {
779                 return new ByteArrayInputStream(stdOut.getBytes());
780             }
781         };
782     }
783 
784     private BazelTest newBazelTestWithProperties(Properties properties) throws Exception {
785         return new BazelTest(newFakeProcessStarter(), properties);
786     }
787 
788     private BazelTest newBazelTestWithProcessStarter(BazelTest.ProcessStarter starter)
789             throws Exception {
790 
791         return new BazelTest(starter, bazelTestProperties());
792     }
793 
794     private BazelTest newBazelTest() throws Exception {
795         return newBazelTestWithProcessStarter(newFakeProcessStarter());
796     }
797 
798     private Properties bazelTestProperties() {
799         Properties properties = new Properties();
800         properties.put("BAZEL_SUITE_ROOT", mWorkspaceArchivePath.toAbsolutePath().toString());
801         properties.put("java.io.tmpdir", mBazelTempPath.toAbsolutePath().toString());
802 
803         return properties;
804     }
805 
806     private FakeProcessStarter newFakeProcessStarter() throws IOException {
807         String targetName = "//bazel/target:default_target_host";
808         FakeProcessStarter processStarter = new FakeProcessStarter();
809         processStarter.put(BazelTest.QUERY_ALL_TARGETS, newPassingProcessWithStdout(targetName));
810         processStarter.put(
811                 BazelTest.QUERY_MAP_MODULES_TO_TARGETS,
812                 newPassingProcessWithStdout("default_target " + targetName));
813         processStarter.put(
814                 BazelTest.RUN_TESTS,
815                 builder -> {
816                     return new FakeBazelTestProcess(builder, mBazelTempPath);
817                 });
818         return processStarter;
819     }
820 
821     private static FailureDescription hasErrorIdentifier(ErrorIdentifier error) {
822         return argThat(
823                 new ArgumentMatcher<FailureDescription>() {
824                     @Override
825                     public boolean matches(FailureDescription right) {
826                         return right.getErrorIdentifier().equals(error);
827                     }
828 
829                     @Override
830                     public String toString() {
831                         return "hasErrorIdentifier(" + error.toString() + ")";
832                     }
833                 });
834     }
835 
836     private static FailureDescription hasFailureStatus(FailureStatus status) {
837         return argThat(
838                 new ArgumentMatcher<FailureDescription>() {
839                     @Override
840                     public boolean matches(FailureDescription right) {
841                         return right.getFailureStatus().equals(status);
842                     }
843 
844                     @Override
845                     public String toString() {
846                         return "hasFailureStatus(" + status.toString() + ")";
847                     }
848                 });
849     }
850 
851     private static IInvocationContext contextHasAttributes(
852             ImmutableMap<String, String> attributes) {
853         return argThat(
854                 new ArgumentMatcher<IInvocationContext>() {
855                     @Override
856                     public boolean matches(IInvocationContext right) {
857                         for (Entry<String, String> entry : attributes.entrySet()) {
858                             if (!right.getAttribute(entry.getKey()).equals(entry.getValue())) {
859                                 return false;
860                             }
861                         }
862                         return true;
863                     }
864 
865                     @Override
866                     public String toString() {
867                         return "contextHasAttributes(" + attributes.toString() + ")";
868                     }
869                 });
870     }
871 
872     private static IInvocationContext hasInvocationAttribute(String key, String value) {
873         return argThat(
874                 new ArgumentMatcher<IInvocationContext>() {
875                     @Override
876                     public boolean matches(IInvocationContext right) {
877                         return right.getAttribute(key).equals(value);
878                     }
879 
880                     @Override
881                     public String toString() {
882                         return "hasInvocationAttribute(" + key + ", " + value + ")";
883                     }
884                 });
885     }
886 
887     private static List<Path> listDirContents(Path dir) throws IOException {
888         try (Stream<Path> fileStream = Files.list(dir)) {
889             return fileStream.collect(Collectors.toList());
890         }
891     }
892 
893     private static final class FakeProcessStarter implements BazelTest.ProcessStarter {
894         private final Map<String, Function<ProcessBuilder, FakeProcess>> mTagToProcess =
895                 new HashMap<>();
896 
897         @Override
898         public Process start(String tag, ProcessBuilder builder) throws IOException {
899             FakeProcess process = mTagToProcess.get(tag).apply(builder);
900             process.start();
901             return process;
902         }
903 
904         public void put(String tag, FakeProcess process) {
905             mTagToProcess.put(
906                     tag,
907                     b -> {
908                         return process;
909                     });
910         }
911 
912         public void put(String tag, Function<ProcessBuilder, FakeProcess> process) {
913             mTagToProcess.put(tag, process);
914         }
915     }
916 
917     private abstract static class FakeProcess extends Process {
918 
919         private volatile boolean destroyed;
920 
921         @Override
922         public void destroy() {
923             destroyed = true;
924         }
925 
926         @Override
927         public int exitValue() {
928             return destroyed ? 42 : 0;
929         }
930 
931         @Override
932         public InputStream getErrorStream() {
933             return new ByteArrayInputStream("".getBytes());
934         }
935 
936         @Override
937         public InputStream getInputStream() {
938             return new ByteArrayInputStream("".getBytes());
939         }
940 
941         @Override
942         public OutputStream getOutputStream() {
943             return new ByteArrayOutputStream(0);
944         }
945 
946         @Override
947         public int waitFor() {
948             return exitValue();
949         }
950 
951         public void start() throws IOException {
952             return;
953         }
954     }
955 
956     private static class FakeBazelTestProcess extends FakeProcess {
957         private final Path mBepFile;
958         private final Path mBazelTempDirectory;
959 
960         public FakeBazelTestProcess(ProcessBuilder builder, Path bazelTempDir) {
961             mBepFile =
962                     Paths.get(
963                             builder.command().stream()
964                                     .map(s -> Splitter.on('=').splitToList(s))
965                                     .filter(s -> s.get(0).equals(BEP_FILE_OPTION_NAME))
966                                     .findFirst()
967                                     .get()
968                                     .get(1));
969             mBazelTempDirectory = bazelTempDir;
970         }
971 
972         @Override
973         public void start() throws IOException {
974             try {
975                 runTests();
976                 writeLastEvent();
977             } catch (ConfigurationException e) {
978                 throw new RuntimeException(e);
979             }
980         }
981 
982         void runTests() throws IOException, ConfigurationException {
983             runSingleTest("test-1");
984         }
985 
986         void runSingleTest(String testName) throws IOException, ConfigurationException {
987             Path outputDir = Files.createTempDirectory(mBazelTempDirectory, testName);
988             try {
989                 singleTestBody();
990                 writeSingleTestOutputs(outputDir, testName);
991                 File outputsZipFile = zipSingleTestOutputsDirectory(outputDir);
992                 writeSingleTestResultEvent(outputsZipFile, mBepFile);
993             } finally {
994                 MoreFiles.deleteRecursively(outputDir);
995             }
996         }
997 
998         void singleTestBody() {
999             // Do nothing.
1000         }
1001 
1002         void writeSingleTestOutputs(Path outputsDir, String testName)
1003                 throws IOException, ConfigurationException {
1004 
1005             defaultWriteSingleTestOutputs(outputsDir, testName, false);
1006         }
1007 
1008         final void defaultWriteSingleTestOutputs(
1009                 Path outputsDir, String testName, boolean writeTraceFile)
1010                 throws IOException, ConfigurationException {
1011 
1012             FileProtoResultReporter reporter = new FileProtoResultReporter();
1013             OptionSetter setter = new OptionSetter(reporter);
1014             Path outputFile = outputsDir.resolve("proto-results");
1015             setter.setOptionValue("proto-output-file", outputFile.toAbsolutePath().toString());
1016 
1017             Path logDir =
1018                     Files.createDirectories(
1019                             outputsDir
1020                                     .resolve(BazelTest.BRANCH_TEST_ARG)
1021                                     .resolve(BazelTest.BUILD_TEST_ARG)
1022                                     .resolve(BazelTest.TEST_TAG_TEST_ARG));
1023             Path isolatedJavaLog = createLogFile("isolated-java-logs.tar.gz", logDir);
1024             Path tfConfig = createLogFile("tradefed-expanded-config.xml", logDir);
1025             if (writeTraceFile) {
1026                 createLogFile("fake-invocation-trace.perfetto-trace", logDir);
1027             }
1028 
1029             InvocationContext context = new InvocationContext();
1030             context.addInvocationAttribute("module-id", TEST_MODULE_MODULE_ID);
1031 
1032             reporter.invocationStarted(context);
1033             reporter.testModuleStarted(context);
1034             reporter.testRunStarted("test-run", 1);
1035             TestDescription testD = new TestDescription("class-name", testName);
1036             reporter.testStarted(testD);
1037             reporter.testEnded(testD, Collections.emptyMap());
1038             reporter.testRunEnded(0, Collections.emptyMap());
1039             reporter.logAssociation(
1040                     "module-log",
1041                     new LogFile(
1042                             isolatedJavaLog.toAbsolutePath().toString(), "", LogDataType.TAR_GZ));
1043             reporter.testModuleEnded();
1044             reporter.logAssociation(
1045                     "invocation-log",
1046                     new LogFile(tfConfig.toAbsolutePath().toString(), "", LogDataType.XML));
1047             reporter.invocationEnded(0);
1048         }
1049 
1050         Path createLogFile(String testName, Path logDir) throws IOException {
1051             Path logFile = logDir.resolve(testName);
1052             Files.write(logFile, testName.getBytes());
1053             return logFile;
1054         }
1055 
1056         File zipSingleTestOutputsDirectory(Path outputsDir) throws IOException {
1057             List<File> files =
1058                     listDirContents(outputsDir).stream()
1059                             .map(f -> f.toFile())
1060                             .collect(Collectors.toList());
1061             return ZipUtil.createZip(files);
1062         }
1063 
1064         void writeSingleTestResultEvent(File outputsZipFile, Path bepFile) throws IOException {
1065             writeSingleTestResultEvent(
1066                     outputsZipFile, bepFile, false, BuildEventStreamProtos.TestStatus.PASSED);
1067         }
1068 
1069         void writeSingleTestResultEvent(File outputsZipFile, Path bepFile, boolean cached)
1070                 throws IOException {
1071             writeSingleTestResultEvent(
1072                     outputsZipFile, bepFile, cached, BuildEventStreamProtos.TestStatus.PASSED);
1073         }
1074 
1075         void writeSingleTestResultEvent(
1076                 File outputsZipFile, Path bepFile, BuildEventStreamProtos.TestStatus status)
1077                 throws IOException {
1078 
1079             writeSingleTestResultEvent(outputsZipFile, bepFile, false, status);
1080         }
1081 
1082         void writeSingleTestResultEvent(
1083                 File outputsZipFile,
1084                 Path bepFile,
1085                 boolean cached,
1086                 BuildEventStreamProtos.TestStatus status)
1087                 throws IOException {
1088             try (FileOutputStream bepOutputStream = new FileOutputStream(bepFile.toFile(), true)) {
1089                 BuildEventStreamProtos.BuildEvent.newBuilder()
1090                         .setId(
1091                                 BuildEventStreamProtos.BuildEventId.newBuilder()
1092                                         .setTestResult(
1093                                                 BuildEventStreamProtos.BuildEventId.TestResultId
1094                                                         .getDefaultInstance())
1095                                         .build())
1096                         .setTestResult(
1097                                 BuildEventStreamProtos.TestResult.newBuilder()
1098                                         .addTestActionOutput(
1099                                                 BuildEventStreamProtos.File.newBuilder()
1100                                                         .setName("test.outputs__outputs.zip")
1101                                                         .setUri(outputsZipFile.getAbsolutePath())
1102                                                         .build())
1103                                         .setExecutionInfo(
1104                                                 BuildEventStreamProtos.TestResult.ExecutionInfo
1105                                                         .newBuilder()
1106                                                         .setCachedRemotely(cached)
1107                                                         .build())
1108                                         .setStatus(status)
1109                                         .build())
1110                         .build()
1111                         .writeDelimitedTo(bepOutputStream);
1112             }
1113         }
1114 
1115         void writeLastEvent() throws IOException {
1116             try (FileOutputStream bepOutputStream = new FileOutputStream(mBepFile.toFile(), true)) {
1117                 BuildEventStreamProtos.BuildEvent.newBuilder()
1118                         .setId(BuildEventStreamProtos.BuildEventId.getDefaultInstance())
1119                         .setProgress(BuildEventStreamProtos.Progress.getDefaultInstance())
1120                         .setLastMessage(true)
1121                         .build()
1122                         .writeDelimitedTo(bepOutputStream);
1123             }
1124         }
1125     }
1126 }
1127