• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2022 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import json
6import pathlib
7import unittest
8from unittest import mock
9
10from crossbench import compat
11from crossbench.browsers.browser import Browser
12from crossbench.env import HostEnvironment
13from crossbench.flags.base import Flags
14from crossbench.helper.state import UnexpectedStateError
15from crossbench.probes import all as all_probes
16from crossbench.probes.probe import ProbeIncompatibleBrowser
17from crossbench.runner.groups.session import BrowserSessionRunGroup
18from crossbench.runner.groups.thread import RunThreadGroup
19from crossbench.runner.runner import Runner, ThreadMode
20from tests import test_helper
21from tests.crossbench.mock_browser import MockChromeDev
22from tests.crossbench.mock_helper import MockBenchmark
23from tests.crossbench.runner.helper import (BaseRunnerTestCase, MockBrowser,
24                                            MockPlatform, MockProbe,
25                                            MockProbeContext, MockRun,
26                                            MockRunner)
27
28
29# Skip strict type checks for better mocking
30# pytype: disable=wrong-arg-types
31class TestThreadModeTestCase(unittest.TestCase):
32  # pylint has some issues with enums.
33  # pylint: disable=no-member
34
35  def create_session(self, browser, index) -> BrowserSessionRunGroup:
36    return BrowserSessionRunGroup(
37        self.env,
38        self.probes,
39        browser,
40        Flags(),
41        index,
42        self.root_dir,
43        create_symlinks=True,
44        throw=True)
45
46  def setUp(self) -> None:
47    self.platform_a = MockPlatform("platform a")
48    self.platform_b = MockPlatform("platform b")
49    self.browser_a_1 = MockBrowser("mock browser a 1", self.platform_a)
50    self.browser_a_2 = MockBrowser("mock browser b 1", self.platform_a)
51    self.browser_b_1 = MockBrowser("mock browser b 1", self.platform_b)
52    self.browser_b_2 = MockBrowser("mock browser b 2", self.platform_b)
53    self.runner = MockRunner()
54    self.root_dir = pathlib.Path()
55    self.env = self.runner.env
56    self.probes = []
57    self.runs = (
58        MockRun(self.runner, self.create_session(self.browser_a_1, 1), "run 1"),
59        MockRun(self.runner, self.create_session(self.browser_a_2, 2), "run 2"),
60        MockRun(self.runner, self.create_session(self.browser_a_1, 3), "run 3"),
61        MockRun(self.runner, self.create_session(self.browser_a_2, 4), "run 4"),
62        MockRun(self.runner, self.create_session(self.browser_b_1, 5), "run 5"),
63        MockRun(self.runner, self.create_session(self.browser_b_2, 6), "run 6"),
64        MockRun(self.runner, self.create_session(self.browser_b_1, 7), "run 7"),
65        MockRun(self.runner, self.create_session(self.browser_b_2, 8), "run 8"),
66    )
67    self.runner.runs = self.runs
68
69  def test_default_runs(self):
70    session_ids = {run.browser_session.index for run in self.runs}
71    self.assertEqual(len(session_ids), len(self.runs))
72
73  def test_group_none(self):
74    groups = ThreadMode.NONE.group(self.runs)
75    self.assertEqual(len(groups), 1)
76    self.assertTupleEqual(groups[0].runs, self.runs)
77    self.assertEqual(groups[0].index, 0)
78
79  def test_group_platform(self):
80    groups = ThreadMode.PLATFORM.group(self.runs)
81    self.assertEqual(len(groups), 2)
82    group_a, group_b = groups
83    self.assertTupleEqual(group_a.runs, self.runs[:4])
84    self.assertTupleEqual(group_b.runs, self.runs[4:])
85    self.assertEqual(group_a.index, 0)
86    self.assertEqual(group_b.index, 1)
87
88  def test_group_browser(self):
89    groups = ThreadMode.BROWSER.group(self.runs)
90    self.assertEqual(len(groups), 4)
91    self.assertTupleEqual(groups[0].runs, (self.runs[0], self.runs[2]))
92    self.assertTupleEqual(groups[1].runs, (self.runs[1], self.runs[3]))
93    self.assertTupleEqual(groups[2].runs, (self.runs[4], self.runs[6]))
94    self.assertTupleEqual(groups[3].runs, (self.runs[5], self.runs[7]))
95    for index, group in enumerate(groups):
96      self.assertEqual(group.index, index)
97
98  def test_group_session(self):
99    groups = ThreadMode.SESSION.group(self.runs)
100    self.assertEqual(len(groups), len(self.runs))
101    for group, run in zip(groups, self.runs):
102      self.assertTupleEqual(group.runs, (run,))
103    for index, group in enumerate(groups):
104      self.assertEqual(group.index, index)
105
106  def test_group_session_2(self):
107    session_1 = self.create_session(self.browser_a_1, 1)
108    session_2 = self.create_session(self.browser_a_2, 2)
109    runs = (
110        MockRun(self.runner, session_1, "story 1"),
111        MockRun(self.runner, session_2, "story 2"),
112        MockRun(self.runner, session_1, "story 3"),
113        MockRun(self.runner, session_2, "story 4"),
114    )
115    groups = ThreadMode.SESSION.group(runs)
116    group_a, group_b = groups
117    self.assertTupleEqual(group_a.runs, (runs[0], runs[2]))
118    self.assertTupleEqual(group_b.runs, (runs[1], runs[3]))
119    for index, group in enumerate(groups):
120      self.assertEqual(group.index, index)
121
122
123class RunnerTestCase(BaseRunnerTestCase):
124
125  def test_default_instance(self):
126    runner = self.default_runner()
127    self.assertSequenceEqual(self.stories, runner.stories)
128    self.assertSequenceEqual(self.browsers, runner.browsers)
129    self.assertEqual(runner.repetitions, 1)
130    self.assertEqual(len(runner.platforms), 1)
131    self.assertTrue(runner.exceptions.is_success)
132    default_probes = list(runner.default_probes)
133    self.assertListEqual(list(runner.probes), default_probes)
134    self.assertEqual(len(default_probes), len(all_probes.INTERNAL_PROBES))
135    self.assertEqual(len(runner.runs), 0)
136    # no runs => is_success == false
137    self.assertFalse(runner.is_success)
138
139  def test_dry_run(self):
140    self.test_run(is_dry_run=True)
141
142  def test_run(self, is_dry_run=False):
143    runner = self.default_runner()
144
145    runner.run(is_dry_run)
146    # Don't reuse the Runner:
147    with self.assertRaises(UnexpectedStateError):
148      runner.run(is_dry_run)
149
150    self.assertEqual(len(runner.runs), 4)
151    self.assertTrue(runner.is_success)
152    for run in runner.runs:
153      self.assertTrue(run.is_success)
154      self.assertEqual(len(run.results), len(all_probes.INTERNAL_PROBES))
155      for probe in runner.probes:
156        self.assertIn(probe, run.results)
157
158  def test_run_mock_probe(self):
159    runner = self.default_runner()
160    probe = MockProbe("custom_probe_data")
161    runner.attach_probe(probe)
162    self.assertIn(probe, runner.probes)
163    for browser in runner.browsers:
164      self.assertIn(probe, browser.probes)
165
166    runner.run()
167    self.assertTrue(runner.is_success)
168    for run in runner.runs:
169      results = run.results[probe]
170      with results.json.open() as f:
171        probe_data = json.load(f)
172        self.assertEqual(probe_data, "custom_probe_data")
173      browser_dir = runner.out_dir / run.browser.unique_name
174      # Pyfakefs is having some issues with relative symlinks, thus we're
175      # manually combining the paths.
176      runs_dir = browser_dir / "runs"
177      run_symlink = runs_dir / compat.readlink(runs_dir / str(run.index))
178      self.assertEqual(run_symlink.resolve(), run.out_dir)
179    for browser in runner.browsers:
180      runs_symlinks = list(
181          (runner.out_dir / browser.unique_name / "runs").iterdir())
182      self.assertEqual(len(runs_symlinks), 2)
183
184
185  def test_attach_probe_twice(self):
186    runner = self.default_runner()
187    probe = MockProbe("custom_probe_data")
188    runner.attach_probe(probe)
189    # Cannot attach same probe twice.
190    with self.assertRaises(ValueError) as cm:
191      runner.attach_probe(probe)
192    self.assertIn("twice", str(cm.exception))
193    self.assertIn(probe, runner.probes)
194    self.assertNotIn(probe, runner.default_probes)
195
196  def test_attach_incompatible_probe(self):
197    runner = self.default_runner()
198    probe = MockProbe("custom_probe_data")
199
200    def mock_validate_browser(env: HostEnvironment, browser: Browser):
201      del env
202      nonlocal probe
203      raise ProbeIncompatibleBrowser(probe, browser, "mock invalid")
204
205    probe.validate_browser = mock_validate_browser
206    with self.assertRaises(ProbeIncompatibleBrowser) as cm:
207      runner.attach_probe(probe)
208    self.assertIn("mock invalid", str(cm.exception))
209    # matching_browser_only = True silence the error
210    runner.attach_probe(probe, matching_browser_only=True)
211    # No browser matches => probe is not available
212    self.assertNotIn(probe, runner.probes)
213    self.assertNotIn(probe, runner.default_probes)
214    for browser in self.browsers:
215      self.assertNotIn(probe, browser.probes)
216
217  def test_attach_partially_incompatible_probe(self):
218    runner = self.default_runner()
219    probe = MockProbe("custom_probe_data")
220    compatible_browser = self.browsers[1]
221
222    def mock_validate_browser(env: HostEnvironment, browser: Browser):
223      del env
224      nonlocal probe
225      nonlocal compatible_browser
226      if browser != compatible_browser:
227        raise ProbeIncompatibleBrowser(probe, browser, "mock invalid")
228
229    # Attaching incompatible probes raises errors by default.
230    probe.validate_browser = mock_validate_browser
231    with self.assertRaises(ProbeIncompatibleBrowser) as cm:
232      runner.attach_probe(probe)
233    self.assertIn("mock invalid", str(cm.exception))
234    # matching_browser_only = True silences the error
235    runner.attach_probe(probe, matching_browser_only=True)
236    self.assertIn(probe, runner.probes)
237    self.assertNotIn(probe, runner.default_probes)
238    for browser in self.browsers:
239      if browser == compatible_browser:
240        self.assertIn(probe, browser.probes)
241      else:
242        self.assertNotIn(probe, browser.probes)
243
244
245class CustomException(Exception):
246  pass
247
248
249def run_setup_fail(is_dry_run):
250  raise CustomException()
251
252
253class RunThreadGroupTestCase(BaseRunnerTestCase):
254
255  def tearDown(self) -> None:
256    for browser in self.browsers:
257      self.assertFalse(browser.is_running)
258    return super().tearDown()
259
260  def test_create_no_runs(self):
261    with self.assertRaises(AssertionError):
262      RunThreadGroup([])
263
264  def test_different_runners(self):
265    runs_a = list(self.default_runner().get_runs())
266    self.out_dir = self.out_dir.parent / "second_out_dir"
267    runner_b = Runner(
268        self.out_dir, [MockChromeDev("chrome-dev-2")],
269        self.benchmark,
270        platform=self.platform,
271        throw=True)
272    runs_b = list(runner_b.get_runs())
273    self.assertNotEqual(runs_a[0].runner, runs_b[0].runner)
274    with self.assertRaises(AssertionError) as cm:
275      RunThreadGroup(runs_a + runs_b)
276    self.assertIn("same Runner", str(cm.exception))
277
278  def test_simple_runs(self):
279    runner = self.default_runner()
280    runs = tuple(runner.get_runs())
281    thread = RunThreadGroup(runs)
282    self.assertEqual(thread.index, 0)
283    self.assertEqual(thread.runner, runner)
284    self.assertSequenceEqual(thread.runs, runs)
285    self.assertTrue(thread.is_success)
286
287    run_count = 0
288
289    def test_run(run_method):
290      nonlocal run_count
291      run_count += 1
292      run_method(is_dry_run=False)
293
294    for run in runs:
295      run.run = (  # pylint: disable=unnecessary-direct-lambda-call
296          lambda run_method: lambda is_dry_run: test_run(run_method))(
297              run.run)
298
299    thread.run()
300
301    self.assertTrue(thread.is_success)
302    self.assertSequenceEqual(thread.runs, runs)
303    self.assertEqual(run_count, 4)
304
305  def test_run_fail_run_probe_get_context(self):
306    # 2 runs, same browser different stories
307    runner = self.default_runner(browsers=[self.browsers[1]], throw=False)
308    probe = MockProbe("custom_probe_data")
309    runner.attach_probe(probe)
310    self.assertTrue(probe.is_attached)
311    runs = tuple(runner.get_runs())
312    thread = RunThreadGroup(runs)
313    failing_session, successful_session = thread.browser_sessions
314    failing_run, successful_run = runs
315
316    setup_fail_count = 0
317
318    def mock_get_context_fail(run):
319      if run == successful_run:
320        return MockProbeContext(probe, run)
321      nonlocal setup_fail_count
322      setup_fail_count += 1
323      raise CustomException()
324
325    probe.get_context = mock_get_context_fail
326
327    self.assertEqual(setup_fail_count, 0)
328    thread.run()
329    self.assertEqual(setup_fail_count, 1)
330
331    self.assertTrue(successful_session.is_success)
332    self.assertTrue(successful_run.is_success)
333
334    # Errors are propagated up:
335    for exceptions_holder in (runner, thread, failing_session, failing_run):
336      self.assertFalse(exceptions_holder.is_success)
337      exceptions = exceptions_holder.exceptions
338      self.assertEqual(len(exceptions), 1)
339      exception_entry = exceptions[0]
340      self.assertIsInstance(exception_entry.exception, CustomException)
341
342  def test_run_fail_run_probe_setup(self):
343    # 2 runs, same browser different stories
344    runner = self.default_runner(browsers=[self.browsers[1]], throw=False)
345    probe = MockProbe("custom_probe_data")
346    runner.attach_probe(probe)
347    self.assertTrue(probe.is_attached)
348    runs = tuple(runner.get_runs())
349    thread = RunThreadGroup(runs)
350    failing_session, successful_session = thread.browser_sessions
351    failing_run, successful_run = runs
352
353    setup_fail_count = 0
354
355    def mock_setup_fail() -> None:
356      nonlocal setup_fail_count
357      setup_fail_count += 1
358      raise CustomException()
359
360    def mock_get_context_fail(run):
361      context = MockProbeContext(probe, run)
362      if run == failing_run:
363        context.setup = mock_setup_fail
364      return context
365
366    probe.get_context = mock_get_context_fail
367
368    self.assertEqual(setup_fail_count, 0)
369    thread.run()
370    self.assertEqual(setup_fail_count, 1)
371
372    self.assertTrue(successful_session.is_success)
373    self.assertTrue(successful_run.is_success)
374
375    # Errors are propagated up:
376    for exceptions_holder in (runner, thread, failing_session, failing_run):
377      self.assertFalse(exceptions_holder.is_success)
378      exceptions = exceptions_holder.exceptions
379      self.assertEqual(len(exceptions), 1)
380      exception_entry = exceptions[0]
381      self.assertIsInstance(exception_entry.exception, CustomException)
382
383  def test_run_fail_one_browser_setup(self):
384    # 2 runs, same story, different browsers
385    benchmark = MockBenchmark(stories=[self.stories[0]])
386    runner = Runner(
387        self.out_dir, self.browsers, benchmark, platform=self.platform)
388    runs = tuple(runner.get_runs())
389    thread = RunThreadGroup(runs)
390    failing_session, successful_session = thread.browser_sessions
391    failing_run, successful_run = runs
392    self.assertNotEqual(failing_run.browser, successful_run.browser)
393
394    setup_fail_count = 0
395
396    def mock_start_fail(session: BrowserSessionRunGroup) -> None:
397      del session
398      nonlocal setup_fail_count
399      setup_fail_count += 1
400      raise CustomException()
401
402    failing_run.browser.start = mock_start_fail
403
404    self.assertEqual(setup_fail_count, 0)
405    thread.run()
406    self.assertEqual(setup_fail_count, 1)
407
408    self.assertTrue(successful_session.is_success)
409    self.assertTrue(successful_run.is_success)
410
411    # browser startup failures should also propagate down to all runs.
412    for exceptions_holder in (runner, thread, failing_session, failing_run):
413      self.assertFalse(exceptions_holder.is_success)
414      exceptions = exceptions_holder.exceptions
415      self.assertEqual(len(exceptions), 1)
416      exception_entry = exceptions[0]
417      self.assertIsInstance(exception_entry.exception, CustomException)
418
419  def test_run_fail_run(self):
420    # 4 runs = (2 browser) x (2 stories)
421    runner = self.default_runner(throw=False)
422    runs = tuple(runner.get_runs())
423    thread = RunThreadGroup(runs)
424    failing_run = runs[0]
425    failing_session = failing_run.browser_session
426
427    run_fail_count = 0
428
429    def mock_run_story_fail():
430      nonlocal run_fail_count
431      run_fail_count += 1
432      raise CustomException()
433
434    with mock.patch.object(failing_run, "_run_story", mock_run_story_fail):
435      self.assertEqual(run_fail_count, 0)
436      thread.run()
437      self.assertEqual(run_fail_count, 1)
438
439      for session in thread.browser_sessions:
440        if session != failing_run.browser_session:
441          self.assertTrue(session.is_success)
442      for run in runs:
443        if run != failing_run:
444          self.assertTrue(run.is_success)
445
446      # Errors are propagate up:
447      for exceptions_holder in (runner, thread, failing_session, failing_run):
448        self.assertFalse(exceptions_holder.is_success)
449        exceptions = exceptions_holder.exceptions
450        self.assertEqual(len(exceptions), 1)
451        exception_entry = exceptions[0]
452        self.assertIsInstance(exception_entry.exception, CustomException)
453
454# pytype: enable=wrong-arg-types
455
456if __name__ == "__main__":
457  test_helper.run_pytest(__file__)
458