• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2024 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from unittest import mock
6
7from crossbench import compat
8from crossbench.helper.state import UnexpectedStateError
9from tests import test_helper
10from tests.crossbench.runner.groups.base import BaseRunGroupTestCase
11from tests.crossbench.runner.helper import MockProbe, MockRun
12
13# Due to laziness we access internal variables in the test here.
14# Adding an accessor would wrongly hint that these variables are public.
15# pylint: disable=protected-access
16
17class BrowserSessionRunGroupTestCase(BaseRunGroupTestCase):
18
19  def test_basic_properties(self):
20    session = self.default_session()
21    self.assertEqual(session.index, 0)
22    self.assertIs(session.browser, self.browsers[0])
23    self.assertFalse(session.is_single_run)
24    self.assertFalse(session.is_running)
25    self.assertEqual(session.root_dir, self.root_dir)
26    self.assertFalse(session.extra_flags)
27    self.assertFalse(session.extra_js_flags)
28    self.assertIn("0", str(session.info_stack))
29    self.assertIn(str(self.browsers[0].unique_name), str(session.info_stack))
30    self.assertEqual(session.info["runs"], 0)
31    self.assertEqual(session.info["index"], 0)
32    self.assertIn("0", str(session))
33    self.assertIn(str(self.browsers[0]), str(session))
34    self.assertTrue(session.browser_tmp_dir.is_dir())
35    with self.assertRaises(IndexError):
36      _ = session.timing
37
38  def test_out_dir_single_run(self):
39    session = self.default_session()
40    with self.assertRaises(UnexpectedStateError):
41      _ = session.out_dir
42    run_1 = MockRun(self.runner, session, "story 1")
43    session.append(run_1)
44    with self.assertRaises(UnexpectedStateError):
45      _ = session.out_dir
46    session.set_ready()
47    self.assertEqual(session.out_dir, run_1.out_dir)
48    self.assertNotEqual(session.out_dir, session.raw_session_dir)
49    # session dirs are only created when opening the session.
50    self.assertFalse(session.out_dir.exists())
51
52  def test_out_dir_multiple_runs(self):
53    session = self.default_session()
54    run_1 = MockRun(self.runner, session, "story 1")
55    run_2 = MockRun(self.runner, session, "story 2")
56    session.append(run_1)
57    session.append(run_2)
58    session.set_ready()
59    self.assertNotEqual(session.out_dir, run_1.out_dir)
60    self.assertEqual(session.out_dir, session.raw_session_dir)
61
62  def test_append(self):
63    session = self.default_session()
64    run_1 = MockRun(self.runner, session, "story 1")
65    session.append(run_1)
66    self.assertListEqual(list(session.runs), [run_1])
67    self.assertEqual(session.info["runs"], 1)
68    self.assertTrue(session.is_single_run)
69    self.assertFalse(session.is_running)
70    self.assertIs(session.first_run, run_1)
71    self.assertIs(session.timing, run_1.timing)
72
73    run_2 = MockRun(self.runner, session, "story 2")
74    session.append(run_2)
75    self.assertListEqual(list(session.runs), [run_1, run_2])
76    self.assertEqual(session.info["runs"], 2)
77
78    session.set_ready()
79    self.assertFalse(session.is_single_run)
80    self.assertFalse(session.is_running)
81    self.assertIs(session.first_run, run_1)
82    self.assertFalse(session.extra_flags)
83    self.assertFalse(session.extra_js_flags)
84
85    self.assertTrue(session.is_first_run(run_1))
86    self.assertFalse(session.is_first_run(run_2))
87
88  def test_append_after_ready(self):
89    session = self.default_session()
90    run_1 = MockRun(self.runner, session, "story 1")
91    session.append(run_1)
92    session.set_ready()
93    with self.assertRaises(UnexpectedStateError):
94      session.append(MockRun(self.runner, session, "story 3"))
95
96  def test_append_wrong_session(self):
97    session_1 = self.default_session()
98    run_1 = MockRun(self.runner, session_1, "run 0")
99    session_1.append(run_1)
100    session_2 = self.default_session(self.browsers[1])
101    run_2 = MockRun(self.runner, session_2, "run 0")
102    with self.assertRaises(AssertionError):
103      session_1.append(run_2)
104    run_3 = MockRun(self.runner, session_1, "run 0")
105    run_3.browser = self.browsers[1]
106    with self.assertRaises(AssertionError):
107      session_1.append(run_3)
108
109  def test_append_different_probes(self):
110    session = self.default_session()
111    run_1 = MockRun(self.runner, session, "story 0")
112    run_1.probes = []
113    run_2 = MockRun(self.runner, session, "story 0")
114    run_2.probes = [MockProbe()]
115    session.append(run_1)
116    session.append(run_2)
117    with self.assertRaises(ValueError):
118      session.set_ready()
119
120  def test_set_ready(self):
121    with self.assertRaises(ValueError):
122      session = self.default_session()
123      session.set_ready()
124    session = self.default_session()
125    session.append(MockRun(self.runner, session, "story 0"))
126    session.set_ready()
127    self.assertFalse(session.extra_flags)
128    self.assertFalse(session.extra_js_flags)
129
130  def test_open_not_ready(self):
131    session = self.default_session()
132    self.assertFalse(session.is_running)
133    did_run = False
134    with self.assertRaises(UnexpectedStateError):
135      with session.open():
136        did_run = True
137    self.assertFalse(session.is_running)
138    self.assertFalse(did_run)
139
140  def test_open_not_ready_with_run(self):
141    session = self.default_session()
142    run = MockRun(self.runner, session, "story 0")
143    session.append(run)
144    self.assertFalse(session.is_running)
145    did_run = False
146    with self.assertRaises(UnexpectedStateError):
147      with session.open():
148        did_run = True
149    self.assertFalse(session.is_running)
150    self.assertTrue(session.is_success)
151    self.assertFalse(run.did_setup)
152    self.assertFalse(run.did_run)
153    self.assertFalse(did_run)
154
155  def test_open(self):
156    session = self.default_session()
157    run = MockRun(self.runner, session, "story 0")
158    session.append(run)
159    session.set_ready()
160    self.assertFalse(session.is_running)
161    self.assertFalse(run.did_setup)
162    self.assertFalse(run.did_teardown)
163    did_run = False
164    with session.open() as startup_is_success:
165      self.assertTrue(session.is_running)
166      self.assertTrue(run.did_setup)
167      self.assertTrue(session.browser.is_running)
168      self.assertFalse(run.did_teardown_browser)
169      self.assertTrue(session._probe_context_manager.is_running)
170      # runs would be triggered here...
171      did_run = True
172    self.assertTrue(startup_is_success)
173    self.assertFalse(session.is_running)
174    self.assertFalse(session.browser.is_running)
175    self.assertFalse(session._probe_context_manager.is_running)
176    self.assertTrue(session.is_success)
177    self.assertTrue(session.path.is_dir())
178    session_symlinks = list((session.browser_dir / "sessions").iterdir())
179    self.assertEqual(len(session_symlinks), 1)
180    self.assertEqual(compat.readlink(session_symlinks[0]), session.path)
181    self.assertTrue(run.did_setup)
182    self.assertFalse(run.did_run)
183    self.assertTrue(run.did_teardown_browser)
184    self.assertTrue(did_run)
185    # Using mock runs here... didn't create runs symlinks
186    self.assertFalse((session.browser_dir / "runs").exists())
187
188  def test_open_dry_run(self):
189    session = self.default_session()
190    run = MockRun(self.runner, session, "story 0")
191    session.append(run)
192    session.set_ready()
193    self.assertFalse(session.is_running)
194    did_run = False
195    with session.open(is_dry_run=True) as startup_is_success:
196      self.assertTrue(session.is_running)
197      self.assertFalse(session.browser.is_running)
198      self.assertTrue(session._probe_context_manager.is_running)
199      # runs would be triggered here...
200      did_run = True
201    self.assertTrue(startup_is_success)
202    self.assertFalse(session.is_running)
203    self.assertFalse(session.browser.is_running)
204    self.assertFalse(session._probe_context_manager.is_running)
205    self.assertTrue(run.did_setup)
206    self.assertFalse(run.did_teardown_browser)
207    self.assertTrue(did_run)
208
209  def test_open_inner_throw(self):
210    session = self.default_session(throw=True)
211    run = MockRun(self.runner, session, "story 0")
212    session.append(run)
213    session.set_ready()
214    did_run = False
215    with self.assertRaises(ValueError):
216      with session.open() as startup_is_success:
217        self.assertTrue(session.browser.is_running)
218        self.assertFalse(run.did_teardown_browser)
219        self.assertTrue(session._probe_context_manager.is_running)
220        did_run = True
221        raise ValueError("Test run failed")
222    self.assertTrue(startup_is_success)
223    self.assertTrue(did_run)
224    self._validate_post_inner_throw(session, run)
225
226  def _validate_post_inner_throw(self, session, run):
227    # Startup succeed, the inner evaluation failed.
228    self.assertFalse(session._probe_context_manager.is_running)
229    self.assertFalse(session.browser.is_running)
230    self.assertFalse(session.is_running)
231    self.assertFalse(session.is_success)
232    self.assertTrue(run.did_setup)
233    self.assertTrue(run.did_teardown_browser)
234    self.assertIn("Test run failed", str(session.exceptions[0].exception))
235
236  def test_open_inner_throw_capture(self):
237    session = self.default_session(throw=False)
238    run = MockRun(self.runner, session, "story 0")
239    session.append(run)
240    session.set_ready()
241    did_run = False
242    with session.open() as startup_is_success:
243      self.assertTrue(session.browser.is_running)
244      self.assertFalse(run.did_teardown_browser)
245      self.assertTrue(session._probe_context_manager.is_running)
246      did_run = True
247      raise ValueError("Test run failed")
248    self.assertTrue(did_run)
249    # Startup succeed, the inner evaluation failed.
250    self.assertTrue(startup_is_success)
251    self._validate_post_inner_throw(session, run)
252    self.assertEqual(len(session.exceptions), 1)
253
254  def test_open_network_error(self):
255    session = self.default_session(throw=False)
256    run = MockRun(self.runner, session, "story 0")
257    session.append(run)
258    session.set_ready()
259    did_run = False
260    with mock.patch.object(
261        session.network,
262        "open",
263        side_effect=ValueError("Network startup error")):
264      with session.open() as startup_is_success:
265        # Due to how context managers work we run the inner code even if
266        # the setup failed.
267        self.assertFalse(startup_is_success)
268        did_run = True
269    self.assertTrue(did_run)
270    self.assertEqual(len(session.exceptions), 1)
271    self._validate_open_network_error(session, run)
272
273  def test_open_network_error_throw(self):
274    session = self.default_session(throw=True)
275    run = MockRun(self.runner, session, "story 0")
276    session.append(run)
277    session.set_ready()
278    did_run = False
279    with self.assertRaises(ValueError) as cm:
280      with mock.patch.object(
281          session.network,
282          "open",
283          side_effect=ValueError("Network startup error")):
284        with session.open():
285          did_run = True
286    self.assertFalse(did_run)
287    self.assertIn("Network startup error", str(cm.exception))
288    self._validate_open_network_error(session, run)
289
290  def _validate_open_network_error(self, session, run):
291    self.assertFalse(session._probe_context_manager.is_running)
292    self.assertFalse(session.browser.is_running)
293    self.assertFalse(session.is_running)
294    self.assertFalse(session.is_success)
295    self.assertTrue(run.did_setup)
296    self.assertFalse(run.did_teardown_browser)
297    self.assertIn("Network startup error", str(session.exceptions[0].exception))
298
299if __name__ == "__main__":
300  test_helper.run_pytest(__file__)
301