• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2024 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import abc
6import datetime as dt
7import json
8import pathlib
9from typing import Any, List, Optional
10
11from crossbench.browsers.browser import Browser
12from crossbench.browsers.settings import Settings
13from crossbench.env import HostEnvironment
14from crossbench.exception import Annotator
15from crossbench.path import safe_filename
16from crossbench.probes.probe import Probe
17from crossbench.probes.probe_context import ProbeContext
18from crossbench.probes.results import LocalProbeResult, ProbeResult
19from crossbench.runner.actions import Actions
20from crossbench.runner.run import Run
21from crossbench.runner.runner import Runner
22from crossbench.runner.timing import Timing
23from tests.crossbench.base import BaseCrossbenchTestCase
24from tests.crossbench.mock_browser import MockChromeDev, MockFirefox
25from tests.crossbench.mock_helper import MockBenchmark, MockStory
26
27
28class MockBrowser:
29
30  def __init__(self, unique_name: str, platform) -> None:
31    self.unique_name = unique_name
32    self.platform = platform
33    self.network = MockNetwork()
34
35  def __str__(self):
36    return self.unique_name
37
38
39class MockRun:
40
41  def __init__(self,
42               runner,
43               browser_session,
44               story="story",
45               repetition=0,
46               is_warmup=False,
47               temperature="default",
48               index=0,
49               name="run 0") -> None:
50    self.runner = runner
51    self.browser_session = browser_session
52    self.browser = browser_session.browser
53    self.browser_platform = self.browser.platform
54    self._exceptions = Annotator(False)
55    self.repetition = repetition
56    self.is_warmup = is_warmup
57    self.temperature = temperature
58    self.name = name
59    self.probes = []
60    self.timing = Timing()
61    self.is_success = True
62    self.index = index
63    self.story = story
64    self.out_dir = (
65        browser_session.root_dir / safe_filename(self.browser.unique_name) /
66        "stories" / name / f"repetition={self.repetition}" / self.temperature)
67    self.group_dir = self.out_dir.parent
68    self.did_setup = False
69    self.did_run = False
70    self.did_teardown = False
71    self.did_teardown_browser = False
72    self.is_dry_run: Optional[bool] = None
73
74  def validate_env(self, env: HostEnvironment):
75    pass
76
77  def setup(self, is_dry_run: bool) -> None:
78    assert self.is_dry_run is None
79    self.is_dry_run = is_dry_run
80    assert not self.did_setup
81    self.did_setup = True
82
83  def actions(self,
84              name: str,
85              verbose: bool = False,
86              measure: bool = True) -> Actions:
87    return Actions(name, self, verbose=verbose, measure=measure)
88
89  @property
90  def exceptions(self) -> Annotator:
91    return self._exceptions
92
93  def max_end_datetime(self) -> dt.datetime:
94    return dt.datetime.max
95
96  def run(self, is_dry_run: bool) -> None:
97    assert self.is_dry_run is is_dry_run
98    assert not self.did_run
99    self.did_run = True
100
101  def teardown(self, is_dry_run: bool) -> None:
102    assert self.is_dry_run is is_dry_run
103    assert not self.did_teardown
104    self.did_teardown = True
105
106  def _teardown_browser(self, is_dry_run: bool) -> None:
107    assert self.is_dry_run is is_dry_run
108    assert not self.did_teardown_browser
109    self.did_teardown_browser = True
110    self.browser.quit()
111
112  def __repr__(self):
113    return f"MockRun({self.name}, id={hex(id(self))})"
114
115  def __str__(self):
116    return self.name
117
118
119class MockPlatform:
120
121  def __init__(self, name) -> None:
122    self.name = name
123
124  def __str__(self):
125    return self.name
126
127
128class MockRunner:
129
130  def __init__(self) -> None:
131    self.benchmark = MockBenchmark(stories=[MockStory("mock_story")])
132    self.runs = tuple()
133    self.platform = MockPlatform("test-platform")
134    self.repetitions = 1
135    self.create_symlinks = True
136    self.probes = []
137    self.browsers = []
138    self.out_dir = pathlib.Path("results/out")
139    self.timing = Timing()
140    self.env = HostEnvironment(self.platform, self.out_dir, self.browsers,
141                               self.probes, self.repetitions)
142
143
144class MockNetwork:
145  pass
146
147
148class MockProbe(Probe):
149  NAME = "test-probe"
150
151  def __init__(self, test_data: Any = ()) -> None:
152    super().__init__()
153    self.test_data = test_data
154
155  @property
156  def result_path_name(self) -> str:
157    return f"{self.name}.json"
158
159  def get_context(self, run: Run):
160    return MockProbeContext(self, run)
161
162
163class MockProbeContext(ProbeContext):
164
165  def start(self) -> None:
166    pass
167
168  def stop(self) -> None:
169    pass
170
171  def teardown(self) -> ProbeResult:
172    with self.result_path.open("w") as f:
173      json.dump(self.probe.test_data, f)
174    return LocalProbeResult(json=(self.result_path,))
175
176
177class BaseRunnerTestCase(BaseCrossbenchTestCase, metaclass=abc.ABCMeta):
178
179  def setUp(self):
180    super().setUp()
181    self.out_dir = pathlib.Path("/testing/out_dir")
182    self.out_dir.parent.mkdir(exist_ok=False, parents=True)
183    self.stories = [MockStory("story_1"), MockStory("story_2")]
184    self.benchmark = MockBenchmark(self.stories)
185    self.browsers: List[Browser] = [
186        MockChromeDev("chrome-dev", settings=Settings(platform=self.platform)),
187        MockFirefox(
188            "firefox-stable", settings=Settings(platform=self.platform))
189    ]
190
191  def default_runner(self,
192                     browsers: Optional[List[Browser]] = None,
193                     throw: bool = True) -> Runner:
194    if browsers is None:
195      browsers = self.browsers
196    return Runner(
197        self.out_dir,
198        browsers,
199        self.benchmark,
200        platform=self.platform,
201        throw=throw)
202