• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2023 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import annotations
6
7import datetime as dt
8import enum
9import logging
10from typing import TYPE_CHECKING, Optional, Type
11
12from crossbench import compat
13from crossbench import path as pth
14from crossbench.exception import Annotator, TInfoStack
15from crossbench.helper import ChangeCWD, Durations, Spinner
16from crossbench.helper.state import State, StateMachine
17from crossbench.probes.probe_context import ProbeContext
18from crossbench.probes.results import ProbeResultDict
19from crossbench.runner.actions import Actions
20from crossbench.runner.exception import StopStoryException
21from crossbench.runner.probe_context_manager import ProbeContextManager
22from crossbench.runner.result_origin import ResultOrigin
23from crossbench.runner.timing import Timing
24
25if TYPE_CHECKING:
26  from selenium.webdriver.common.options import ArgOptions
27
28  from crossbench.benchmarks.base import Benchmark
29  from crossbench.browsers.browser import Browser
30  from crossbench.env import HostEnvironment
31  from crossbench.probes.probe import Probe, ProbeT
32  from crossbench.runner.groups.session import BrowserSessionRunGroup
33  from crossbench.runner.probe_context_manager import ProbeContextT
34  from crossbench.runner.runner import Runner
35  from crossbench.stories.story import Story
36  from crossbench.types import JsonDict
37
38
39@enum.unique
40class Temperature(compat.StrEnumWithHelp):
41  COLD = ("cold", "first run")
42  WARM = ("warm", "second run")
43  HOT = ("hot", "third run")
44
45
46class Run(ResultOrigin):
47
48  def __init__(self,
49               runner: Runner,
50               browser_session: BrowserSessionRunGroup,
51               story: Story,
52               repetition: int,
53               is_warmup: bool,
54               temperature: str,
55               index: int,
56               name: Optional[str] = None,
57               timeout: dt.timedelta = dt.timedelta(),
58               throw: bool = False):
59    self._state = StateMachine(State.INITIAL)
60    self._runner = runner
61    self._browser_session = browser_session
62    self._browser: Browser = browser_session.browser
63    browser_session.append(self)
64    self._story = story
65    assert repetition >= 0
66    self._repetition = repetition
67    self._is_warmup = is_warmup
68    assert temperature, "Missing cache-temperature value."
69    self._temperature = temperature
70    assert index >= 0
71    self._index = index
72    self._name = name
73    self._out_dir = self._get_out_dir().absolute()
74    self._probe_results = ProbeResultDict(self._out_dir)
75    self._durations = Durations()
76    self._start_datetime = dt.datetime.utcfromtimestamp(0)
77    self._timeout = timeout
78    self._exceptions = Annotator(throw)
79    self._browser_tmp_dir: Optional[pth.AnyPath] = None
80    self._probe_context_manager = ProbeRunContextManager(
81        self, self._probe_results)
82
83  def __str__(self) -> str:
84    return f"Run({self.name}, {self._state}, {self.browser})"
85
86  def _get_out_dir(self) -> pth.LocalPath:
87    return (self._browser_session.browser_dir / "stories" /
88            pth.safe_filename(self.story.name) / str(self.repetition_name) /
89            str(self._temperature))
90
91  @property
92  def group_dir(self) -> pth.LocalPath:
93    return self.out_dir.parent
94
95  def actions(self,
96              name: str,
97              verbose: bool = False,
98              measure: bool = True) -> Actions:
99    return Actions(name, self, verbose=verbose, measure=measure)
100
101  @property
102  def info_stack(self) -> TInfoStack:
103    return (
104        f"Run({self.name})",
105        (f"browser={self.browser.type_name} label={self.browser.label} "
106         f"binary={self.browser.path}"),
107        f"story={self.story}",
108        f"repetition={self.repetition_name}",
109    )
110
111  def details_json(self) -> JsonDict:
112    return {
113        "cwd": str(self.out_dir),
114        "name": self.name,
115        "story": self.story.details_json(),
116        "browser": self.get_browser_details_json(),
117        "run": {
118            "name": self.name,
119            "index": self.index,
120            "repetition": self.repetition,
121            "temperature": self.temperature,
122            "isWarmup": self.is_warmup,
123        },
124        "session": {
125            "index": self.browser_session.index,
126            "cwd": str(self.browser_session.path)
127        },
128        "probes": self.results.to_json(),
129        "timing": {
130            "startDateTime": str(self.start_datetime),
131            "duration": self.story.duration.total_seconds(),
132            "durations": self.durations.to_json(),
133            "timeout": self.timeout.total_seconds(),
134            "global": self.timing.to_json(),
135        },
136        "success": self.is_success,
137        "errors": self.exceptions.error_messages()
138    }
139
140  @property
141  def temperature(self) -> str:
142    return self._temperature
143
144  @property
145  def timing(self) -> Timing:
146    return self.runner.timing
147
148  @property
149  def durations(self) -> Durations:
150    return self._durations
151
152  @property
153  def start_datetime(self) -> dt.datetime:
154    return self._start_datetime
155
156  def max_end_datetime(self) -> dt.datetime:
157    if not self._timeout:
158      return dt.datetime.max
159    return self._start_datetime + self._timeout
160
161  @property
162  def timeout(self) -> dt.timedelta:
163    return self._timeout
164
165  @property
166  def repetition_name(self) -> str:
167    if self.is_warmup:
168      return f"warmup_{self.repetition}"
169    return str(self.repetition)
170
171  @property
172  def repetition(self) -> int:
173    return self._repetition
174
175  @property
176  def is_warmup(self) -> bool:
177    return self._is_warmup
178
179  @property
180  def index(self) -> int:
181    return self._index
182
183  @property
184  def runner(self) -> Runner:
185    return self._runner
186
187  @property
188  def benchmark(self) -> Benchmark:
189    return self._runner.benchmark
190
191  @property
192  def browser_session(self) -> BrowserSessionRunGroup:
193    return self._browser_session
194
195  @property
196  def browser(self) -> Browser:
197    return self._browser
198
199  @property
200  def environment(self) -> HostEnvironment:
201    # TODO: replace with custom BrowserEnvironment
202    return self.runner.env
203
204  @property
205  def out_dir(self) -> pth.LocalPath:
206    """A local directory where all result files are gathered.
207    Results from browsers on remote platforms are transferred to this dir
208    as well."""
209    return self._out_dir
210
211  @property
212  def browser_tmp_dir(self) -> pth.AnyPath:
213    """Returns a path to a tmp dir on the browser platform."""
214    if not self._browser_tmp_dir:
215      prefix = "cb_run_results"
216      self._browser_tmp_dir = self.browser_platform.mkdtemp(prefix)
217    return self._browser_tmp_dir
218
219  @property
220  def results(self) -> ProbeResultDict:
221    return self._probe_results
222
223  @property
224  def story(self) -> Story:
225    return self._story
226
227  @property
228  def name(self) -> Optional[str]:
229    return self._name
230
231  @property
232  def exceptions(self) -> Annotator:
233    return self._exceptions
234
235  @property
236  def is_success(self) -> bool:
237    return self._exceptions.is_success
238
239  @property
240  def session(self) -> BrowserSessionRunGroup:
241    return self._browser_session
242
243  def get_browser_details_json(self) -> JsonDict:
244    details_json = self.browser.details_json()
245    self.session.add_flag_details(details_json)
246    return details_json
247
248  def get_local_probe_result_path(self, probe: Probe) -> pth.LocalPath:
249    file = self._out_dir / probe.result_path_name
250    assert not file.exists(), f"Probe results file exists already. file={file}"
251    return file
252
253  def validate_env(self, env: HostEnvironment) -> None:
254    """Called before starting a browser / browser session to perform
255    a pre-run checklist."""
256
257  def setup(self, is_dry_run: bool) -> None:
258    self._state.transition(State.INITIAL, to=State.SETUP)
259    self._setup_dirs()
260    with ChangeCWD(self._out_dir), self.exception_info(*self.info_stack):
261      self._probe_context_manager.setup(self.probes, is_dry_run)
262    self._log_setup()
263
264  def setup_selenium_options(self, options: ArgOptions):
265    # TODO: move explicitly to session.
266    self._probe_context_manager.setup_selenium_options(options)
267
268  def _setup_dirs(self) -> None:
269    self._start_datetime = dt.datetime.now()
270    logging.debug("Creating Run(%s) out dir: %s", self, self._out_dir)
271    self._out_dir.mkdir(parents=True, exist_ok=True)
272    if not self.runner.create_symlinks:
273      logging.debug("Symlinks disabled by command line option")
274      return
275    self._create_runs_dir()
276    self._create_session_dir()
277
278  def _create_runs_dir(self) -> None:
279    browser_dir = self.browser_session.browser_dir
280    runs_dir = browser_dir / "runs"
281    runs_dir.mkdir(parents=True, exist_ok=True)
282    # Source: BROWSER / "runs" / RUN
283    # Target: BROWSER / "stories" / STORY / REPETITION / CACHE_TEMP
284    run_dir = runs_dir / str(self.index)
285    relative_out_dir = (
286        pth.LocalPath("../") / self.out_dir.relative_to(browser_dir))
287    run_dir.symlink_to(relative_out_dir, target_is_directory=True)
288
289  def _create_session_dir(self) -> None:
290    session_run_dir = self._out_dir / "session"
291    assert not session_run_dir.exists(), (
292        f"Cannot setup session dir twice: {session_run_dir}")
293    if self.host_platform.is_win:
294      logging.debug("Skipping session_dir symlink on windows.")
295      return
296    # Source: BROWSER / "stories" / STORY / REPETITION / CACHE_TEMP / "session"
297    # Target: BROWSER / "sessions" / SESSION
298    relative_session_dir = (
299        pth.LocalPath("../../../..") /
300        self.browser_session.path.relative_to(self.out_dir.parents[3]))
301    session_run_dir.symlink_to(relative_session_dir, target_is_directory=True)
302
303  def _log_setup(self) -> None:
304    logging.debug("SETUP")
305    logging.info(
306        "PROBES: %s",
307        ", ".join(probe.NAME for probe in self.probes if not probe.is_internal))
308    logging.debug("PROBES ALL: %s",
309                  ", ".join(probe.NAME for probe in self.probes))
310    self.story.log_run_details(self)
311    logging.info("RUN DIR: %s", self._out_dir)
312    logging.debug("CWD %s", self._out_dir)
313
314  def run(self, is_dry_run: bool) -> None:
315    self._state.transition(State.SETUP, to=State.READY)
316    self._start_datetime = dt.datetime.now()
317    with ChangeCWD(self._out_dir), self.exception_info(*self.info_stack):
318      assert self._probe_context_manager.is_ready
319      try:
320        self._run(is_dry_run)
321      except Exception as e:  # pylint: disable=broad-except
322        self._exceptions.append(e)
323      finally:
324        self.teardown(is_dry_run)
325
326  def _run(self, is_dry_run: bool) -> None:
327    self._state.transition(State.READY, to=State.RUN)
328    self.browser.splash_screen.run(self)
329    with self._probe_context_manager.open(is_dry_run):
330      logging.info("RUNNING STORY")
331      self._state.expect(State.RUN)
332      try:
333        with self.measure("run"), Spinner(), self.exceptions.capture():
334          if not is_dry_run:
335            self._run_story()
336      except TimeoutError as e:
337        # Handle TimeoutError earlier since they might be caused by
338        # throttled down non-foreground browser.
339        self._exceptions.append(e)
340      if self.is_success:
341        with self.exceptions.capture():
342          self.environment.check_browser_focused(self.browser)
343
344  def _run_story(self) -> None:
345    self._run_story_setup()
346    try:
347      self._story.run(self)
348    except StopStoryException as e:
349      logging.debug("Stop story: %s", e)
350    finally:
351      self._run_story_teardown()
352
353  def _run_story_setup(self) -> None:
354    with self.measure("story-setup"):
355      self._story.setup(self)
356    self._probe_context_manager.start_story()
357
358  def _run_story_teardown(self) -> None:
359    self._probe_context_manager.stop_story()
360    with self.measure("story-tear-down"):
361      self._story.teardown(self)
362
363  def teardown(self, is_dry_run: bool) -> None:
364    self._state.transition(State.RUN, to=State.DONE)
365    self._teardown_browser(is_dry_run)
366    self._probe_context_manager.teardown(is_dry_run)
367    if not is_dry_run:
368      self._rm_browser_tmp_dir()
369
370  def _teardown_browser(self, is_dry_run: bool) -> None:
371    if is_dry_run:
372      return
373    if not self.browser_session.is_last_run(self):
374      logging.debug("Skipping browser teardown (not last in session): %s", self)
375      return
376    if self._browser.is_running is False:
377      logging.warning("Browser is no longer running (crashed or closed).")
378      return
379    with self.measure("browser-teardown"), self._exceptions.capture(
380        "Quit browser"):
381      try:
382        self._browser.quit()
383      except Exception as e:  # pylint: disable=broad-except
384        logging.warning("Error quitting browser: %s", e)
385        return
386
387  def _rm_browser_tmp_dir(self) -> None:
388    if not self._browser_tmp_dir:
389      return
390    self.browser_platform.rm(self._browser_tmp_dir, dir=True)
391
392  def log_results(self) -> None:
393    for probe in self.probes:
394      probe.log_run_result(self)
395
396  def find_probe_context(self,
397                         cls: Type[ProbeT]) -> Optional[ProbeContext[ProbeT]]:
398    return self._probe_context_manager.find_probe_context(cls)
399
400
401class ProbeRunContextManager(ProbeContextManager[Run, ProbeContext]):
402
403  def __init__(self, run: Run, probe_results: ProbeResultDict):
404    super().__init__(run, probe_results)
405
406  def get_probe_context(self, probe: Probe) -> Optional[ProbeContext]:
407    return probe.get_context(self._origin)
408
409  def setup_selenium_options(self, options: ArgOptions):
410    for probe_context in self._probe_contexts.values():
411      probe_context.setup_selenium_options(options)
412
413  def start_story(self) -> None:
414    with self.measure("probes-start_story_run"):
415      for probe_context in self._probe_contexts.values():
416        with self._origin.exception_handler(
417            f"Probe {probe_context.name} start_story_run"):
418          probe_context.start_story_run()
419
420  def stop_story(self) -> None:
421    with self.measure("probes-stop_story_run"):
422      for probe_context in self._probe_contexts.values():
423        with self._origin.exception_handler(
424            f"Probe {probe_context.name} stop_story_run"):
425          probe_context.stop_story_run()
426