• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2023 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""
5An adapter class for CBB to interact with the crossbench runner.
6The goal is to abstract out the crossbench runner interface details and
7provide an integration point for CBB.
8
9Any breaking changes in the function definitions here need to be coordinated
10with corresponding changes in CBB in google3
11"""
12
13from __future__ import annotations
14
15import datetime as dt
16from typing import TYPE_CHECKING, List, Optional, Type
17
18from selenium import webdriver
19
20import crossbench.benchmarks.all as benchmarks
21import crossbench.browsers.browser
22import crossbench.browsers.webdriver as cb_webdriver
23import crossbench.env
24import crossbench.runner.runner
25from crossbench import path as pth
26from crossbench.runner.run import Run
27
28if TYPE_CHECKING:
29  from crossbench.benchmarks.base import PressBenchmark
30  from crossbench.runner.groups.session import BrowserSessionRunGroup
31  from crossbench.stories.press_benchmark import PressBenchmarkStory
32  from crossbench.stories.story import Story
33
34press_benchmarks: List[Type[PressBenchmark]] = [
35    benchmarks.Speedometer20Benchmark,
36    benchmarks.Speedometer21Benchmark,
37    benchmarks.Speedometer30Benchmark,
38    benchmarks.MotionMark12Benchmark,
39    benchmarks.MotionMark13Benchmark,
40    benchmarks.JetStream20Benchmark,
41    benchmarks.JetStream21Benchmark,
42    benchmarks.JetStream22Benchmark,
43    benchmarks.JetStream30Benchmark,
44]
45
46press_benchmarks_dict = {cls.NAME: cls for cls in press_benchmarks}
47
48
49def get_pressbenchmark_cls(
50    benchmark_name: str) -> Optional[Type[PressBenchmark]]:
51  """Returns the class of the specified pressbenchmark.
52
53  Args:
54    benchmark_name: Name of the benchmark.
55
56  Returns:
57    An instance of the benchmark implementation
58  """
59  return press_benchmarks_dict.get(benchmark_name)
60
61
62def get_pressbenchmark_story_cls(
63    benchmark_name: str) -> Optional[Type[PressBenchmarkStory]]:
64  """Returns the class of the specified pressbenchmark story.
65
66  Args:
67    benchmark_name: Name of the benchmark.
68
69  Returns:
70    An instance of the benchmark default story
71  """
72
73  benchmark_cls = get_pressbenchmark_cls(benchmark_name)
74  if benchmark_cls is not None:
75    return benchmark_cls.DEFAULT_STORY_CLS
76
77  return None
78
79
80def create_remote_webdriver(driver: webdriver.Remote
81                           ) -> cb_webdriver.RemoteWebDriver:
82  """Creates a remote webdriver instance for the driver.
83
84  Args:
85    driver: Remote web driver instance.
86  """
87
88  browser = cb_webdriver.RemoteWebDriver("default", driver)
89  browser.version = driver.capabilities["browserVersion"]
90  return browser
91
92
93def get_probe_result_file(benchmark_name: str,
94                          browser: crossbench.browsers.browser.Browser,
95                          output_dir: pth.LocalPathLike,
96                          probe_name: Optional[str] = None) -> Optional[str]:
97  """Returns the path to the probe result file.
98
99  Args:
100    benchmark_name: Name of the benchmark.
101    browser: Browser instance.
102    output_dir: Path to the directory where the output of the benchmark
103                execution was written.
104    probe_name: Optional name of the probe for the result file. If not
105                specified, the first probe from the default benchmark story
106                will be used."""
107  output_dir_path = pth.LocalPath(output_dir)
108  if probe_name is None:
109    if benchmark_name not in press_benchmarks_dict:
110      return None
111    benchmark_cls = press_benchmarks_dict[benchmark_name]
112    probe_cls = benchmark_cls.PROBES[0]
113    probe_name = probe_cls.NAME
114
115  result_file: pth.LocalPath = (
116      output_dir_path / browser.unique_name / "stories" / f"{probe_name}.json")
117  return str(result_file)
118
119
120class CbbRunner(crossbench.runner.runner.Runner):
121
122  def create_run(self, browser_session: BrowserSessionRunGroup, story: Story,
123                 repetition: int, is_warmup: bool, temperature: str, index: int,
124                 name: str, timeout: dt.timedelta, throw: bool) -> Run:
125    return CbbRun(self, browser_session, story, repetition, is_warmup,
126                  temperature, index, name, timeout, throw)
127
128
129class CbbRun(Run):
130
131  def _create_session_dir(self) -> None:
132    # Don't create symlink loops and skip this step
133    pass
134
135
136def run_benchmark(output_folder: pth.LocalPathLike,
137                  browser_list: List[crossbench.browsers.browser.Browser],
138                  benchmark: PressBenchmark) -> None:
139  """Runs the benchmark using crossbench runner.
140
141  Args:
142    output_folder: Path to the directory where the output of the benchmark
143                  execution will be written.
144    browser_list: List of browsers to run the benchmark on.
145    benchmark: The Benchmark instance to run.
146  """
147  runner = CbbRunner(
148      out_dir=pth.LocalPath(output_folder),
149      browsers=browser_list,
150      benchmark=benchmark,
151      env_validation_mode=crossbench.env.ValidationMode.SKIP)
152
153  runner.run()
154