• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2023 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import annotations
6
7import argparse
8import datetime as dt
9import enum
10import inspect
11import logging
12from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional,
13                    Sequence, Set, Tuple, Type, Union)
14
15from crossbench import compat, exception, helper
16from crossbench import path as pth
17from crossbench import plt
18from crossbench.benchmarks.base import BenchmarkProbeMixin
19from crossbench.env import (HostEnvironment, HostEnvironmentConfig,
20                            ValidationMode)
21from crossbench.helper.state import BaseState, StateMachine
22from crossbench.parse import NumberParser, ObjectParser
23from crossbench.probes import all as all_probes
24from crossbench.probes.internal import ResultsSummaryProbe
25from crossbench.probes.perfetto.trace_processor.trace_processor import \
26    TraceProcessorProbe
27from crossbench.probes.probe import Probe, ProbeIncompatibleBrowser
28from crossbench.probes.thermal_monitor import ThermalStatus
29from crossbench.runner.groups.browsers import BrowsersRunGroup
30from crossbench.runner.groups.cache_temperatures import \
31    CacheTemperaturesRunGroup
32from crossbench.runner.groups.repetitions import RepetitionsRunGroup
33from crossbench.runner.groups.session import BrowserSessionRunGroup
34from crossbench.runner.groups.stories import StoriesRunGroup
35from crossbench.runner.groups.thread import RunThreadGroup
36from crossbench.runner.run import Run
37from crossbench.runner.timing import Timing
38
39if TYPE_CHECKING:
40  from crossbench.benchmarks.base import Benchmark
41  from crossbench.browsers.browser import Browser
42  from crossbench.stories.story import Story
43
44
45
46class RunnerException(exception.MultiException):
47  pass
48
49
50@enum.unique
51class ThreadMode(compat.StrEnumWithHelp):
52  NONE = ("none", (
53      "Execute all browser-sessions sequentially, default. "
54      "Low interference risk, use for worry-free time-critical measurements."))
55  PLATFORM = ("platform", (
56      "Execute browser-sessions from each platform in parallel threads. "
57      "Might cause some interference with probes that do heavy "
58      "post-processing."))
59  BROWSER = ("browser", (
60      "Execute browser-sessions from each browser in parallel thread. "
61      "High interference risk, don't use for time-critical measurements."))
62  SESSION = ("session", (
63      "Execute run from each browser-session in a parallel thread. "
64      "High interference risk, don't use for time-critical measurements."))
65
66  def group(self, runs: List[Run]) -> List[RunThreadGroup]:
67    if self == ThreadMode.NONE:
68      return [RunThreadGroup(runs)]
69    groups: Dict[Any, List[Run]] = {}
70    if self == ThreadMode.SESSION:
71      groups = helper.group_by(
72          runs, lambda run: run.browser_session, sort_key=None)
73    elif self == ThreadMode.PLATFORM:
74      groups = helper.group_by(
75          runs, lambda run: run.browser_platform, sort_key=None)
76    elif self == ThreadMode.BROWSER:
77      groups = helper.group_by(runs, lambda run: run.browser, sort_key=None)
78    else:
79      raise ValueError(f"Unexpected thread mode: {self}")
80    return [
81        RunThreadGroup(runs, index=index)
82        for index, runs in enumerate(groups.values())
83    ]
84
85
86@enum.unique
87class RunnerState(BaseState):
88  INITIAL = enum.auto()
89  SETUP = enum.auto()
90  RUNNING = enum.auto()
91  TEARDOWN = enum.auto()
92
93class Runner:
94
95  @classmethod
96  def get_out_dir(cls,
97                  cwd: pth.LocalPath,
98                  suffix: str = "",
99                  test: bool = False) -> pth.LocalPath:
100    if test:
101      return cwd / "results" / "test"
102    if suffix:
103      suffix = "_" + suffix
104    return (cwd / "results" /
105            f"{dt.datetime.now().strftime('%Y-%m-%d_%H%M%S')}{suffix}")
106
107  @classmethod
108  def add_cli_parser(
109      cls, benchmark_cls: Type[Benchmark],
110      parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
111    parser.add_argument(
112        "--repetitions",
113        "--repeat",
114        "--invocations",
115        "-r",
116        default=benchmark_cls.DEFAULT_REPETITIONS,
117        type=NumberParser.positive_int,
118        help=("Number of times each benchmark story is repeated. "
119              f"Defaults to {benchmark_cls.DEFAULT_REPETITIONS}. "
120              "Metrics are aggregated over multiple repetitions"))
121    parser.add_argument(
122        "--warmup-repetitions",
123        "--warmups",
124        default=0,
125        type=NumberParser.positive_zero_int,
126        help=("Number of times each benchmark story is repeated for warmup. "
127              "Defaults to 0. "
128              "Metrics for warmup-repetitions are discarded."))
129    parser.add_argument(
130        "--cache-temperatures",
131        default=["default"],
132        const=["cold", "warm", "hot"],
133        action="store_const",
134        help=("Repeat each run with different cache temperatures without "
135              "closing the browser in between."))
136
137    parser.add_argument(
138        "--thread-mode",
139        "--parallel",
140        default=ThreadMode.NONE,
141        type=ThreadMode,
142        help=("Change how Runs are executed.\n" +
143              ThreadMode.help_text(indent=2)))
144
145    out_dir_group = parser.add_argument_group("Output Directory Options")
146    out_dir_group.add_argument(
147        "--no-symlinks",
148        "--nosymlinks",
149        dest="create_symlinks",
150        action="store_false",
151        default=True,
152        help="Do not create symlinks in the output directory.")
153
154    out_dir_xor_group = out_dir_group.add_mutually_exclusive_group()
155    out_dir_xor_group.add_argument(
156        "--out-dir",
157        "--output-directory",
158        "-o",
159        type=pth.LocalPath,
160        help=("Results will be stored in this directory. "
161              "Defaults to results/${DATE}_${LABEL}"))
162    out_dir_xor_group.add_argument(
163        "--label",
164        "--name",
165        type=ObjectParser.non_empty_str,
166        default=benchmark_cls.NAME,
167        help=("Add a name to the default output directory. "
168              "Defaults to the benchmark name"))
169    return parser
170
171  @classmethod
172  def kwargs_from_cli(cls, args: argparse.Namespace) -> Dict[str, Any]:
173    if args.out_dir:
174      out_dir = args.out_dir
175    else:
176      label = args.label
177      assert label
178      root_dir = pth.LocalPath(__file__).parents[2]
179      out_dir = cls.get_out_dir(root_dir, label)
180    return {
181        "out_dir": out_dir,
182        "browsers": args.browser,
183        "repetitions": args.repetitions,
184        "warmup_repetitions": args.warmup_repetitions,
185        "cache_temperatures": args.cache_temperatures,
186        "thread_mode": args.thread_mode,
187        "throw": args.throw,
188        "create_symlinks": args.create_symlinks,
189        "cool_down_threshold": args.cool_down_threshold,
190    }
191
192  def __init__(self,
193               out_dir: pth.LocalPath,
194               browsers: Sequence[Browser],
195               benchmark: Benchmark,
196               additional_probes: Iterable[Probe] = (),
197               platform: plt.Platform = plt.PLATFORM,
198               env_config: Optional[HostEnvironmentConfig] = None,
199               env_validation_mode: ValidationMode = ValidationMode.THROW,
200               repetitions: int = 1,
201               warmup_repetitions: int = 0,
202               cache_temperatures: Iterable[str] = ("default",),
203               timing: Timing = Timing(),
204               cool_down_threshold: Optional[ThermalStatus] = None,
205               thread_mode: ThreadMode = ThreadMode.NONE,
206               throw: bool = False,
207               create_symlinks: bool = True):
208    self._state = StateMachine(RunnerState.INITIAL)
209    self.out_dir = out_dir.absolute()
210    assert not self.out_dir.exists(), f"out_dir={self.out_dir} exists already"
211    self.out_dir.mkdir(parents=True)
212    self._timing = timing
213    self._cool_down_threshold: Optional[ThermalStatus] = cool_down_threshold
214    self._browsers: Tuple[Browser, ...] = tuple(browsers)
215    self._validate_browser_labels()
216    self._benchmark = benchmark
217    self._stories = tuple(benchmark.stories)
218    self._repetitions = NumberParser.positive_int(repetitions, "repetitions")
219    self._warmup_repetitions = NumberParser.positive_zero_int(
220        warmup_repetitions, "warmup repetitions")
221    self._cache_temperatures: Tuple[str, ...] = tuple(cache_temperatures)
222    self._probes: List[Probe] = []
223    self._default_probes: List[Probe] = []
224    # Contains both measure and warmup runs:
225    self._all_runs: List[Run] = []
226    self._measured_runs: List[Run] = []
227    self._thread_mode = thread_mode
228    self._exceptions = exception.Annotator(throw)
229    self._platform = platform
230    self._env = HostEnvironment(self.platform, self.out_dir, self.browsers,
231                                self.probes, self.repetitions, env_config,
232                                env_validation_mode)
233    self._attach_default_probes(additional_probes)
234    self._prepare_benchmark()
235    self._cache_temperatures_groups: Tuple[CacheTemperaturesRunGroup, ...] = ()
236    self._repetitions_groups: Tuple[RepetitionsRunGroup, ...] = ()
237    self._story_groups: Tuple[StoriesRunGroup, ...] = ()
238    self._browser_group: Optional[BrowsersRunGroup] = None
239    self._create_symlinks: bool = create_symlinks
240
241  def _prepare_benchmark(self) -> None:
242    benchmark_probe_cls: Type[BenchmarkProbeMixin]
243    for benchmark_probe_cls in self._benchmark.PROBES:
244      assert inspect.isclass(benchmark_probe_cls), (
245          f"{self._benchmark}.PROBES must contain classes only, "
246          f"but got {type(benchmark_probe_cls)}")
247      assert issubclass(
248          benchmark_probe_cls,
249          Probe), (f"Expected Probe class but got {type(benchmark_probe_cls)}")
250      assert issubclass(benchmark_probe_cls, BenchmarkProbeMixin), (
251          f"{benchmark_probe_cls} should be BenchmarkProbeMixin "
252          f"for {type(self._benchmark)}.PROBES")
253      assert benchmark_probe_cls.NAME, (
254          f"Expected probe.NAME for {benchmark_probe_cls}")
255      self.attach_probe(benchmark_probe_cls(benchmark=self._benchmark))
256
257  def _validate_browser_labels(self) -> None:
258    assert self.browsers, "No browsers provided"
259    browser_unique_names = [browser.unique_name for browser in self.browsers]
260    ObjectParser.unique_sequence(browser_unique_names, "browser names")
261
262  def _attach_default_probes(self, probe_list: Iterable[Probe]) -> None:
263    assert len(self._probes) == 0
264    assert len(self._default_probes) == 0
265    for probe_cls in all_probes.INTERNAL_PROBES:
266      if probe_cls == all_probes.ThermalMonitorProbe:
267        thermal_monitor_probe = all_probes.ThermalMonitorProbe(
268            cool_down_time=self._timing.cool_down_time,
269            threshold=self._cool_down_threshold)
270        self._attach_default_probe(thermal_monitor_probe)
271      else:
272        default_probe: Probe = probe_cls()  # pytype: disable=not-instantiable
273        self._attach_default_probe(default_probe)
274
275    for index, probe in enumerate(probe_list):
276      assert (not isinstance(probe, TraceProcessorProbe) or index == 0), (
277          f"TraceProcessorProbe must be first in the list to be able "
278          f"to process other probes data. Found it at index: {index}")
279      self.attach_probe(probe)
280    # Results probe must be first in the list, and thus last to be processed
281    # so all other probes have data by the time we write the results summary.
282    assert isinstance(self._probes[0], ResultsSummaryProbe)
283
284  def _attach_default_probe(self, probe: Probe) -> None:
285    self.attach_probe(probe)
286    self._default_probes.append(probe)
287
288  def attach_probe(self,
289                   probe: Probe,
290                   matching_browser_only: bool = False) -> Probe:
291    if probe in self._probes:
292      raise ValueError(f"Cannot add the same probe twice: {probe.NAME}")
293    probe_was_used = False
294    for browser in self.browsers:
295      try:
296        probe.validate_browser(self.env, browser)
297        browser.attach_probe(probe)
298        probe_was_used = True
299      except ProbeIncompatibleBrowser as e:
300        if matching_browser_only:
301          logging.error("Skipping incompatible probe=%s for browser=%s:",
302                        probe.name, browser.unique_name)
303          logging.error("    %s", e)
304          continue
305        raise
306    if probe_was_used:
307      self._probes.append(probe)
308    return probe
309
310  @property
311  def timing(self) -> Timing:
312    return self._timing
313
314  @property
315  def cache_temperatures(self) -> Tuple[str, ...]:
316    return self._cache_temperatures
317
318  @property
319  def browsers(self) -> Tuple[Browser, ...]:
320    return self._browsers
321
322  @property
323  def stories(self) -> Tuple[Story, ...]:
324    return self._stories
325
326  @property
327  def probes(self) -> Iterable[Probe]:
328    return iter(self._probes)
329
330  @property
331  def default_probes(self) -> Iterable[Probe]:
332    return iter(self._default_probes)
333
334  @property
335  def benchmark(self) -> Benchmark:
336    return self._benchmark
337
338  @property
339  def repetitions(self) -> int:
340    return self._repetitions
341
342  @property
343  def warmup_repetitions(self) -> int:
344    return self._warmup_repetitions
345
346  @property
347  def create_symlinks(self) -> bool:
348    return self._create_symlinks
349
350  @property
351  def exceptions(self) -> exception.Annotator:
352    return self._exceptions
353
354  @property
355  def is_success(self) -> bool:
356    return len(self._measured_runs) > 0 and self._exceptions.is_success
357
358  @property
359  def platform(self) -> plt.Platform:
360    return self._platform
361
362  @property
363  def env(self) -> HostEnvironment:
364    return self._env
365
366  @property
367  def platforms(self) -> Set[plt.Platform]:
368    return set(browser.platform for browser in self.browsers)
369
370  @property
371  def all_runs(self) -> Tuple[Run, ...]:
372    return tuple(self._all_runs)
373
374  @property
375  def runs(self) -> Tuple[Run, ...]:
376    return tuple(self._measured_runs)
377
378  @property
379  def cache_temperatures_groups(self) -> Tuple[CacheTemperaturesRunGroup, ...]:
380    assert self._cache_temperatures_groups, (
381        f"No CacheTemperatureRunGroup in {self}")
382    return self._cache_temperatures_groups
383
384  @property
385  def repetitions_groups(self) -> Tuple[RepetitionsRunGroup, ...]:
386    assert self._repetitions_groups, f"No RepetitionsRunGroup in {self}"
387    return self._repetitions_groups
388
389  @property
390  def story_groups(self) -> Tuple[StoriesRunGroup, ...]:
391    assert self._story_groups, f"No StoriesRunGroup in {self}"
392    return self._story_groups
393
394  @property
395  def browser_group(self) -> BrowsersRunGroup:
396    assert self._browser_group, f"No BrowsersRunGroup in {self}"
397    return self._browser_group
398
399  @property
400  def has_browser_group(self) -> bool:
401    return self._browser_group is not None
402
403  def wait(self,
404           time: Union[int, float, dt.timedelta],
405           absolute_time: bool = False) -> None:
406    if not time:
407      return
408    if not absolute_time:
409      delta = self.timing.timedelta(time)
410    else:
411      if isinstance(time, (int, float)):
412        delta = dt.timedelta(seconds=time)
413      else:
414        delta = time
415    self._platform.sleep(delta)
416
417  def run(self, is_dry_run: bool = False) -> None:
418    self._state.expect(RunnerState.INITIAL)
419    with helper.SystemSleepPreventer():
420      with self._exceptions.annotate("Preparing"):
421        self._setup()
422      with self._exceptions.annotate("Running"):
423        self._run(is_dry_run)
424
425    if self._exceptions.throw:
426      # Ensure that we bail out on the first exception.
427      self.assert_successful_sessions_and_runs()
428    if not is_dry_run:
429      self._teardown()
430    self.assert_successful_sessions_and_runs()
431
432  def _setup(self) -> None:
433    self._state.transition(RunnerState.INITIAL, to=RunnerState.SETUP)
434    logging.info("-" * 80)
435    logging.info("SETUP")
436    logging.info("-" * 80)
437    assert self.repetitions > 0, (
438        f"Invalid repetitions count: {self.repetitions}")
439    assert self.browsers, "No browsers provided: self.browsers is empty"
440    assert self.stories, "No stories provided: self.stories is empty"
441    self._validate_browsers()
442    self._exceptions.assert_success()
443    with self._exceptions.annotate("Preparing Runs"):
444      self._all_runs = list(self.get_runs())
445      assert self._all_runs, f"{type(self)}.get_runs() produced no runs"
446      logging.info("DISCOVERED %d RUN(S)", len(self._all_runs))
447      self._measured_runs = [run for run in self._all_runs if not run.is_warmup]
448    with self._exceptions.capture("Preparing Environment"):
449      self._env.setup()
450    with self._exceptions.annotate(
451        f"Preparing Benchmark: {self._benchmark.NAME}"):
452      self._benchmark.setup(self)
453
454  def _validate_browsers(self) -> None:
455    logging.info("PREPARING %d BROWSER(S)", len(self.browsers))
456    for browser in self.browsers:
457      with self._exceptions.capture(
458          f"Preparing browser type={browser.type_name} "
459          f"unique_name={browser.unique_name}"):
460        self._validate_browser(browser)
461
462  def _validate_browser(self, browser: Browser) -> None:
463    browser.validate_binary()
464    for probe in browser.probes:
465      assert probe in self._probes, (
466          f"Browser {browser} probe {probe} not in Runner.probes. "
467          "Use Runner.attach_probe()")
468
469  def has_any_live_network(self) -> bool:
470    return any(browser.network.is_live for browser in self.browsers)
471
472  def has_all_live_network(self) -> bool:
473    return all(browser.network.is_live for browser in self.browsers)
474
475  def get_runs(self) -> Iterable[Run]:
476    index = 0
477    session_index = 0
478    throw = self._exceptions.throw
479    total_repetitions = self.repetitions + self.warmup_repetitions
480    for repetition in range(total_repetitions):
481      is_warmup: bool = repetition < self.warmup_repetitions
482      for story in self.stories:
483        for browser in self.browsers:
484          # TODO: implement browser-session start/stop
485          extra_benchmark_flags = self.benchmark.extra_flags(browser.attributes)
486          browser_session = BrowserSessionRunGroup(self.env, self.probes,
487                                                   browser,
488                                                   extra_benchmark_flags,
489                                                   session_index, self.out_dir,
490                                                   self.create_symlinks, throw)
491          session_index += 1
492          for t_index, temperature in enumerate(self.cache_temperatures):
493            name_parts = [f"story={story.name}"]
494            if total_repetitions > 1:
495              name_parts.append(f"repetition={repetition}")
496            if len(self.cache_temperatures) > 1:
497              name_parts.append(f"temperature={temperature_icon(temperature)}")
498            name_parts.append(f"index={index}")
499            yield self.create_run(
500                browser_session,
501                story,
502                repetition,
503                is_warmup,
504                f"{t_index}_{temperature}",
505                index,
506                name=", ".join(name_parts),
507                timeout=self.timing.run_timeout,
508                throw=throw)
509            index += 1
510          browser_session.set_ready()
511
512  def create_run(self, browser_session: BrowserSessionRunGroup, story: Story,
513                 repetition: int, is_warmup: bool, temperature: str, index: int,
514                 name: str, timeout: dt.timedelta, throw: bool) -> Run:
515    return Run(self, browser_session, story, repetition, is_warmup, temperature,
516               index, name, timeout, throw)
517
518  def assert_successful_sessions_and_runs(self) -> None:
519    failed_runs = list(run for run in self.runs if not run.is_success)
520    self._exceptions.assert_success(
521        f"Runs Failed: {len(failed_runs)}/{len(tuple(self.runs))} runs failed.",
522        RunnerException)
523
524  def _get_thread_groups(self) -> List[RunThreadGroup]:
525    # Also include warmup runs here.
526    return self._thread_mode.group(self._all_runs)
527
528  def _run(self, is_dry_run: bool = False) -> None:
529    self._state.transition(RunnerState.SETUP, to=RunnerState.RUNNING)
530    thread_groups: List[RunThreadGroup] = []
531    with self._exceptions.info("Creating thread groups for all Runs"):
532      thread_groups = self._get_thread_groups()
533
534    group_count = len(thread_groups)
535    if group_count == 1:
536      self._run_single_threaded(thread_groups[0])
537      return
538
539    with self._exceptions.annotate(f"Starting {group_count} thread groups."):
540      for thread_group in thread_groups:
541        thread_group.is_dry_run = is_dry_run
542        thread_group.start()
543    with self._exceptions.annotate(
544        "Waiting for all thread groups to complete."):
545      for thread_group in thread_groups:
546        thread_group.join()
547
548  def _run_single_threaded(self, thread_group: RunThreadGroup) -> None:
549    # Special case single thread groups
550    with self._exceptions.annotate("Running single thread group"):
551      thread_group.run()
552
553  def _teardown(self) -> None:
554    self._state.transition(RunnerState.RUNNING, to=RunnerState.TEARDOWN)
555    logging.info("=" * 80)
556    logging.info("RUNS COMPLETED")
557    logging.info("-" * 80)
558    logging.info("MERGING PROBE DATA")
559
560    throw = self._exceptions.throw
561
562    logging.debug("MERGING PROBE DATA: cache temperatures")
563    self._cache_temperatures_groups = CacheTemperaturesRunGroup.groups(
564        self._measured_runs, throw)
565    for cache_temp_group in self._cache_temperatures_groups:
566      cache_temp_group.merge(self.probes)
567      self._exceptions.extend(cache_temp_group.exceptions, is_nested=True)
568
569    logging.debug("MERGING PROBE DATA: repetitions")
570    self._repetitions_groups = RepetitionsRunGroup.groups(
571        self._cache_temperatures_groups, throw)
572    for repetition_group in self._repetitions_groups:
573      repetition_group.merge(self.probes)
574      self._exceptions.extend(repetition_group.exceptions, is_nested=True)
575
576    logging.debug("MERGING PROBE DATA: stories")
577    self._story_groups = StoriesRunGroup.groups(self._repetitions_groups, throw)
578    for story_group in self._story_groups:
579      story_group.merge(self.probes)
580      self._exceptions.extend(story_group.exceptions, is_nested=True)
581
582    logging.debug("MERGING PROBE DATA: browsers")
583    self._browser_group = BrowsersRunGroup(self._story_groups, throw)
584    self._browser_group.merge(self.probes)
585    self._exceptions.extend(self._browser_group.exceptions, is_nested=True)
586
587
588TEMPERATURE_ICONS = {
589    "cold": "��",
590    "warm": "⛅",
591    "hot": "��",
592}
593
594
595def temperature_icon(temperature: str) -> str:
596  if icon := TEMPERATURE_ICONS.get(temperature):
597    return icon
598  return temperature
599