• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2024 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import annotations
6
7import abc
8import argparse
9import logging
10from typing import TYPE_CHECKING, Optional, Sequence, Tuple
11
12import numpy as np
13import pandas as pd
14from tabulate import tabulate
15
16from crossbench import config
17from crossbench import path as pth
18from crossbench.benchmarks.base import BenchmarkProbeMixin
19from crossbench.benchmarks.loading.config.pages import PagesConfig
20from crossbench.benchmarks.loading.loading_benchmark import (LoadingPageFilter,
21                                                             PageLoadBenchmark)
22from crossbench.flags.base import Flags
23from crossbench.probes.perfetto.trace_processor.trace_processor import \
24    TraceProcessorProbe
25from crossbench.probes.probe import Probe, ProbeContext
26from crossbench.probes.results import EmptyProbeResult, ProbeResult
27
28if TYPE_CHECKING:
29  from crossbench.benchmarks.loading.page.base import Page
30  from crossbench.browsers.attributes import BrowserAttributes
31  from crossbench.runner.groups.browsers import BrowsersRunGroup
32  from crossbench.runner.runner import Run
33
34CONFIG_DIR = config.config_dir()
35LOADLINE_DIR = CONFIG_DIR / "benchmark" / "loadline"
36
37# We should increase the minor version number every time there are any changes
38# that might affect the benchmark score.
39VERSION_STRING = "1.1.0"
40
41
42class LoadLinePageFilter(LoadingPageFilter):
43  """LoadLine benchmark for phone/tablet."""
44  CAN_COMBINE_STORIES: bool = False
45
46  @classmethod
47  def add_page_config_parser(cls, parser: argparse.ArgumentParser) -> None:
48    pass
49
50  @classmethod
51  def default_stories(cls) -> Tuple[Page, ...]:
52    return cls.all_stories()
53
54  @classmethod
55  def all_stories(cls) -> Tuple[Page, ...]:
56    return ()
57
58
59class LoadLineProbe(BenchmarkProbeMixin, Probe):
60  IS_GENERAL_PURPOSE = False
61  NAME = "loadline_probe"
62
63  def get_context(self, run: Run) -> Optional[LoadLineProbeContext]:
64    return LoadLineProbeContext(self, run)
65
66  def log_browsers_result(self, group: BrowsersRunGroup) -> None:
67    logging.info("-" * 80)
68    logging.critical("LoadLine Benchmark (%s)", VERSION_STRING)
69    logging.critical("LoadLine results:")
70    logging.info("- " * 40)
71    logging.critical(
72        tabulate(
73            pd.read_csv(
74                group.get_local_probe_result_path(self).with_suffix(".csv")),
75            headers="keys",
76            tablefmt="plain"))
77
78  def merge_browsers(self, group: BrowsersRunGroup) -> ProbeResult:
79    csv_file = group.get_local_probe_result_path(self).with_suffix(".csv")
80    self._compute_score(group).to_csv(csv_file)
81    return ProbeResult(csv=(csv_file,))
82
83  def _compute_score(self, group: BrowsersRunGroup) -> pd.DataFrame:
84    all_results = group.results.get_by_name(TraceProcessorProbe.NAME).csv_list
85    loadline_result: Optional[pth.LocalPath] = None
86    for result in all_results:
87      # Look for the "loadline/benchmark_score" trace processor query result.
88      if result.name == "loadline_benchmark_score.csv":
89        loadline_result = result
90        break
91    assert loadline_result is not None, "LoadLine: query result not found"
92
93    df = pd.read_csv(loadline_result)
94    df = df.groupby(["cb_browser",
95                     "cb_story"])["score"].mean().reset_index().pivot(
96                         columns=["cb_story"],
97                         index=["cb_browser"],
98                         values=["score"])
99    df = df.droplevel(0, axis=1)
100    df["TOTAL_SCORE"] = np.exp(np.log(df).mean(axis=1))
101    df.index.rename("browser", inplace=True)
102    return df.reindex(
103        columns=(["TOTAL_SCORE"] +
104                 sorted(list(c for c in df.columns if c != "TOTAL_SCORE"))))
105
106
107class LoadLineProbeContext(ProbeContext[LoadLineProbe]):
108
109  def start(self) -> None:
110    pass
111
112  def start_story_run(self) -> None:
113    self.browser.performance_mark(
114        f"LoadLine/{self.probe.benchmark.NAME}/{self.run.story.name}")
115
116  def stop(self) -> None:
117    pass
118
119  def teardown(self) -> ProbeResult:
120    return EmptyProbeResult()
121
122
123class LoadLineBenchmark(PageLoadBenchmark, metaclass=abc.ABCMeta):
124  STORY_FILTER_CLS = LoadLinePageFilter
125  PROBES = (LoadLineProbe,)
126  DEFAULT_REPETITIONS = 100
127
128  @classmethod
129  def requires_separate(cls, args: argparse.Namespace) -> bool:
130    # Perfetto metrics used in the benchmark require a separate Perfetto
131    # session for each run.
132    return True
133
134  @classmethod
135  def default_probe_config_path(cls) -> pth.LocalPath:
136    return pth.LocalPath(LOADLINE_DIR) / "probe_config.hjson"
137
138  @classmethod
139  @abc.abstractmethod
140  def default_network_config_path(cls) -> pth.LocalPath:
141    pass
142
143  @classmethod
144  @abc.abstractmethod
145  def default_pages_config_path(cls) -> pth.LocalPath:
146    pass
147
148  @classmethod
149  def get_pages_config(
150      cls, args: Optional[argparse.Namespace] = None) -> PagesConfig:
151    return PagesConfig.parse(cls.default_pages_config_path())
152
153  @classmethod
154  def all_story_names(cls) -> Sequence[str]:
155    return tuple(page.any_label for page in cls.get_pages_config().pages)
156
157
158class LoadLineTabletBenchmark(LoadLineBenchmark):
159  """LoadLine benchmark for tablet.
160  """
161  NAME = "loadline-tablet"
162
163  @classmethod
164  def default_pages_config_path(cls) -> pth.LocalPath:
165    return pth.LocalPath(LOADLINE_DIR) / "page_config_tablet.hjson"
166
167  @classmethod
168  def default_network_config_path(cls) -> pth.LocalPath:
169    return pth.LocalPath(LOADLINE_DIR) / "network_config_tablet.hjson"
170
171  @classmethod
172  def aliases(cls) -> Tuple[str, ...]:
173    return ("loading-tablet", "load-tablet", "ld-tablet")
174
175  @classmethod
176  def extra_flags(cls, browser_attributes: BrowserAttributes) -> Flags:
177    assert browser_attributes.is_chromium_based
178    return Flags(["--request-desktop-sites"])
179
180
181class LoadLinePhoneBenchmark(LoadLineBenchmark):
182  """LoadLine benchmark for phones.
183  """
184  NAME = "loadline-phone"
185
186  @classmethod
187  def default_pages_config_path(cls) -> pth.LocalPath:
188    return pth.LocalPath(LOADLINE_DIR) / "page_config_phone.hjson"
189
190  @classmethod
191  def default_network_config_path(cls) -> pth.LocalPath:
192    return pth.LocalPath(LOADLINE_DIR) / "network_config_phone.hjson"
193
194  @classmethod
195  def aliases(cls) -> Tuple[str, ...]:
196    return ("loading-phone", "load-phone", "ld-phone")
197