• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2024 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import annotations
6
7import abc
8import datetime as dt
9import itertools
10import json
11import logging
12from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
13
14from crossbench.benchmarks.base import BenchmarkProbeMixin
15from crossbench.benchmarks.motionmark.base import MotionMarkBenchmark
16from crossbench.helper import update_url_query
17from crossbench.probes.helper import Flatten
18from crossbench.probes.json import JsonResultProbe
19from crossbench.probes.metric import Metric, MetricsMerger
20from crossbench.probes.results import ProbeResult, ProbeResultDict
21from crossbench.stories.press_benchmark import PressBenchmarkStory
22
23if TYPE_CHECKING:
24  from crossbench.path import LocalPath
25  from crossbench.runner.actions import Actions
26  from crossbench.runner.groups.browsers import BrowsersRunGroup
27  from crossbench.runner.groups.stories import StoriesRunGroup
28  from crossbench.runner.run import Run
29  from crossbench.types import Json
30
31
32def _clean_up_path_segments(path: Tuple[str, ...]) -> Optional[str]:
33  name = path[-1]
34  if name.startswith("segment") or name == "data":
35    return None
36  if path[:2] == ("testsResults", "MotionMark"):
37    path = path[2:]
38  return "/".join(path)
39
40
41class MotionMark1Probe(BenchmarkProbeMixin, JsonResultProbe, abc.ABC):
42  """
43  MotionMark-specific Probe.
44  Extracts all MotionMark times and scores.
45  """
46  JS = """
47    return window.benchmarkRunnerClient.results.results;
48  """
49
50  def to_json(self, actions: Actions) -> Json:
51    return actions.js(self.JS)
52
53  def flatten_json_data(self, json_data: List) -> Json:
54    assert isinstance(json_data, list) and len(json_data) == 1, (
55        "Motion12MarkProbe requires a results list.")
56    return Flatten(json_data[0], key_fn=_clean_up_path_segments).data
57
58  def merge_stories(self, group: StoriesRunGroup) -> ProbeResult:
59    merged = MetricsMerger.merge_json_list(
60        story_group.results[self].json
61        for story_group in group.repetitions_groups)
62    return self.write_group_result(group, merged)
63
64  def merge_browsers(self, group: BrowsersRunGroup) -> ProbeResult:
65    return self.merge_browsers_json_list(group).merge(
66        self.merge_browsers_csv_list(group))
67
68  def log_run_result(self, run: Run) -> None:
69    self._log_result(run.results, single_result=True)
70
71  def log_browsers_result(self, group: BrowsersRunGroup) -> None:
72    self._log_result(group.results, single_result=False)
73
74  def _log_result(self, result_dict: ProbeResultDict,
75                  single_result: bool) -> None:
76    if self not in result_dict:
77      return
78    results_json: LocalPath = result_dict[self].json
79    logging.info("-" * 80)
80    logging.critical("Motionmark results:")
81    if not single_result:
82      logging.critical("  %s", result_dict[self].csv)
83    logging.info("- " * 40)
84
85    with results_json.open(encoding="utf-8") as f:
86      data = json.load(f)
87      if single_result:
88        score = data.get("score") or data["Score"]
89        logging.critical("Score %s", score)
90      else:
91        self._log_result_metrics(data)
92
93  def _extract_result_metrics_table(self, metrics: Dict[str, Any],
94                                    table: Dict[str, List[str]]) -> None:
95    for metric_key, metric in metrics.items():
96      if not self._valid_metric_key(metric_key):
97        continue
98      table[metric_key].append(
99          Metric.format(metric["average"], metric["stddev"]))
100    # Separate runs don't produce a score
101    if total_metric := metrics.get("score") or metrics.get("Score"):
102      table["Score"].append(
103          Metric.format(total_metric["average"], total_metric["stddev"]))
104
105  def _valid_metric_key(self, metric_key: str) -> bool:
106    parts = metric_key.split("/")
107    return len(parts) == 2 or parts[-1] == "score"
108
109
110class MotionMark1Story(PressBenchmarkStory):
111  URL_LOCAL: str = "http://localhost:8000/"
112  ALL_STORIES = {
113      "MotionMark": (
114          "Multiply",
115          "Canvas Arcs",
116          "Leaves",
117          "Paths",
118          "Canvas Lines",
119          "Images",
120          "Design",
121          "Suits",
122      ),
123      "HTML suite": (
124          "CSS bouncing circles",
125          "CSS bouncing clipped rects",
126          "CSS bouncing gradient circles",
127          "CSS bouncing blend circles",
128          "CSS bouncing filter circles",
129          # "CSS bouncing SVG images",
130          "CSS bouncing tagged images",
131          "Focus 2.0",
132          "DOM particles, SVG masks",
133          # "Composited Transforms",
134      ),
135      "Canvas suite": (
136          "canvas bouncing clipped rects",
137          "canvas bouncing gradient circles",
138          # "canvas bouncing SVG images",
139          # "canvas bouncing PNG images",
140          "Stroke shapes",
141          "Fill shapes",
142          "Canvas put/get image data",
143      ),
144      "SVG suite": (
145          "SVG bouncing circles",
146          "SVG bouncing clipped rects",
147          "SVG bouncing gradient circles",
148          # "SVG bouncing SVG images",
149          # "SVG bouncing PNG images",
150      ),
151      "Leaves suite": (
152          "Translate-only Leaves",
153          "Translate + Scale Leaves",
154          "Translate + Opacity Leaves",
155      ),
156      "Multiply suite": (
157          "Multiply: CSS opacity only",
158          "Multiply: CSS display only",
159          "Multiply: CSS visibility only",
160      ),
161      "Text suite": (
162          "Design: Latin only (12 items)",
163          "Design: CJK only (12 items)",
164          "Design: RTL and complex scripts only (12 items)",
165          "Design: Latin only (6 items)",
166          "Design: CJK only (6 items)",
167          "Design: RTL and complex scripts only (6 items)",
168      ),
169      "Suits suite": (
170          "Suits: clip only",
171          "Suits: shape only",
172          "Suits: clip, shape, rotation",
173          "Suits: clip, shape, gradient",
174          "Suits: static",
175      ),
176      "3D Graphics": (
177          "Triangles (WebGL)",
178          # "Triangles (WebGPU)",
179      ),
180      "Basic canvas path suite": (
181          "Canvas line segments, butt caps",
182          "Canvas line segments, round caps",
183          "Canvas line segments, square caps",
184          "Canvas line path, bevel join",
185          "Canvas line path, round join",
186          "Canvas line path, miter join",
187          "Canvas line path with dash pattern",
188          "Canvas quadratic segments",
189          "Canvas quadratic path",
190          "Canvas bezier segments",
191          "Canvas bezier path",
192          "Canvas arcTo segments",
193          "Canvas arc segments",
194          "Canvas rects",
195          "Canvas ellipses",
196          "Canvas line path, fill",
197          "Canvas quadratic path, fill",
198          "Canvas bezier path, fill",
199          "Canvas arcTo segments, fill",
200          "Canvas arc segments, fill",
201          "Canvas rects, fill",
202          "Canvas ellipses, fill",
203      )
204  }
205  SUBSTORIES = tuple(itertools.chain.from_iterable(ALL_STORIES.values()))
206  READY_TIMEOUT: dt.timedelta = dt.timedelta(seconds=10)
207  DEVELOPER_READY_JS: str = (
208      "return document.querySelector('tree > li') !== undefined;")
209  # The default page is ready immediately.
210  READY_JS: str = "return true;"
211
212  @classmethod
213  def default_story_names(cls) -> Tuple[str, ...]:
214    return cls.ALL_STORIES["MotionMark"]
215
216  @property
217  def substory_duration(self) -> dt.timedelta:
218    return dt.timedelta(seconds=35)
219
220  @property
221  def url_params(self) -> Dict[str, str]:
222    return {}
223
224  def prepare_test_url(self) -> str:
225    if (url_params := self.url_params) or not self.has_default_substories:
226      updated_url = update_url_query(f"{self.url}/developer.html", url_params)
227      logging.info("CUSTOM URL: %s", updated_url)
228      return updated_url
229    return self.url
230
231  def setup(self, run: Run) -> None:
232    test_url = self.prepare_test_url()
233    use_developer_url = test_url != self.url
234    with run.actions("Setup") as actions:
235      actions.show_url(test_url)
236      self._setup_wait_until_ready(actions, use_developer_url)
237      if use_developer_url:
238        self._setup_filter_stories(actions)
239
240  def _setup_wait_until_ready(self, actions: Actions,
241                              use_developer_url: bool) -> None:
242    if use_developer_url:
243      wait_js = self.DEVELOPER_READY_JS
244    else:
245      wait_js = self.READY_JS
246    actions.wait_js_condition(wait_js, 0.2, self.READY_TIMEOUT)
247
248  def _setup_filter_stories(self, actions: Actions) -> None:
249    num_enabled = actions.js(
250        """
251      let benchmarks = arguments[0];
252      const list = document.querySelectorAll(".tree li");
253      let counter = 0;
254      for (const row of list) {
255        const name = row.querySelector("label.tree-label").textContent.trim();
256        let checked = benchmarks.includes(name);
257        const labels = row.querySelectorAll("input[type=checkbox]");
258        for (const label of labels) {
259          if (checked) {
260            label.click()
261            counter++;
262          }
263        }
264      }
265      return counter
266      """,
267        arguments=[self._substories])
268    assert num_enabled > 0, "No tests were enabled"
269    actions.wait(0.1)
270
271  def run(self, run: Run) -> None:
272    with run.actions("Running") as actions:
273      actions.js("window.benchmarkController.startBenchmark()")
274      actions.wait(self.fast_duration)
275    with run.actions("Waiting for completion") as actions:
276      actions.wait_js_condition(
277          """
278          return window.benchmarkRunnerClient.results._results != undefined
279          """,
280          0.5,
281          self.slow_duration,
282          delay=self.substory_duration / 4)
283
284
285class MotionMark1Benchmark(MotionMarkBenchmark):
286  pass
287