• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2022 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import abc
6import copy
7import csv
8from typing import Optional, Type
9from unittest import mock
10
11from crossbench.benchmarks.motionmark.motionmark_1 import (MotionMark1Benchmark,
12                                                           MotionMark1Probe,
13                                                           MotionMark1Story)
14from crossbench.benchmarks.motionmark.motionmark_1_2 import (
15    MotionMark12Benchmark, MotionMark12Probe, MotionMark12Story)
16from crossbench.benchmarks.motionmark.motionmark_1_3 import (
17    MotionMark13Benchmark, MotionMark13Probe, MotionMark13Story)
18from crossbench.env import (HostEnvironment, HostEnvironmentConfig,
19                            ValidationMode)
20from crossbench.runner.runner import Runner
21from tests import test_helper
22from tests.crossbench.benchmarks import helper
23
24
25class MotionMark1BaseTestCase(
26    helper.PressBaseBenchmarkTestCase, metaclass=abc.ABCMeta):
27
28  @property
29  @abc.abstractmethod
30  def benchmark_cls(self) -> Type[MotionMark1Benchmark]:
31    pass
32
33  @property
34  @abc.abstractmethod
35  def story_cls(self) -> Type[MotionMark1Story]:
36    pass
37
38  @property
39  @abc.abstractmethod
40  def probe_cls(self) -> Type[MotionMark1Probe]:
41    pass
42
43
44  EXAMPLE_PROBE_DATA = [{
45      "testsResults": {
46          "MotionMark": {
47              "Multiply": {
48                  "complexity": {
49                      "complexity":
50                          1169.7666313745012,
51                      "stdev":
52                          2.6693101402239985,
53                      "bootstrap": {
54                          "confidenceLow": 1154.0859381321234,
55                          "confidenceHigh": 1210.464520355893,
56                          "median": 1180.8987652049277,
57                          "mean": 1163.0061487765158,
58                          "confidencePercentage": 0.8
59                      },
60                      "segment1": [[1, 16.666666666666668],
61                                   [1, 16.666666666666668]],
62                      "segment2": [[1, 6.728874992470971],
63                                   [3105, 13.858528114770454]]
64                  },
65                  "controller": {
66                      "score": 1168.106104032434,
67                      "average": 1168.106104032434,
68                      "stdev": 37.027504395081785,
69                      "percent": 3.1698750881669624
70                  },
71                  "score": 1180.8987652049277,
72                  "scoreLowerBound": 1154.0859381321234,
73                  "scoreUpperBound": 1210.464520355893
74              }
75          }
76      },
77      "score": 1180.8987652049277,
78      "scoreLowerBound": 1154.0859381321234,
79      "scoreUpperBound": 1210.464520355893
80  }]
81
82  def test_all_stories(self):
83    stories = self.story_filter(["all"], separate=True).stories
84    self.assertGreater(len(stories), 1)
85    for story in stories:
86      self.assertIsInstance(story, self.story_cls)
87    names = set(story.name for story in stories)
88    self.assertEqual(len(names), len(stories))
89    self.assertEqual(len(names), len(self.story_cls.SUBSTORIES))
90
91  def test_default_stories(self):
92    stories = self.story_filter(["default"], separate=True).stories
93    self.assertGreater(len(stories), 1)
94    for story in stories:
95      self.assertIsInstance(story, self.story_cls)
96    names = set(story.name for story in stories)
97    self.assertEqual(len(names), len(stories))
98    self.assertEqual(len(names), len(self.story_cls.ALL_STORIES["MotionMark"]))
99
100  def test_run_throw(self):
101    self._test_run(throw=True)
102
103  def test_run_default(self):
104    self._test_run()
105    for browser in self.browsers:
106      urls = self.filter_splashscreen_urls(browser.url_list)
107      self.assertIn(f"{self.story_cls.URL}/developer.html", urls)
108      self.assertNotIn(self.story_cls.URL_LOCAL, urls)
109      self.assertNotIn(f"{self.story_cls.URL_LOCAL}/developer.html", urls)
110
111  def test_run_custom_url(self):
112    custom_url = "http://test.example.com/motionmark"
113    self._test_run(custom_url)
114    for browser in self.browsers:
115      urls = self.filter_splashscreen_urls(browser.url_list)
116      self.assertIn(f"{custom_url}/developer.html", urls)
117      self.assertNotIn(self.story_cls.URL, urls)
118      self.assertNotIn(self.story_cls.URL_LOCAL, urls)
119
120  def _test_run(self, custom_url: Optional[str] = None, throw: bool = False):
121    stories = self.story_cls.from_names(["Multiply"], url=custom_url)
122    repetitions = 3
123    # The order should match Runner.get_runs
124    for _ in range(repetitions):
125      for _ in stories:
126        for browser in self.browsers:
127          # Page is ready
128          browser.expect_js(result=True)
129          # NOF enabled benchmarks
130          browser.expect_js(result=1)
131          # Start running benchmark
132          browser.expect_js()
133          # Wait until done
134          browser.expect_js(result=True)
135          browser.expect_js(result=self.EXAMPLE_PROBE_DATA)
136    for browser in self.browsers:
137      browser.expected_js = copy.deepcopy(browser.expected_js)
138    benchmark = self.benchmark_cls(stories, custom_url=custom_url)
139    self.assertTrue(len(benchmark.describe()) > 0)
140    runner = Runner(
141        self.out_dir,
142        self.browsers,
143        benchmark,
144        env_config=HostEnvironmentConfig(),
145        env_validation_mode=ValidationMode.SKIP,
146        platform=self.platform,
147        repetitions=repetitions,
148        throw=throw)
149    with mock.patch.object(
150        HostEnvironment, "validate_url", return_value=True) as cm:
151      runner.run()
152    cm.assert_called_once()
153    assert runner.is_success
154    for browser in self.browsers:
155      urls = self.filter_splashscreen_urls(browser.url_list)
156      self.assertEqual(len(urls), repetitions)
157      self.assertTrue(browser.was_js_invoked(self.probe_cls.JS))
158    with (self.out_dir /
159          f"{self.probe_cls.NAME}.csv").open(encoding="utf-8") as f:
160      csv_data = list(csv.DictReader(f, delimiter="\t"))
161    self.assertListEqual(
162        list(csv_data[0].keys()), ["label", "", "dev", "stable"])
163    self.assertDictEqual(
164        csv_data[1],
165        {
166            "label": "version",
167            "dev": "102.22.33.44",
168            "stable": "100.22.33.44",
169            # One padding element (after "label"):
170            "": "",
171        })
172
173
174class MotionMark12TestCase(MotionMark1BaseTestCase):
175
176  @property
177  def benchmark_cls(self):
178    return MotionMark12Benchmark
179
180  @property
181  def story_cls(self):
182    return MotionMark12Story
183
184  @property
185  def probe_cls(self):
186    return MotionMark12Probe
187
188
189class MotionMark13TestCase(MotionMark1BaseTestCase):
190
191  @property
192  def benchmark_cls(self):
193    return MotionMark13Benchmark
194
195  @property
196  def story_cls(self):
197    return MotionMark13Story
198
199  @property
200  def probe_cls(self):
201    return MotionMark13Probe
202
203
204del MotionMark1BaseTestCase
205
206if __name__ == "__main__":
207  test_helper.run_pytest(__file__)
208