• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2022 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import abc
6import copy
7import csv
8from typing import Optional, Type
9from unittest import mock
10
11from crossbench.benchmarks.jetstream.jetstream_2 import (JetStream2Benchmark,
12                                                         JetStream2Probe,
13                                                         JetStream2Story)
14from crossbench.env import (HostEnvironment, HostEnvironmentConfig,
15                            ValidationMode)
16from crossbench.runner.runner import Runner
17from tests.crossbench.benchmarks import helper
18
19
20class JetStream2BaseTestCase(
21    helper.PressBaseBenchmarkTestCase, metaclass=abc.ABCMeta):
22
23  @property
24  @abc.abstractmethod
25  def benchmark_cls(self) -> Type[JetStream2Benchmark]:
26    pass
27
28  @property
29  @abc.abstractmethod
30  def story_cls(self) -> Type[JetStream2Story]:
31    pass
32
33  @property
34  @abc.abstractmethod
35  def probe_cls(self) -> Type[JetStream2Probe]:
36    pass
37
38  def test_run_throw(self):
39    self._test_run(throw=True)
40
41  def test_run_default(self):
42    self._test_run()
43    for browser in self.browsers:
44      urls = self.filter_splashscreen_urls(browser.url_list)
45      self.assertIn(self.story_cls.URL, urls)
46      self.assertNotIn(self.story_cls.URL_LOCAL, urls)
47
48  def test_run_custom_url(self):
49    custom_url = "http://test.example.com/jetstream"
50    self._test_run(custom_url)
51    for browser in self.browsers:
52      urls = self.filter_splashscreen_urls(browser.url_list)
53      self.assertIn(custom_url, urls)
54      self.assertNotIn(self.story_cls.URL, urls)
55      self.assertNotIn(self.story_cls.URL_LOCAL, urls)
56
57  def _test_run(self, custom_url: Optional[str] = None, throw: bool = False):
58    repetitions = 3
59    stories = self.story_cls.from_names(["WSL"], url=custom_url)
60    example_story_data = {
61        "firstIteration": 1,
62        "average": 0.1,
63        "worst4": 1.1,
64        "score": 1
65    }
66    # The order should match Runner.get_runs
67    for _ in range(repetitions):
68      for _ in stories:
69        jetstream_probe_results = {
70            story.name: example_story_data for story in stories
71        }
72        for browser in self.browsers:
73          # Page is ready
74          browser.expect_js(result=True)
75          # filter benchmarks
76          browser.expect_js()
77          # UI is updated and ready,
78          browser.expect_js(result=True)
79          # Start running benchmark
80          browser.expect_js()
81          # Wait until done
82          browser.expect_js(result=True)
83          browser.expect_js(result=jetstream_probe_results)
84    for browser in self.browsers:
85      browser.expected_js = copy.deepcopy(browser.expected_js)
86
87    benchmark = self.benchmark_cls(stories, custom_url=custom_url)  # pytype: disable=not-instantiable
88    self.assertTrue(len(benchmark.describe()) > 0)
89    runner = Runner(
90        self.out_dir,
91        self.browsers,
92        benchmark,
93        env_config=HostEnvironmentConfig(),
94        env_validation_mode=ValidationMode.SKIP,
95        platform=self.platform,
96        repetitions=repetitions,
97        throw=throw)
98    with mock.patch.object(
99        HostEnvironment, "validate_url", return_value=True) as cm:
100      runner.run()
101    cm.assert_called_once()
102    for browser in self.browsers:
103      urls = self.filter_splashscreen_urls(browser.url_list)
104      self.assertEqual(len(urls), repetitions)
105      self.assertTrue(browser.was_js_invoked(self.probe_cls.JS))
106
107    csv_file = self.out_dir / f"{self.probe_cls.NAME}.csv"
108    with csv_file.open(encoding="utf-8") as f:
109      csv_data = list(csv.DictReader(f, delimiter="\t"))
110    self.assertListEqual(
111        list(csv_data[0].keys()), ["label", "", "dev", "stable"])
112    self.assertDictEqual(
113        csv_data[1],
114        {
115            "label": "version",
116            "dev": "102.22.33.44",
117            "stable": "100.22.33.44",
118            # One padding element
119            "": ""
120        })
121
122    with self.assertLogs(level="INFO") as cm:
123      for probe in runner.probes:
124        for run in runner.runs:
125          probe.log_run_result(run)
126    output = "\n".join(cm.output)
127    self.assertIn("JetStream results", output)
128
129    with self.assertLogs(level="INFO") as cm:
130      for probe in runner.probes:
131        probe.log_browsers_result(runner.browser_group)
132    output = "\n".join(cm.output)
133    self.assertIn("JetStream results", output)
134    self.assertIn("102.22.33.44", output)
135    self.assertIn("100.22.33.44", output)
136
137
138# TODO: introduce JetStreamBaseTestCase
139class JetStream3BaseTestCase(JetStream2BaseTestCase, metaclass=abc.ABCMeta):
140  pass
141