• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2024 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import annotations
6
7import copy
8import json
9from typing import (TYPE_CHECKING, Any, Callable, Iterable, List, Sequence,
10                    Tuple, Union)
11
12from crossbench.benchmarks.loading.loading_benchmark import PageLoadBenchmark
13from crossbench.benchmarks.loading.page.combined import CombinedPage
14from crossbench.env import HostEnvironmentConfig, ValidationMode
15from crossbench.probes.probe import Probe
16from crossbench.runner.runner import Runner
17from tests.crossbench.base import BaseCrossbenchTestCase
18
19if TYPE_CHECKING:
20  from crossbench.benchmarks.loading.page.base import Page
21
22class GenericProbeTestCase(BaseCrossbenchTestCase):
23
24  def create_runner(self,
25                    stories: Sequence[Page],
26                    js_side_effects: Union[List[Any], Callable[[Page],
27                                                               List[Any]]],
28                    separate: bool = False,
29                    repetitions: int = 3,
30                    warmup_repetitions: int = 0,
31                    cache_temperatures: Iterable[str] = ("default",),
32                    throw: bool = True) -> Runner:
33    self.assertTrue(stories)
34    if not separate and len(stories) > 1:
35      stories = [CombinedPage(stories)]
36    if isinstance(js_side_effects, list):
37      js_side_effects_fn = lambda story: js_side_effects  # pylint: disable=unnecessary-lambda-assignment
38    else:
39      js_side_effects_fn = js_side_effects
40    # The order should match Runner.get_runs
41    for _ in range(warmup_repetitions + repetitions):
42      for story in stories:
43        story_js_side_effects = js_side_effects_fn(story)
44        for browser in self.browsers:
45          for js_result in story_js_side_effects:
46            browser.expect_js(result=js_result)
47
48    for browser in self.browsers:
49      browser.expected_js = copy.deepcopy(browser.expected_js)
50
51    benchmark = PageLoadBenchmark(stories)  # pytype: disable=not-instantiable
52    self.assertTrue(len(benchmark.describe()) > 0)
53    runner = Runner(
54        self.out_dir,
55        self.browsers,
56        benchmark,
57        env_config=HostEnvironmentConfig(),
58        env_validation_mode=ValidationMode.SKIP,
59        platform=self.platform,
60        repetitions=repetitions,
61        warmup_repetitions=warmup_repetitions,
62        cache_temperatures=cache_temperatures,
63        throw=throw)
64    return runner
65
66  def get_non_empty_json_results(self, runner: Runner,
67                                probe: Probe) -> Tuple[Any, Any, Any, Any]:
68    story_json_file = runner.runs[0].results[probe].json
69    with story_json_file.open() as f:
70      story_json_data = json.load(f)
71    self.assertIsNotNone(story_json_data)
72
73    repetitions_json_file = runner.repetitions_groups[0].results[probe].json
74    with repetitions_json_file.open() as f:
75      repetitions_json_data = json.load(f)
76    self.assertIsNotNone(repetitions_json_data)
77
78    stories_json_file = runner.story_groups[0].results[probe].json
79    with stories_json_file.open() as f:
80      stories_json_data = json.load(f)
81    self.assertIsNotNone(stories_json_data)
82
83    browsers_json_file = runner.browser_group.results[probe].json
84    with browsers_json_file.open() as f:
85      browsers_json_data = json.load(f)
86    self.assertIsNotNone(browsers_json_data)
87    return (story_json_data, repetitions_json_data, stories_json_data,
88            browsers_json_data)
89
90  def get_non_empty_results_str(
91      self,
92      runner: Runner,
93      probe: Probe,
94      suffix: str,
95      has_browsers_data: bool = True) -> Tuple[str, str, str, str]:
96    story_file = runner.runs[0].results[probe].get_all(suffix)[0]
97    story_data = story_file.read_text()
98    self.assertTrue(story_data)
99
100    repetitions_file = runner.repetitions_groups[0].results[probe].get_all(
101        suffix)[0]
102    repetitions_data = repetitions_file.read_text()
103    self.assertTrue(repetitions_data)
104
105    stories_file = runner.story_groups[0].results[probe].get_all(suffix)[0]
106    stories_data = stories_file.read_text()
107    self.assertTrue(stories_data)
108
109    if has_browsers_data:
110      browsers_file = runner.browser_group.results[probe].get_all(suffix)[0]
111      browsers_data = browsers_file.read_text()
112      self.assertTrue(browsers_data)
113    else:
114      browsers_data = ""
115
116    return (story_data, repetitions_data, stories_data, browsers_data)
117