• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2023 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import json
6import pathlib
7
8import pytest
9
10from crossbench import plt
11from crossbench.benchmarks import all as benchmarks
12from crossbench.benchmarks.base import PressBenchmark
13from crossbench.browsers.chrome import webdriver as chrome_webdriver
14from crossbench.browsers.settings import Settings
15from crossbench.cbb import cbb_adapter
16from tests import test_helper
17
18# pytest.fixtures rely on params having the same name as the fixture function
19# pylint: disable=redefined-outer-name
20
21
22def get_benchmark(benchmark_cls) -> PressBenchmark:
23  """Returns a benchmark instance for the corresponding benchmark_name"""
24  story_class = cbb_adapter.get_pressbenchmark_story_cls(benchmark_cls.NAME)
25  assert story_class
26  stories = story_class.default_story_names()[:1]
27  workload = story_class(  # pytype: disable=not-instantiable
28      substories=stories)
29  benchmark_cls_lookup = cbb_adapter.get_pressbenchmark_cls(benchmark_cls.NAME)
30  assert benchmark_cls_lookup, (
31      f"Could not find benchmark class for '{benchmark_cls.NAME}'")
32  assert benchmark_cls_lookup == benchmark_cls
33  benchmark = benchmark_cls_lookup(stories=[workload])  # pytype: disable=not-instantiable
34  return benchmark
35
36
37@pytest.fixture
38def webdriver(driver_path, browser_path):
39  return chrome_webdriver.ChromeWebDriver("Chrome", browser_path,
40                                          Settings(driver_path=driver_path))
41
42
43def run_benchmark(output_dir, webdriver, benchmark_cls) -> None:
44  """Tests that we can execute the specified benchmark and obtain result data
45  post execution.
46  This test uses Chrome browser to run the benchmarks.
47  """
48  benchmark = get_benchmark(benchmark_cls)
49  assert benchmark
50  results_dir = output_dir / "result"
51
52  maybe_results_file = cbb_adapter.get_probe_result_file(
53      benchmark_cls.NAME, webdriver, results_dir)
54  assert maybe_results_file
55  results_file = pathlib.Path(maybe_results_file)
56  assert not results_file.exists()
57
58  cbb_adapter.run_benchmark(
59      output_folder=results_dir, browser_list=[webdriver], benchmark=benchmark)
60
61  assert results_file.exists()
62  with results_file.open(encoding="utf-8") as f:
63    benchmark_data = json.load(f)
64  assert benchmark_data
65
66
67@pytest.mark.skipif(
68    plt.PLATFORM.is_linux, reason="Tests temporarily skipped on linux")
69def test_speedometer_20(output_dir, webdriver):
70  run_benchmark(output_dir, webdriver, benchmarks.Speedometer20Benchmark)
71
72
73@pytest.mark.skipif(
74    plt.PLATFORM.is_linux, reason="Tests temporarily skipped on linux")
75def test_speedometer_21(output_dir, webdriver):
76  run_benchmark(output_dir, webdriver, benchmarks.Speedometer21Benchmark)
77
78
79@pytest.mark.skipif(
80    plt.PLATFORM.is_linux, reason="Tests temporarily skipped on linux")
81def test_motionmark_12(output_dir, webdriver):
82  run_benchmark(output_dir, webdriver, benchmarks.MotionMark12Benchmark)
83
84
85@pytest.mark.skipif(
86    plt.PLATFORM.is_linux, reason="Tests temporarily skipped on linux")
87def test_motionmark_13(output_dir, webdriver):
88  run_benchmark(output_dir, webdriver, benchmarks.MotionMark13Benchmark)
89
90
91@pytest.mark.skipif(
92    plt.PLATFORM.is_linux, reason="Tests temporarily skipped on linux")
93def test_jetstream_20(output_dir, webdriver):
94  run_benchmark(output_dir, webdriver, benchmarks.JetStream20Benchmark)
95
96
97@pytest.mark.skipif(
98    plt.PLATFORM.is_linux, reason="Tests temporarily skipped on linux")
99def test_jetstream_21(output_dir, webdriver):
100  run_benchmark(output_dir, webdriver, benchmarks.JetStream21Benchmark)
101
102
103if __name__ == "__main__":
104  test_helper.run_pytest(__file__)
105