• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2022 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import abc
6from typing import Sequence, Type
7
8from crossbench.benchmarks import base as benchmark
9from tests.crossbench.base import BaseCrossbenchTestCase
10
11
12class BaseBenchmarkTestCase(BaseCrossbenchTestCase, metaclass=abc.ABCMeta):
13
14  @property
15  @abc.abstractmethod
16  def benchmark_cls(self):
17    pass
18
19  @property
20  def story_cls(self):
21    return self.benchmark_cls.DEFAULT_STORY_CLS
22
23  def setUp(self):
24    super().setUp()
25    self.assertTrue(
26        issubclass(self.benchmark_cls, benchmark.Benchmark),
27        f"Expected Benchmark subclass, but got: BENCHMARK={self.benchmark_cls}")
28
29  def test_describe(self):
30    self.assertIsInstance(self.benchmark_cls.describe(), dict)
31
32
33class SubStoryTestCase(BaseBenchmarkTestCase, metaclass=abc.ABCMeta):
34
35  @property
36  def story_filter_cls(self) -> Type[benchmark.StoryFilter]:
37    return self.benchmark_cls.STORY_FILTER_CLS
38
39  def story_filter(self, patterns: Sequence[str],
40                   **kwargs) -> benchmark.StoryFilter:
41    return self.story_filter_cls(  # pytype: disable=not-instantiable
42        story_cls=self.story_cls,
43        patterns=patterns,
44        **kwargs)
45
46  def test_instantiate_no_stories(self):
47    with self.assertRaises(AssertionError):
48      self.benchmark_cls(stories=[])
49    with self.assertRaises(AssertionError):
50      self.benchmark_cls(stories="")
51    with self.assertRaises(AssertionError):
52      self.benchmark_cls(stories=["", ""])
53
54  def test_stories_creation(self):
55    for name in self.story_cls.all_story_names():
56      stories = self.story_filter([name]).stories
57      self.assertTrue(len(stories) == 1)
58      story = stories[0]
59      self.assertIsInstance(story, self.story_cls)
60      self.assertIsInstance(story.details_json(), dict)
61      self.assertTrue(len(str(story)) > 0)
62
63  def test_instantiate_single_story(self):
64    any_story_name = self.story_cls.all_story_names()[0]
65    any_story = self.story_filter([any_story_name]).stories[0]
66    # Instantiate with single story,
67    with self.assertRaises(Exception):
68      self.benchmark_cls(any_story)
69    # with single story array
70    self.benchmark_cls([any_story])
71    with self.assertRaises(AssertionError):
72      # Accidentally nested array.
73      self.benchmark_cls([[any_story]])
74
75  def test_instantiate_all_stories(self):
76    stories = self.story_filter(self.story_cls.all_story_names()).stories
77    self.benchmark_cls(stories)
78
79
80class PressBaseBenchmarkTestCase(SubStoryTestCase, metaclass=abc.ABCMeta):
81
82  def test_invalid_story_names(self):
83    # Only StoryFilter can filter stories by regexp
84    with self.assertRaises(Exception):
85      self.story_cls.from_names(".*", separate=True)
86    with self.assertRaises(Exception):
87      self.story_cls.from_names([".*"], separate=True)
88    with self.assertRaises(Exception):
89      self.story_cls.from_names([".*", "name does not exist"], separate=True)
90    with self.assertRaises(Exception):
91      self.story_cls.from_names([""], separate=True)
92
93  def test_all(self):
94    all_stories = [story.name for story in self.story_cls.all(separate=True)]
95    all_regexp = [
96        story.name for story in self.story_filter([".*"], separate=True).stories
97    ]
98    all_string = [
99        story.name
100        for story in self.story_filter(["all"], separate=True).stories
101    ]
102    self.assertListEqual(all_stories, all_regexp)
103    self.assertListEqual(all_stories, all_string)
104
105  def test_default(self):
106    default_stories = [
107        story.name for story in self.story_cls.default(separate=True)
108    ]
109    default_string = [
110        story.name
111        for story in self.story_filter(["default"], separate=True).stories
112    ]
113    self.assertListEqual(default_stories, default_string)
114
115  def test_remove(self):
116    assert len(self.story_cls.all_story_names()) > 1
117    story_name = self.story_cls.all_story_names()[0]
118    all_stories = [story.name for story in self.story_cls.all(separate=True)]
119    filtered_stories = [
120        story.name for story in self.story_filter([".*", f"-{story_name}"],
121                                                  separate=True).stories
122    ]
123    self.assertEqual(len(filtered_stories) + 1, len(all_stories))
124    for name in filtered_stories:
125      self.assertIn(name, all_stories)
126
127  def test_remove_invalid(self):
128    assert len(self.story_cls.all_story_names()) > 1
129    story_name = self.story_cls.all_story_names()[0]
130    with self.assertRaises(ValueError):
131      self.story_filter(["-"])
132    with self.assertRaises(ValueError):
133      self.story_filter(["--"])
134    with self.assertRaises(ValueError):
135      self.story_filter(["-.*"])
136    with self.assertRaises(ValueError):
137      self.story_filter(["-all"])
138    with self.assertRaises(ValueError):
139      self.story_filter(["-does not exist name"])
140    with self.assertRaises(ValueError):
141      self.story_filter([f"-{story_name}"])
142
143  def test_invalid_remove_all(self):
144    assert len(self.story_cls.all_story_names()) > 1
145    story_name = self.story_cls.all_story_names()[0]
146    with self.assertRaises(ValueError):
147      self.story_filter([story_name, f"-{story_name}"])
148    with self.assertRaises(ValueError):
149      self.story_filter([story_name, "-[^ ]+"])
150
151  def test_invalid_add_all(self):
152    assert len(self.story_cls.all_story_names()) > 1
153    story_name = self.story_cls.all_story_names()[0]
154    with self.assertRaises(ValueError):
155      # Add all stories again after filtering out some
156      self.story_filter([".*", f"-{story_name}", ".*|[^ ]+"])
157
158  def test_remove_non_existent(self):
159    assert len(self.story_cls.all_story_names()) > 1
160    story_name = self.story_cls.all_story_names()[0]
161    other_story_name = self.story_cls.all_story_names()[1]
162    with self.assertRaises(ValueError):
163      self.story_filter([other_story_name, f"-{story_name}"])
164