• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import unittest
6
7from google.appengine.ext import ndb
8
9from dashboard import testing_common
10from dashboard import utils
11from dashboard.models import alert_group
12from dashboard.models import anomaly
13from dashboard.models import sheriff
14from dashboard.models import stoppage_alert
15
16
17class AnomalyGroupingTest(testing_common.TestCase):
18  """Test case for the behavior of updating anomaly groups."""
19
20  def _AddAnomalies(self):
21    """Adds a set of sample data used in the tests below."""
22    testing_common.AddTests(
23        ['ChromiumGPU'], ['linux-release'],
24        {'scrolling_benchmark': {'first_paint': {}}})
25    first_paint_key = utils.TestKey(
26        'ChromiumGPU/linux-release/scrolling_benchmark/first_paint')
27    first_paint_test = first_paint_key.get()
28    first_paint_test.improvement_direction = anomaly.DOWN
29    first_paint_test.put()
30
31    group_keys = [
32        alert_group.AlertGroup(
33            start_revision=3000,
34            end_revision=4000,
35            alert_kind='Anomaly',
36            test_suites=['scrolling_benchmark']).put(),
37        alert_group.AlertGroup(
38            start_revision=6000,
39            end_revision=8000,
40            alert_kind='Anomaly',
41            test_suites=['scrolling_benchmark']).put(),
42    ]
43
44    anomaly_keys = [
45        anomaly.Anomaly(
46            start_revision=2000,
47            end_revision=4000,
48            bug_id=12345,
49            test=first_paint_key).put(),
50        anomaly.Anomaly(
51            start_revision=3000,
52            end_revision=5000,
53            bug_id=12345,
54            test=first_paint_key).put(),
55        anomaly.Anomaly(
56            start_revision=6000,
57            end_revision=8000,
58            bug_id=None,
59            test=first_paint_key).put(),
60    ]
61
62    anomalies = ndb.get_multi(anomaly_keys)
63
64    # Add these anomalies to groups and put them again. When anomalies are
65    # put for the second time onward, the pre-put hook will be called and
66    # the groups of the anomalies will be updated.
67    anomalies[0].group = group_keys[0]
68    anomalies[0].put()
69    anomalies[1].group = group_keys[0]
70    anomalies[1].put()
71    anomalies[2].group = group_keys[1]
72    anomalies[2].put()
73
74    # Note that after these anomalies are added, the state of the two groups
75    # is updated. Also, the first two anomalies are in the same group.
76    self.assertEqual(anomalies[0].group, anomalies[1].group)
77    self.assertNotEqual(anomalies[0].group, anomalies[2].group)
78    return anomalies
79
80  def testUpdateAnomalyBugId_UpdatesGroupOfAnomaly(self):
81    anomalies = self._AddAnomalies()
82
83    # At first, two anomalies are in separate groups, and the second anomaly
84    # has not been assigned a bug ID.
85    self.assertNotEqual(anomalies[1].group, anomalies[2].group)
86    self.assertEqual(12345, anomalies[1].bug_id)
87    self.assertIsNone(anomalies[2].bug_id)
88
89    # Test setting bug_id. Anomaly should be moved to new group.
90    anomalies[2].bug_id = 12345
91    anomalies[2].put()
92    self.assertEqual(anomalies[1].bug_id, anomalies[2].bug_id)
93
94  def testMarkAnomalyInvalid_AnomalyIsRemovedFromGroup(self):
95    anomalies = self._AddAnomalies()
96
97    # At first, two anomalies are in the same group.
98    self.assertEqual(anomalies[0].group, anomalies[1].group)
99
100    # Mark one of the alerts as invalid.
101    self.assertEqual(12345, anomalies[1].bug_id)
102    anomalies[1].bug_id = -1
103    anomalies[1].put()
104
105    # Now, the alert marked as invalid has no group.
106    # Also, the group's revision range has been updated accordingly.
107    self.assertNotEqual(anomalies[0].group, anomalies[1].group)
108    self.assertIsNone(anomalies[1].group)
109    group = anomalies[0].group.get()
110    self.assertEqual(2000, group.start_revision)
111    self.assertEqual(4000, group.end_revision)
112
113  def testUpdateAnomalyRevisionRange_UpdatesGroupRevisionRange(self):
114    anomalies = self._AddAnomalies()
115
116    # Add another anomaly to the same group as the first two anomalies,
117    # but with a non-overlapping revision range.
118    new_anomaly = anomaly.Anomaly(
119        start_revision=3000,
120        end_revision=4000,
121        group=anomalies[0].group)
122    new_anomaly.put()
123
124    # Associate it with a group; the pre-put hook will update the group's
125    # revision range here.
126    new_anomaly.start_revision = 3010
127    new_anomaly.end_revision = 3020
128    new_anomaly.put()
129
130    # Now the group's revision range is updated.
131    group = anomalies[0].group.get()
132    self.assertEqual(3010, group.start_revision)
133    self.assertEqual(3020, group.end_revision)
134
135  def testUpdateGroup_InvalidRange_PropertiesAreUpdated(self):
136    anomalies = self._AddAnomalies()
137
138    # Add another anomaly to the same group as the first two anomalies
139    # by setting its bug ID to match that of an existing group.
140    new_anomaly = anomaly.Anomaly(
141        start_revision=3000,
142        end_revision=4000,
143        group=anomalies[0].group)
144    new_anomaly_key = new_anomaly.put()
145
146    # Change the anomaly revision to invalid range.
147    new_anomaly.start_revision = 10
148    new_anomaly.end_revision = 20
149    new_anomaly.put()
150
151    # After adding this new anomaly, it belongs to the group, and the group
152    # no longer has a minimum revision range.
153    group = anomalies[0].group.get()
154    self.assertEqual(anomalies[0].group, new_anomaly_key.get().group)
155    self.assertIsNone(group.start_revision)
156    self.assertIsNone(group.end_revision)
157
158    # Remove the new anomaly from the group by marking it invalid.
159    new_anomaly = new_anomaly_key.get()
160    new_anomaly.bug_id = -1
161    new_anomaly.put()
162
163    # Now, the anomaly group's revision range is valid again.
164    group = anomalies[0].group.get()
165    self.assertEqual(3000, group.start_revision)
166    self.assertEqual(4000, group.end_revision)
167
168
169class StoppageAlertGroupingTest(testing_common.TestCase):
170  """Test case for the behavior of updating StoppageAlert groups."""
171
172  def _AddStoppageAlerts(self):
173    testing_common.AddTests(
174        ['ChromiumGPU'], ['linux-release'],
175        {
176            'scrolling_benchmark': {
177                'dropped_foo': {},
178                'dropped_bar': {},
179            }
180        })
181    foo_path = 'ChromiumGPU/linux-release/scrolling_benchmark/dropped_foo'
182    bar_path = 'ChromiumGPU/linux-release/scrolling_benchmark/dropped_bar'
183    foo_test = utils.TestKey(foo_path).get()
184    bar_test = utils.TestKey(bar_path).get()
185    foo_row = testing_common.AddRows(foo_path, {200})[0]
186    bar_row = testing_common.AddRows(bar_path, {200})[0]
187    foo_alert_key = stoppage_alert.CreateStoppageAlert(foo_test, foo_row).put()
188    bar_alert_key = stoppage_alert.CreateStoppageAlert(bar_test, bar_row).put()
189    return [foo_alert_key.get(), bar_alert_key.get()]
190
191  def testStoppageAlertGroup_GroupAssignedUponCreation(self):
192    foo_test, bar_test = self._AddStoppageAlerts()
193    self.assertIsNotNone(foo_test.group)
194    self.assertIsNotNone(bar_test.group)
195    self.assertEqual('StoppageAlert', foo_test.group.get().alert_kind)
196
197
198class GroupAlertsTest(testing_common.TestCase):
199
200  def _CreateAnomalyForTests(
201      self, revision_range, test, sheriff_key, bug_id, is_improvement):
202    """Returns a sample anomaly with some default properties."""
203    anomaly_entity = anomaly.Anomaly(
204        start_revision=revision_range[0], end_revision=revision_range[1],
205        test=test, median_before_anomaly=100, median_after_anomaly=200,
206        sheriff=sheriff_key, bug_id=bug_id, is_improvement=is_improvement)
207    return anomaly_entity
208
209  def _AddSheriffs(self):
210    sheriff1 = sheriff.Sheriff(
211        id='Chromium Perf Sheriff', email='chrisphan@google.com').put()
212    sheriff2 = sheriff.Sheriff(
213        id='QA Perf Sheriff', email='chrisphan@google.com').put()
214    return [sheriff1, sheriff2]
215
216  def _AddTests(self):
217    test_data = {
218        'scrolling_benchmark': {
219            'first_paint': {},
220        },
221        'tab_capture': {
222            'capture': {},
223        }
224    }
225    testing_common.AddTests(['ChromiumGPU'], ['linux-release'], test_data)
226    testing_common.AddTests(['QAPerf'], ['linux-release'], test_data)
227    scrolling_test = utils.TestKey(
228        'ChromiumGPU/linux-release/scrolling_benchmark/first_paint')
229    tab_capture_test = utils.TestKey(
230        'ChromiumGPU/linux-release/tab_capture/capture')
231    for test_key in [scrolling_test, tab_capture_test]:
232      test = test_key.get()
233      test.improvement_direction = anomaly.DOWN
234      test.put()
235    return [scrolling_test, tab_capture_test]
236
237  def testGroupAlerts_WithNoAssociation_MakesNewGroup(self):
238    sheriffs = self._AddSheriffs()
239    tests = self._AddTests()
240
241    # Add some anomaly groups.
242    alert_group.AlertGroup(
243        bug_id=None,
244        start_revision=3000,
245        end_revision=6000,
246        alert_kind='Anomaly',
247        test_suites=['scrolling_benchmark']).put()
248    alert_group.AlertGroup(
249        bug_id=104,
250        start_revision=7000,
251        end_revision=9000,
252        alert_kind='Anomaly',
253        test_suites=['tab_capture']).put()
254
255    improvement_anomaly = self._CreateAnomalyForTests(
256        revision_range=(1000, 2000), test=tests[0], sheriff_key=sheriffs[0],
257        bug_id=None, is_improvement=True)
258    regression_anomaly = self._CreateAnomalyForTests(
259        revision_range=(1000, 2000), test=tests[0], sheriff_key=sheriffs[0],
260        bug_id=None, is_improvement=False)
261    test_suite = 'scrolling_benchmark'
262
263    alert_group.GroupAlerts(
264        [regression_anomaly, improvement_anomaly], test_suite, 'Anomaly')
265
266    # The regression Anomaly was not grouped with a group that has a bug ID,
267    # so the bug ID is not changed.
268    self.assertIsNone(regression_anomaly.bug_id)
269
270    # Improvement Anomaly should not be auto-triaged.
271    self.assertIsNone(improvement_anomaly.group)
272
273    alert_groups = alert_group.AlertGroup.query().fetch()
274    self.assertEqual(3, len(alert_groups))
275    self.assertEqual(
276        (1000, 2000),
277        (alert_groups[2].start_revision, alert_groups[2].end_revision))
278    self.assertIsNone(alert_groups[2].bug_id)
279    self.assertEqual(alert_groups[2].test_suites, [test_suite])
280
281  def testGroupAlerts_WithExistingGroup(self):
282    sheriffs = self._AddSheriffs()
283    tests = self._AddTests()
284
285    # Add some anomaly groups.
286    alert_group.AlertGroup(
287        bug_id=None,
288        start_revision=3000,
289        end_revision=6000,
290        alert_kind='Anomaly',
291        test_suites=['scrolling_benchmark']).put()
292    tab_capture_group = alert_group.AlertGroup(
293        bug_id=104,
294        start_revision=7000,
295        end_revision=9000,
296        alert_kind='Anomaly',
297        test_suites=['tab_capture']).put()
298
299    improvement_anomaly = self._CreateAnomalyForTests(
300        revision_range=(6000, 8000), test=tests[1], sheriff_key=sheriffs[0],
301        bug_id=None, is_improvement=True)
302    regression_anomaly = self._CreateAnomalyForTests(
303        revision_range=(6000, 8000), test=tests[1], sheriff_key=sheriffs[0],
304        bug_id=None, is_improvement=False)
305
306    alert_group.GroupAlerts(
307        [regression_anomaly, improvement_anomaly], 'tab_capture', 'Anomaly')
308
309    # The regression Anomaly's bug ID is changed because it has been grouped.
310    self.assertEqual(104, regression_anomaly.bug_id)
311    self.assertEqual(tab_capture_group, regression_anomaly.group)
312
313    # Improvement Anomaly should not be grouped.
314    self.assertIsNone(improvement_anomaly.group)
315    alert_groups = alert_group.AlertGroup.query().fetch()
316    self.assertEqual(2, len(alert_groups))
317    self.assertEqual(
318        (7000, 8000),
319        (alert_groups[1].start_revision, alert_groups[1].end_revision))
320
321  def testGroupAlerts_WithExistingGroupThatHasDifferentKind_DoesntGroup(self):
322    sheriffs = self._AddSheriffs()
323    tests = self._AddTests()
324    group_key = alert_group.AlertGroup(
325        bug_id=None,
326        start_revision=3000,
327        end_revision=6000,
328        alert_kind='OtherAlert',
329        test_suites=['tab_capture']).put()
330    my_alert = self._CreateAnomalyForTests(
331        revision_range=(4000, 5000), test=tests[1], sheriff_key=sheriffs[0],
332        bug_id=None, is_improvement=False)
333
334    alert_group.GroupAlerts([my_alert], 'tab_capture', 'Anomaly')
335    self.assertNotEqual(group_key, my_alert.group)
336    self.assertEqual('Anomaly', my_alert.group.get().alert_kind)
337
338    # If the alert kind that's passed when calling GroupAlerts matches
339    # the alert kind of the existing group, then it will be grouped.
340    alert_group.GroupAlerts([my_alert], 'tab_capture', 'OtherAlert')
341    self.assertEqual(group_key, my_alert.group)
342    self.assertEqual('OtherAlert', my_alert.group.get().alert_kind)
343
344
345if __name__ == '__main__':
346  unittest.main()
347