• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Unit tests for update_perf_expectations."""
6import copy
7from StringIO import StringIO
8import unittest
9import make_expectations as perf_ex_lib
10import update_perf_expectations as upe_mod
11
12
13# A separate .json file contains the list of test cases we'll use.
14# The tests used to be defined inline here, but are >80 characters in length.
15# Now they are expected to be defined in file ./sample_test_cases.json.
16# Create a dictionary of tests using .json file.
17all_tests = perf_ex_lib.ConvertJsonIntoDict(
18    perf_ex_lib.ReadFile('sample_test_cases.json'))
19# Get all keys.
20all_tests_keys = all_tests.keys()
21
22
23def VerifyPreparedTests(self, tests_to_update, reva, revb):
24  # Work with a copy of the set of tests.
25  all_tests_copy = copy.deepcopy(all_tests)
26  upe_mod.PrepareTestsForUpdate(tests_to_update, all_tests_copy, reva, revb)
27  # Make sure reva < revb
28  if reva > revb:
29    temp = reva
30    reva = revb
31    revb = temp
32  # Run through all tests and make sure only those that were
33  # specified to be modified had their 'sha1' value removed.
34  for test_key in all_tests_keys:
35    new_test_value = all_tests_copy[test_key]
36    original_test_value = all_tests[test_key]
37    if test_key in tests_to_update:
38      # Make sure there is no "sha1".
39      self.assertFalse('sha1' in new_test_value)
40      # Make sure reva and revb values are correctly set.
41      self.assertEqual(reva, new_test_value['reva'])
42      self.assertEqual(revb, new_test_value['revb'])
43    else:
44      # Make sure there is an "sha1" value
45      self.assertTrue('sha1' in new_test_value)
46      # Make sure the sha1, reva and revb values have not changed.
47      self.assertEqual(original_test_value['sha1'], new_test_value['sha1'])
48      self.assertEqual(original_test_value['reva'], new_test_value['reva'])
49      self.assertEqual(original_test_value['revb'], new_test_value['revb'])
50
51
52class UpdatePerfExpectationsTest(unittest.TestCase):
53  def testFilterMatch(self):
54    """Verifies different regular expressions test filter."""
55    self.maxDiff = None
56    # Tests to update specified by a single literal string.
57    tests_to_update = 'win-release/media_tests_av_perf/fps/tulip2.webm'
58    expected_tests_list = ['win-release/media_tests_av_perf/fps/tulip2.webm']
59    self.assertEqual(expected_tests_list,
60                     upe_mod.GetMatchingTests(tests_to_update,
61                                              all_tests_keys))
62
63    # Tests to update specified by a single reg-ex
64    tests_to_update = 'win-release/media_tests_av_perf/fps.*'
65    expected_tests_list = ['win-release/media_tests_av_perf/fps/crowd1080.webm',
66                           'win-release/media_tests_av_perf/fps/crowd2160.webm',
67                           'win-release/media_tests_av_perf/fps/crowd360.webm',
68                           'win-release/media_tests_av_perf/fps/crowd480.webm',
69                           'win-release/media_tests_av_perf/fps/crowd720.webm',
70                           'win-release/media_tests_av_perf/fps/tulip2.m4a',
71                           'win-release/media_tests_av_perf/fps/tulip2.mp3',
72                           'win-release/media_tests_av_perf/fps/tulip2.mp4',
73                           'win-release/media_tests_av_perf/fps/tulip2.ogg',
74                           'win-release/media_tests_av_perf/fps/tulip2.ogv',
75                           'win-release/media_tests_av_perf/fps/tulip2.wav',
76                           'win-release/media_tests_av_perf/fps/tulip2.webm']
77    actual_list = upe_mod.GetMatchingTests(tests_to_update,
78                                           all_tests_keys)
79    actual_list.sort()
80    self.assertEqual(expected_tests_list, actual_list)
81
82    # Tests to update are specified by a single reg-ex, spanning multiple OSes.
83    tests_to_update = '.*-release/media_tests_av_perf/fps.*'
84    expected_tests_list = ['linux-release/media_tests_av_perf/fps/tulip2.m4a',
85                           'linux-release/media_tests_av_perf/fps/tulip2.mp3',
86                           'linux-release/media_tests_av_perf/fps/tulip2.mp4',
87                           'linux-release/media_tests_av_perf/fps/tulip2.ogg',
88                           'linux-release/media_tests_av_perf/fps/tulip2.ogv',
89                           'linux-release/media_tests_av_perf/fps/tulip2.wav',
90                           'win-release/media_tests_av_perf/fps/crowd1080.webm',
91                           'win-release/media_tests_av_perf/fps/crowd2160.webm',
92                           'win-release/media_tests_av_perf/fps/crowd360.webm',
93                           'win-release/media_tests_av_perf/fps/crowd480.webm',
94                           'win-release/media_tests_av_perf/fps/crowd720.webm',
95                           'win-release/media_tests_av_perf/fps/tulip2.m4a',
96                           'win-release/media_tests_av_perf/fps/tulip2.mp3',
97                           'win-release/media_tests_av_perf/fps/tulip2.mp4',
98                           'win-release/media_tests_av_perf/fps/tulip2.ogg',
99                           'win-release/media_tests_av_perf/fps/tulip2.ogv',
100                           'win-release/media_tests_av_perf/fps/tulip2.wav',
101                           'win-release/media_tests_av_perf/fps/tulip2.webm']
102    actual_list = upe_mod.GetMatchingTests(tests_to_update,
103                                           all_tests_keys)
104    actual_list.sort()
105    self.assertEqual(expected_tests_list, actual_list)
106
107  def testLinesFromInputFile(self):
108    """Verifies different string formats specified in input file."""
109
110    # Tests to update have been specified by a single literal string in
111    # an input file.
112    # Use the StringIO class to mock a file object.
113    lines_from_file = StringIO(
114        'win-release/media_tests_av_perf/fps/tulip2.webm')
115    contents = lines_from_file.read()
116    expected_tests_list = ['win-release/media_tests_av_perf/fps/tulip2.webm']
117    actual_list = upe_mod.GetTestsToUpdate(contents, all_tests_keys)
118    actual_list.sort()
119    self.assertEqual(expected_tests_list, actual_list)
120    lines_from_file.close()
121
122    # Tests to update specified by a single reg-ex in an input file.
123    lines_from_file = StringIO('win-release/media_tests_av_perf/fps/tulip2.*\n')
124    contents = lines_from_file.read()
125    expected_tests_list = ['win-release/media_tests_av_perf/fps/tulip2.m4a',
126                           'win-release/media_tests_av_perf/fps/tulip2.mp3',
127                           'win-release/media_tests_av_perf/fps/tulip2.mp4',
128                           'win-release/media_tests_av_perf/fps/tulip2.ogg',
129                           'win-release/media_tests_av_perf/fps/tulip2.ogv',
130                           'win-release/media_tests_av_perf/fps/tulip2.wav',
131                           'win-release/media_tests_av_perf/fps/tulip2.webm']
132    actual_list = upe_mod.GetTestsToUpdate(contents, all_tests_keys)
133    actual_list.sort()
134    self.assertEqual(expected_tests_list, actual_list)
135    lines_from_file.close()
136
137    # Tests to update specified by multiple lines in an input file.
138    lines_from_file = StringIO(
139        '.*-release/media_tests_av_perf/fps/tulip2.*\n'
140        'win-release/media_tests_av_perf/dropped_fps/tulip2.*\n'
141        'linux-release/media_tests_av_perf/audio_latency/latency')
142    contents = lines_from_file.read()
143    expected_tests_list = [
144        'linux-release/media_tests_av_perf/audio_latency/latency',
145        'linux-release/media_tests_av_perf/fps/tulip2.m4a',
146        'linux-release/media_tests_av_perf/fps/tulip2.mp3',
147        'linux-release/media_tests_av_perf/fps/tulip2.mp4',
148        'linux-release/media_tests_av_perf/fps/tulip2.ogg',
149        'linux-release/media_tests_av_perf/fps/tulip2.ogv',
150        'linux-release/media_tests_av_perf/fps/tulip2.wav',
151        'win-release/media_tests_av_perf/dropped_fps/tulip2.wav',
152        'win-release/media_tests_av_perf/dropped_fps/tulip2.webm',
153        'win-release/media_tests_av_perf/fps/tulip2.m4a',
154        'win-release/media_tests_av_perf/fps/tulip2.mp3',
155        'win-release/media_tests_av_perf/fps/tulip2.mp4',
156        'win-release/media_tests_av_perf/fps/tulip2.ogg',
157        'win-release/media_tests_av_perf/fps/tulip2.ogv',
158        'win-release/media_tests_av_perf/fps/tulip2.wav',
159        'win-release/media_tests_av_perf/fps/tulip2.webm']
160    actual_list = upe_mod.GetTestsToUpdate(contents, all_tests_keys)
161    actual_list.sort()
162    self.assertEqual(expected_tests_list, actual_list)
163    lines_from_file.close()
164
165  def testPreparingForUpdate(self):
166    """Verifies that tests to be modified are changed as expected."""
167    tests_to_update = [
168        'linux-release/media_tests_av_perf/audio_latency/latency',
169        'linux-release/media_tests_av_perf/fps/tulip2.m4a',
170        'linux-release/media_tests_av_perf/fps/tulip2.mp3',
171        'linux-release/media_tests_av_perf/fps/tulip2.mp4',
172        'linux-release/media_tests_av_perf/fps/tulip2.ogg',
173        'linux-release/media_tests_av_perf/fps/tulip2.ogv',
174        'linux-release/media_tests_av_perf/fps/tulip2.wav',
175        'win-release/media_tests_av_perf/dropped_fps/tulip2.wav',
176        'win-release/media_tests_av_perf/dropped_fps/tulip2.webm',
177        'win-release/media_tests_av_perf/fps/tulip2.mp3',
178        'win-release/media_tests_av_perf/fps/tulip2.mp4',
179        'win-release/media_tests_av_perf/fps/tulip2.ogg',
180        'win-release/media_tests_av_perf/fps/tulip2.ogv',
181        'win-release/media_tests_av_perf/fps/tulip2.wav',
182        'win-release/media_tests_av_perf/fps/tulip2.webm']
183    # Test regular positive integers.
184    reva = 12345
185    revb = 54321
186    VerifyPreparedTests(self, tests_to_update, reva, revb)
187    # Test negative values.
188    reva = -54321
189    revb = 12345
190    with self.assertRaises(ValueError):
191      upe_mod.PrepareTestsForUpdate(tests_to_update, all_tests, reva, revb)
192    # Test reva greater than revb.
193    reva = 54321
194    revb = 12345
195    upe_mod.PrepareTestsForUpdate(tests_to_update, all_tests, reva, revb)
196    # Test non-integer values
197    reva = 'sds'
198    revb = 12345
199    with self.assertRaises(ValueError):
200      upe_mod.PrepareTestsForUpdate(tests_to_update, all_tests, reva, revb)
201
202
203if __name__ == '__main__':
204  unittest.main()
205