• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4import os
5import unittest
6
7import tradefed_utils
8
9
10def _load_data(filename):
11    """Loads the test data of the given file name."""
12    with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
13                           'tradefed_utils_unittest_data', filename), 'r') as f:
14        return f.read()
15
16
17class TradefedTestTest(unittest.TestCase):
18    """Unittest for tradefed_utils."""
19
20    def test_parse_tradefed_result(self):
21        """Test for parse_tradefed_result."""
22
23        waivers = set([
24            'android.app.cts.SystemFeaturesTest#testUsbAccessory',
25            'android.widget.cts.GridViewTest#testSetNumColumns',
26        ])
27
28        # b/35605415 and b/36520623
29        # http://pantheon/storage/browser/chromeos-autotest-results/108103986-chromeos-test/
30        # CTS: Tradefed may split a module to multiple chunks.
31        # Besides, the module name may not end with "TestCases".
32        waived, _ = tradefed_utils.parse_tradefed_result(
33            _load_data('CtsHostsideNetworkTests.txt'),
34            waivers=waivers)
35        self.assertEquals(0, len(waived))
36
37        # b/35530394
38        # http://pantheon/storage/browser/chromeos-autotest-results/108291418-chromeos-test/
39        # Crashed, but the automatic retry by tradefed executed the rest.
40        waived, _ = tradefed_utils.parse_tradefed_result(
41            _load_data('CtsMediaTestCases.txt'),
42            waivers=waivers)
43        self.assertEquals(0, len(waived))
44
45        # b/35530394
46        # http://pantheon/storage/browser/chromeos-autotest-results/106540705-chromeos-test/
47        # Crashed in the middle, and the device didn't came back.
48        waived, _ = tradefed_utils.parse_tradefed_result(
49            _load_data('CtsSecurityTestCases.txt'),
50            waivers=waivers)
51        self.assertEquals(0, len(waived))
52
53        # b/36629187
54        # http://pantheon/storage/browser/chromeos-autotest-results/108855595-chromeos-test/
55        # Crashed in the middle. Tradefed decided not to continue.
56        waived, _ = tradefed_utils.parse_tradefed_result(
57            _load_data('CtsViewTestCases.txt'),
58            waivers=waivers)
59        self.assertEquals(0, len(waived))
60
61        # b/36375690
62        # http://pantheon/storage/browser/chromeos-autotest-results/109040174-chromeos-test/
63        # Mixture of real failures and waivers.
64        waived, _ = tradefed_utils.parse_tradefed_result(
65            _load_data('CtsAppTestCases.txt'),
66            waivers=waivers)
67        self.assertEquals(1, len(waived))
68        # ... and the retry of the above failing iteration.
69        waived, _ = tradefed_utils.parse_tradefed_result(
70            _load_data('CtsAppTestCases-retry.txt'),
71            waivers=waivers)
72        self.assertEquals(1, len(waived))
73
74        # http://pantheon/storage/browser/chromeos-autotest-results/116875512-chromeos-test/
75        # When a test case crashed during teardown, tradefed prints the "fail"
76        # message twice. Tolerate it and still return an (inconsistent) count.
77        waived, _ = tradefed_utils.parse_tradefed_result(
78            _load_data('CtsWidgetTestCases.txt'),
79            waivers=waivers)
80        self.assertEquals(1, len(waived))
81
82        # http://pantheon/storage/browser/chromeos-autotest-results/117914707-chromeos-test/
83        # When a test case unrecoverably crashed during teardown, tradefed
84        # prints the "fail" and failure summary message twice. Tolerate it.
85        waived, _ = tradefed_utils.parse_tradefed_result(
86            _load_data('CtsPrintTestCases.txt'),
87            waivers=waivers)
88        self.assertEquals(0, len(waived))
89
90        gts_waivers = set([
91            ('com.google.android.placement.gts.CoreGmsAppsTest#' +
92                'testCoreGmsAppsPreloaded'),
93            ('com.google.android.placement.gts.CoreGmsAppsTest#' +
94                'testGoogleDuoPreloaded'),
95            'com.google.android.placement.gts.UiPlacementTest#testPlayStore'
96        ])
97
98        # crbug.com/748116
99        # http://pantheon/storage/browser/chromeos-autotest-results/130080763-chromeos-test/
100        # 3 ABIS: x86, x86_64, and armeabi-v7a
101        waived, _ = tradefed_utils.parse_tradefed_result(
102            _load_data('GtsPlacementTestCases.txt'),
103            waivers=gts_waivers)
104        self.assertEquals(9, len(waived))
105
106        # b/64095702
107        # http://pantheon/storage/browser/chromeos-autotest-results/130211812-chromeos-test/
108        # The result of the last chunk not reported by tradefed.
109        # The actual dEQP log is too big, hence the test data here is trimmed.
110        waived, _ = tradefed_utils.parse_tradefed_result(
111            _load_data('CtsDeqpTestCases-trimmed.txt'),
112            waivers=waivers)
113        self.assertEquals(0, len(waived))
114
115        # b/80160772
116        # http://pantheon/storage/browser/chromeos-autotest-results/201962931-kkanchi/
117        # The newer tradefed requires different parsing to count waivers.
118        waived, _ = tradefed_utils.parse_tradefed_result(
119            _load_data('CtsAppTestCases_P_simplified.txt'),
120            waivers=waivers)
121        self.assertEquals(1, len(waived))
122
123        # b/66899135, tradefed may reported inaccuratly with `list results`.
124        # Check if summary section shows that the result is inacurrate.
125        _, accurate = tradefed_utils.parse_tradefed_result(
126            _load_data('CtsAppTestCases_P_simplified.txt'),
127            waivers=waivers)
128        self.assertTrue(accurate)
129
130        _, accurate = tradefed_utils.parse_tradefed_result(
131            _load_data('CtsDeqpTestCases-trimmed-inaccurate.txt'),
132            waivers=waivers)
133        self.assertFalse(accurate)
134
135    def test_get_test_result_xml_path(self):
136        path = tradefed_utils.get_test_result_xml_path(os.path.join(
137            os.path.dirname(os.path.realpath(__file__)),
138            'tradefed_utils_unittest_data', 'results'))
139        self.assertEqual(path, os.path.join(
140            os.path.dirname(os.path.realpath(__file__)),
141            'tradefed_utils_unittest_data', 'results', '2019.11.07_10.14.55',
142            'test_result.xml'))
143
144    def test_get_perf_metrics_from_test_result_xml(self):
145        perf_result = tradefed_utils.get_perf_metrics_from_test_result_xml(
146            os.path.join(os.path.dirname(os.path.realpath(__file__)),
147                         'tradefed_utils_unittest_data', 'test_result.xml'),
148            os.path.join('/', 'resultsdir'))
149        expected_result = [
150            {'units': 'ms',
151             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
152             'description': 'android.media.cts.AudioRecordTest'
153                            '#testAudioRecordLocalMono16Bit',
154             'value': '7.1688596491228065', 'higher_is_better': False},
155            {'units': 'ms',
156             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
157             'description': 'android.media.cts.AudioRecordTest'
158                            '#testAudioRecordLocalMono16BitShort',
159             'value': '2.5416666666666665', 'higher_is_better': False},
160            {'units': 'ms',
161             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
162             'description': 'android.media.cts.AudioRecordTest'
163                            '#testAudioRecordLocalNonblockingStereoFloat',
164             'value': '1.75', 'higher_is_better': False},
165            {'units': 'ms',
166             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
167             'description': 'android.media.cts.AudioRecordTest'
168                            '#testAudioRecordMonoFloat',
169             'value': '12.958881578947368', 'higher_is_better': False},
170            {'units': 'ms',
171             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
172             'description': 'android.media.cts.AudioRecordTest'
173                            '#testAudioRecordResamplerMono8Bit',
174             'value': '0.0', 'higher_is_better': False},
175            {'units': 'ms',
176             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
177             'description': 'android.media.cts.AudioRecordTest'
178                            '#testAudioRecordResamplerStereo8Bit',
179             'value': '3.5', 'higher_is_better': False},
180            {'units': 'ms',
181             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
182             'description': 'android.media.cts.AudioRecordTest'
183                            '#testAudioRecordStereo16Bit',
184             'value': '3.5', 'higher_is_better': False},
185            {'units': 'ms',
186             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
187             'description': 'android.media.cts.AudioTrackTest'
188                            '#testFastTimestamp',
189             'value': '0.1547618955373764', 'higher_is_better': False},
190            {'units': 'ms',
191             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
192             'description': 'android.media.cts.AudioTrackTest'
193                            '#testGetTimestamp',
194             'value': '0.1490119844675064', 'higher_is_better': False},
195            {'units': 'ms',
196             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
197             'description': 'android.media.cts.AudioTrack_ListenerTest'
198                            '#testAudioTrackCallback',
199             'value': '9.347127739984884', 'higher_is_better': False},
200            {'units': 'ms',
201             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
202             'description': 'android.media.cts.AudioTrack_ListenerTest'
203                            '#testAudioTrackCallbackWithHandler',
204             'value': '7.776177955844914', 'higher_is_better': False},
205            {'units': 'ms',
206             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
207             'description': 'android.media.cts.AudioTrack_ListenerTest'
208                            '#testStaticAudioTrackCallback',
209             'value': '7.776177955844914', 'higher_is_better': False},
210            {'units': 'ms',
211             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
212             'description': 'android.media.cts.AudioTrack_ListenerTest'
213                            '#testStaticAudioTrackCallbackWithHandler',
214             'value': '9.514361300075587', 'higher_is_better': False},
215            {'units': 'count',
216             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
217             'description': 'android.media.cts.DecoderTest'
218                            '#testH264ColorAspects',
219             'value': '1.0', 'higher_is_better': True},
220            {'units': 'count',
221             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
222             'description': 'android.media.cts.DecoderTest'
223                            '#testH265ColorAspects',
224             'value': '1.0', 'higher_is_better': True},
225            {'units': 'fps',
226             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
227             'description': 'android.media.cts.VideoDecoderPerfTest'
228                            '#testAvcGoog0Perf0320x0240',
229             'value': '580.1607045151507', 'higher_is_better': True},
230            {'units': 'fps',
231             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
232             'description': 'android.media.cts.VideoDecoderPerfTest'
233                            '#testAvcGoog0Perf0720x0480',
234             'value': '244.18184010611358', 'higher_is_better': True},
235            {'units': 'fps',
236             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
237             'description': 'android.media.cts.VideoDecoderPerfTest'
238                            '#testAvcGoog0Perf1280x0720',
239             'value': '70.96290491279275', 'higher_is_better': True},
240            {'units': 'fps',
241             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
242             'description': 'android.media.cts.VideoDecoderPerfTest'
243                            '#testAvcGoog0Perf1920x1080',
244             'value': '31.299613935451564', 'higher_is_better': True},
245            {'units': 'fps',
246             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
247             'description': 'android.media.cts.VideoDecoderPerfTest'
248                            '#testAvcOther0Perf0320x0240',
249             'value': '1079.6843075197307', 'higher_is_better': True},
250            {'units': 'fps',
251             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
252             'description': 'android.media.cts.VideoDecoderPerfTest'
253                            '#testAvcOther0Perf0720x0480',
254             'value': '873.7785366761784', 'higher_is_better': True},
255            {'units': 'fps',
256             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
257             'description': 'android.media.cts.VideoDecoderPerfTest'
258                            '#testAvcOther0Perf1280x0720',
259             'value': '664.6463289568261', 'higher_is_better': True},
260            {'units': 'fps',
261             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
262             'description': 'android.media.cts.VideoDecoderPerfTest'
263                            '#testAvcOther0Perf1920x1080',
264             'value': '382.10811352923474', 'higher_is_better': True},
265            {'units': 'fps',
266             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
267             'description': 'android.media.cts.VideoDecoderPerfTest'
268                            '#testH263Goog0Perf0176x0144',
269             'value': '1511.3027429644353', 'higher_is_better': True},
270            {'units': 'fps',
271             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
272             'description': 'android.media.cts.VideoDecoderPerfTest'
273                            '#testHevcGoog0Perf0352x0288',
274             'value': '768.8737453173384', 'higher_is_better': True},
275            {'units': 'fps',
276             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
277             'description': 'android.media.cts.VideoDecoderPerfTest'
278                            '#testHevcGoog0Perf0640x0360',
279             'value': '353.7226028743237', 'higher_is_better': True},
280            {'units': 'fps',
281             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
282             'description': 'android.media.cts.VideoDecoderPerfTest'
283                            '#testHevcGoog0Perf0720x0480',
284             'value': '319.3122874170939', 'higher_is_better': True},
285            {'units': 'fps',
286             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
287             'description': 'android.media.cts.VideoDecoderPerfTest'
288                            '#testHevcGoog0Perf1280x0720',
289             'value': '120.89218432028369', 'higher_is_better': True},
290            {'units': 'fps',
291             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
292             'description': 'android.media.cts.VideoDecoderPerfTest'
293                            '#testMpeg4Goog0Perf0176x0144',
294             'value': '1851.890822618321', 'higher_is_better': True},
295            {'units': 'fps',
296             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
297             'description': 'android.media.cts.VideoDecoderPerfTest'
298                            '#testVp8Goog0Perf0320x0180',
299             'value': '1087.946513466716', 'higher_is_better': True},
300            {'units': 'fps',
301             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
302             'description': 'android.media.cts.VideoDecoderPerfTest'
303                            '#testVp8Goog0Perf0640x0360',
304             'value': '410.18461316281423', 'higher_is_better': True},
305            {'units': 'fps',
306             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
307             'description': 'android.media.cts.VideoDecoderPerfTest'
308                            '#testVp8Goog0Perf1920x1080',
309             'value': '36.26433070651982', 'higher_is_better': True},
310            {'units': 'fps',
311             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
312             'description': 'android.media.cts.VideoDecoderPerfTest'
313                            '#testVp8Other0Perf0320x0180',
314             'value': '1066.7819511702078', 'higher_is_better': True},
315            {'units': 'fps',
316             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
317             'description': 'android.media.cts.VideoDecoderPerfTest'
318                            '#testVp8Other0Perf0640x0360',
319             'value': '930.261434505189', 'higher_is_better': True},
320            {'units': 'fps',
321             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
322             'description': 'android.media.cts.VideoDecoderPerfTest'
323                            '#testVp8Other0Perf1280x0720',
324             'value': '720.4170603577236', 'higher_is_better': True},
325            {'units': 'fps',
326             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
327             'description': 'android.media.cts.VideoDecoderPerfTest'
328                            '#testVp8Other0Perf1920x1080',
329             'value': '377.55742437554915', 'higher_is_better': True},
330            {'units': 'fps',
331             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
332             'description': 'android.media.cts.VideoDecoderPerfTest'
333                            '#testVp9Goog0Perf0320x0180',
334             'value': '988.6158776121617', 'higher_is_better': True},
335            {'units': 'fps',
336             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
337             'description': 'android.media.cts.VideoDecoderPerfTest'
338                            '#testVp9Goog0Perf0640x0360',
339             'value': '409.8162085338674', 'higher_is_better': True},
340            {'units': 'fps',
341             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
342             'description': 'android.media.cts.VideoDecoderPerfTest'
343                            '#testVp9Goog0Perf1280x0720',
344             'value': '147.75847359424512', 'higher_is_better': True},
345            {'units': 'fps',
346             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
347             'description': 'android.media.cts.VideoDecoderPerfTest'
348                            '#testVp9Goog0Perf1920x1080',
349             'value': '83.95677136649255', 'higher_is_better': True}
350        ]
351        self.assertListEqual(list(perf_result), expected_result)
352
353        perf_result = tradefed_utils.get_perf_metrics_from_test_result_xml(
354            os.path.join(os.path.dirname(os.path.realpath(__file__)),
355                         'tradefed_utils_unittest_data',
356                         'malformed_test_result.xml'),
357            os.path.join('/', 'resultsdir'))
358        self.assertListEqual(list(perf_result), [])
359
360
361if __name__ == '__main__':
362    unittest.main()
363