• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4import common
5import os
6import unittest
7
8import tradefed_test
9
10
11def _load_data(filename):
12    """Loads the test data of the given file name."""
13    with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
14                           'tradefed_test_unittest_data', filename), 'r') as f:
15        return f.read()
16
17
18class TradefedTestTest(unittest.TestCase):
19    """Unittest for tradefed_test."""
20
21    def test_parse_tradefed_result(self):
22        """Test for parse_tradefed_result."""
23
24        waivers = set([
25            'android.app.cts.SystemFeaturesTest#testUsbAccessory',
26            'android.widget.cts.GridViewTest#testSetNumColumns',
27        ])
28
29        # b/35605415 and b/36520623
30        # http://pantheon/storage/browser/chromeos-autotest-results/108103986-chromeos-test/
31        # CTS: Tradefed may split a module to multiple chunks.
32        # Besides, the module name may not end with "TestCases".
33        waived, _ = tradefed_test.parse_tradefed_result(
34            _load_data('CtsHostsideNetworkTests.txt'),
35            waivers=waivers)
36        self.assertEquals(0, len(waived))
37
38        # b/35530394
39        # http://pantheon/storage/browser/chromeos-autotest-results/108291418-chromeos-test/
40        # Crashed, but the automatic retry by tradefed executed the rest.
41        waived, _ = tradefed_test.parse_tradefed_result(
42            _load_data('CtsMediaTestCases.txt'),
43            waivers=waivers)
44        self.assertEquals(0, len(waived))
45
46        # b/35530394
47        # http://pantheon/storage/browser/chromeos-autotest-results/106540705-chromeos-test/
48        # Crashed in the middle, and the device didn't came back.
49        waived, _ = tradefed_test.parse_tradefed_result(
50            _load_data('CtsSecurityTestCases.txt'),
51            waivers=waivers)
52        self.assertEquals(0, len(waived))
53
54        # b/36629187
55        # http://pantheon/storage/browser/chromeos-autotest-results/108855595-chromeos-test/
56        # Crashed in the middle. Tradefed decided not to continue.
57        waived, _ = tradefed_test.parse_tradefed_result(
58            _load_data('CtsViewTestCases.txt'),
59            waivers=waivers)
60        self.assertEquals(0, len(waived))
61
62        # b/36375690
63        # http://pantheon/storage/browser/chromeos-autotest-results/109040174-chromeos-test/
64        # Mixture of real failures and waivers.
65        waived, _ = tradefed_test.parse_tradefed_result(
66            _load_data('CtsAppTestCases.txt'),
67            waivers=waivers)
68        self.assertEquals(1, len(waived))
69        # ... and the retry of the above failing iteration.
70        waived, _ = tradefed_test.parse_tradefed_result(
71            _load_data('CtsAppTestCases-retry.txt'),
72            waivers=waivers)
73        self.assertEquals(1, len(waived))
74
75        # http://pantheon/storage/browser/chromeos-autotest-results/116875512-chromeos-test/
76        # When a test case crashed during teardown, tradefed prints the "fail"
77        # message twice. Tolerate it and still return an (inconsistent) count.
78        waived, _ = tradefed_test.parse_tradefed_result(
79            _load_data('CtsWidgetTestCases.txt'),
80            waivers=waivers)
81        self.assertEquals(1, len(waived))
82
83        # http://pantheon/storage/browser/chromeos-autotest-results/117914707-chromeos-test/
84        # When a test case unrecoverably crashed during teardown, tradefed
85        # prints the "fail" and failure summary message twice. Tolerate it.
86        waived, _ = tradefed_test.parse_tradefed_result(
87            _load_data('CtsPrintTestCases.txt'),
88            waivers=waivers)
89        self.assertEquals(0, len(waived))
90
91        gts_waivers = set([
92            ('com.google.android.placement.gts.CoreGmsAppsTest#' +
93                'testCoreGmsAppsPreloaded'),
94            ('com.google.android.placement.gts.CoreGmsAppsTest#' +
95                'testGoogleDuoPreloaded'),
96            'com.google.android.placement.gts.UiPlacementTest#testPlayStore'
97        ])
98
99        # crbug.com/748116
100        # http://pantheon/storage/browser/chromeos-autotest-results/130080763-chromeos-test/
101        # 3 ABIS: x86, x86_64, and armeabi-v7a
102        waived, _ = tradefed_test.parse_tradefed_result(
103            _load_data('GtsPlacementTestCases.txt'),
104            waivers=gts_waivers)
105        self.assertEquals(9, len(waived))
106
107        # b/64095702
108        # http://pantheon/storage/browser/chromeos-autotest-results/130211812-chromeos-test/
109        # The result of the last chunk not reported by tradefed.
110        # The actual dEQP log is too big, hence the test data here is trimmed.
111        waived, _ = tradefed_test.parse_tradefed_result(
112            _load_data('CtsDeqpTestCases-trimmed.txt'),
113            waivers=waivers)
114        self.assertEquals(0, len(waived))
115
116        # b/80160772
117        # http://pantheon/storage/browser/chromeos-autotest-results/201962931-kkanchi/
118        # The newer tradefed requires different parsing to count waivers.
119        waived, _ = tradefed_test.parse_tradefed_result(
120            _load_data('CtsAppTestCases_P_simplified.txt'),
121            waivers=waivers)
122        self.assertEquals(1, len(waived))
123
124        # b/66899135, tradefed may reported inaccuratly with `list results`.
125        # Check if summary section shows that the result is inacurrate.
126        _, accurate = tradefed_test.parse_tradefed_result(
127            _load_data('CtsAppTestCases_P_simplified.txt'),
128            waivers=waivers)
129        self.assertTrue(accurate)
130
131        _, accurate = tradefed_test.parse_tradefed_result(
132            _load_data('CtsDeqpTestCases-trimmed-inaccurate.txt'),
133            waivers=waivers)
134        self.assertFalse(accurate)
135
136if __name__ == '__main__':
137    unittest.main()
138