• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env vpython3
2# Copyright 2020 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import json
7import os
8import shutil
9import sys
10import tempfile
11import unittest
12import six
13
14# The following non-std imports are fetched via vpython. See the list at
15# //.vpython
16import mock  # pylint: disable=import-error
17from parameterized import parameterized  # pylint: disable=import-error
18
19import test_runner
20
21_TAST_TEST_RESULTS_JSON = {
22    "name": "login.Chrome",
23    "errors": None,
24    "start": "2020-01-01T15:41:30.799228462-08:00",
25    "end": "2020-01-01T15:41:53.318914698-08:00",
26    "skipReason": ""
27}
28
29
30class TestRunnerTest(unittest.TestCase):
31
32  def setUp(self):
33    self._tmp_dir = tempfile.mkdtemp()
34    self.mock_rdb = mock.patch.object(
35        test_runner.result_sink, 'TryInitClient', return_value=None)
36    self.mock_rdb.start()
37
38  def tearDown(self):
39    shutil.rmtree(self._tmp_dir, ignore_errors=True)
40    self.mock_rdb.stop()
41
42  def safeAssertItemsEqual(self, list1, list2):
43    """A Py3 safe version of assertItemsEqual.
44
45    See https://bugs.python.org/issue17866.
46    """
47    if six.PY3:
48      self.assertSetEqual(set(list1), set(list2))
49    else:
50      self.assertCountEqual(list1, list2)
51
52
53class TastTests(TestRunnerTest):
54
55  def get_common_tast_args(self, use_vm):
56    return [
57        'script_name',
58        'tast',
59        '--suite-name=chrome_all_tast_tests',
60        '--board=eve',
61        '--flash',
62        '--path-to-outdir=out_eve/Release',
63        '--logs-dir=%s' % self._tmp_dir,
64        '--use-vm' if use_vm else '--device=localhost:2222',
65    ]
66
67  def get_common_tast_expectations(self, use_vm, is_lacros=False):
68    expectation = [
69        test_runner.CROS_RUN_TEST_PATH,
70        '--board',
71        'eve',
72        '--cache-dir',
73        test_runner.DEFAULT_CROS_CACHE,
74        '--results-dest-dir',
75        '%s/system_logs' % self._tmp_dir,
76        '--flash',
77        '--build-dir',
78        'out_eve/Release',
79        '--results-dir',
80        self._tmp_dir,
81        '--tast-total-shards=1',
82        '--tast-shard-index=0',
83    ]
84    expectation.extend(['--start', '--copy-on-write']
85                       if use_vm else ['--device', 'localhost:2222'])
86    for p in test_runner.SYSTEM_LOG_LOCATIONS:
87      expectation.extend(['--results-src', p])
88
89    if not is_lacros:
90      expectation += [
91          '--mount',
92          '--deploy',
93          '--nostrip',
94      ]
95    return expectation
96
97  def test_tast_gtest_filter(self):
98    """Tests running tast tests with a gtest-style filter."""
99    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
100      json.dump(_TAST_TEST_RESULTS_JSON, f)
101
102    args = self.get_common_tast_args(False) + [
103        '--attr-expr=( "group:mainline" && "dep:chrome" && !informational)',
104        '--gtest_filter=login.Chrome:ui.WindowControl',
105    ]
106    with mock.patch.object(sys, 'argv', args),\
107         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
108      mock_popen.return_value.returncode = 0
109
110      test_runner.main()
111      # The gtest filter should cause the Tast expr to be replaced with a list
112      # of the tests in the filter.
113      expected_cmd = self.get_common_tast_expectations(False) + [
114          '--tast=("name:login.Chrome" || "name:ui.WindowControl")'
115      ]
116
117      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
118
119  @parameterized.expand([
120      [True],
121      [False],
122  ])
123  def test_tast_attr_expr(self, use_vm):
124    """Tests running a tast tests specified by an attribute expression."""
125    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
126      json.dump(_TAST_TEST_RESULTS_JSON, f)
127
128    args = self.get_common_tast_args(use_vm) + [
129        '--attr-expr=( "group:mainline" && "dep:chrome" && !informational)',
130    ]
131    with mock.patch.object(sys, 'argv', args),\
132         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
133      mock_popen.return_value.returncode = 0
134
135      test_runner.main()
136      expected_cmd = self.get_common_tast_expectations(use_vm) + [
137          '--tast=( "group:mainline" && "dep:chrome" && !informational)',
138      ]
139
140      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
141
142  @parameterized.expand([
143      [True],
144      [False],
145  ])
146  def test_tast_lacros(self, use_vm):
147    """Tests running a tast tests for Lacros."""
148    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
149      json.dump(_TAST_TEST_RESULTS_JSON, f)
150
151    args = self.get_common_tast_args(use_vm) + [
152        '-t=lacros.Basic',
153        '--deploy-lacros',
154    ]
155
156    with mock.patch.object(sys, 'argv', args),\
157         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
158      mock_popen.return_value.returncode = 0
159
160      test_runner.main()
161      expected_cmd = self.get_common_tast_expectations(
162          use_vm, is_lacros=True) + [
163              '--tast',
164              'lacros.Basic',
165              '--deploy-lacros',
166              '--lacros-launcher-script',
167              test_runner.LACROS_LAUNCHER_SCRIPT_PATH,
168          ]
169
170      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
171
172  @parameterized.expand([
173      [True],
174      [False],
175  ])
176  def test_tast_with_vars(self, use_vm):
177    """Tests running a tast tests with runtime variables."""
178    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
179      json.dump(_TAST_TEST_RESULTS_JSON, f)
180
181    args = self.get_common_tast_args(use_vm) + [
182        '-t=login.Chrome',
183        '--tast-var=key=value',
184    ]
185    with mock.patch.object(sys, 'argv', args),\
186         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
187      mock_popen.return_value.returncode = 0
188      test_runner.main()
189      expected_cmd = self.get_common_tast_expectations(use_vm) + [
190          '--tast', 'login.Chrome', '--tast-var', 'key=value'
191      ]
192
193      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
194
195  @parameterized.expand([
196      [True],
197      [False],
198  ])
199  def test_tast_retries(self, use_vm):
200    """Tests running a tast tests with retries."""
201    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
202      json.dump(_TAST_TEST_RESULTS_JSON, f)
203
204    args = self.get_common_tast_args(use_vm) + [
205        '-t=login.Chrome',
206        '--tast-retries=1',
207    ]
208    with mock.patch.object(sys, 'argv', args),\
209         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
210      mock_popen.return_value.returncode = 0
211      test_runner.main()
212      expected_cmd = self.get_common_tast_expectations(use_vm) + [
213          '--tast', 'login.Chrome', '--tast-retries=1'
214      ]
215
216      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
217
218  @parameterized.expand([
219      [True],
220      [False],
221  ])
222  def test_tast(self, use_vm):
223    """Tests running a tast tests."""
224    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
225      json.dump(_TAST_TEST_RESULTS_JSON, f)
226
227    args = self.get_common_tast_args(use_vm) + [
228        '-t=login.Chrome',
229    ]
230    with mock.patch.object(sys, 'argv', args),\
231         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
232      mock_popen.return_value.returncode = 0
233
234      test_runner.main()
235      expected_cmd = self.get_common_tast_expectations(use_vm) + [
236          '--tast', 'login.Chrome'
237      ]
238
239      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
240
241
242class GTestTest(TestRunnerTest):
243
244  @parameterized.expand([
245      [True],
246      [False],
247  ])
248  def test_gtest(self, use_vm):
249    """Tests running a gtest."""
250    fd_mock = mock.mock_open()
251
252    args = [
253        'script_name',
254        'gtest',
255        '--test-exe=out_eve/Release/base_unittests',
256        '--board=eve',
257        '--path-to-outdir=out_eve/Release',
258        '--use-vm' if use_vm else '--device=localhost:2222',
259    ]
260    with mock.patch.object(sys, 'argv', args),\
261         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen,\
262         mock.patch.object(os, 'fdopen', fd_mock),\
263         mock.patch.object(os, 'remove') as mock_remove,\
264         mock.patch.object(tempfile, 'mkstemp',
265            return_value=(3, 'out_eve/Release/device_script.sh')),\
266         mock.patch.object(os, 'fchmod'):
267      mock_popen.return_value.returncode = 0
268
269      test_runner.main()
270      self.assertEqual(1, mock_popen.call_count)
271      expected_cmd = [
272          test_runner.CROS_RUN_TEST_PATH, '--board', 'eve', '--cache-dir',
273          test_runner.DEFAULT_CROS_CACHE, '--as-chronos', '--remote-cmd',
274          '--cwd', 'out_eve/Release', '--files',
275          'out_eve/Release/device_script.sh'
276      ]
277      expected_cmd.extend(['--start', '--copy-on-write']
278                          if use_vm else ['--device', 'localhost:2222'])
279      expected_cmd.extend(['--', './device_script.sh'])
280      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
281
282      fd_mock().write.assert_called_once_with(
283          '#!/bin/sh\nexport HOME=/usr/local/tmp\n'
284          'export TMPDIR=/usr/local/tmp\n'
285          'LD_LIBRARY_PATH=./ ./out_eve/Release/base_unittests '
286          '--test-launcher-shard-index=0 --test-launcher-total-shards=1\n')
287      mock_remove.assert_called_once_with('out_eve/Release/device_script.sh')
288
289  def test_gtest_with_vpython(self):
290    """Tests building a gtest with --vpython-dir."""
291    args = mock.MagicMock()
292    args.test_exe = 'base_unittests'
293    args.test_launcher_summary_output = None
294    args.trace_dir = None
295    args.runtime_deps_path = None
296    args.path_to_outdir = self._tmp_dir
297    args.vpython_dir = self._tmp_dir
298    args.logs_dir = self._tmp_dir
299
300    # With vpython_dir initially empty, the test_runner should error out
301    # due to missing vpython binaries.
302    gtest = test_runner.GTestTest(args, None)
303    with self.assertRaises(test_runner.TestFormatError):
304      gtest.build_test_command()
305
306    # Create the two expected tools, and the test should be ready to run.
307    with open(os.path.join(args.vpython_dir, 'vpython3'), 'w'):
308      pass  # Just touch the file.
309    os.mkdir(os.path.join(args.vpython_dir, 'bin'))
310    with open(os.path.join(args.vpython_dir, 'bin', 'python3'), 'w'):
311      pass
312    gtest = test_runner.GTestTest(args, None)
313    gtest.build_test_command()
314
315
316class HostCmdTests(TestRunnerTest):
317
318  @parameterized.expand([
319      [True, False, True],
320      [False, True, True],
321      [True, True, False],
322      [False, True, False],
323  ])
324  def test_host_cmd(self, is_lacros, is_ash, strip_chrome):
325    args = [
326        'script_name',
327        'host-cmd',
328        '--board=eve',
329        '--flash',
330        '--path-to-outdir=out/Release',
331        '--device=localhost:2222',
332    ]
333    if is_lacros:
334      args += ['--deploy-lacros']
335    if is_ash:
336      args += ['--deploy-chrome']
337    if strip_chrome:
338      args += ['--strip-chrome']
339    args += [
340        '--',
341        'fake_cmd',
342    ]
343    with mock.patch.object(sys, 'argv', args),\
344         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
345      mock_popen.return_value.returncode = 0
346
347      test_runner.main()
348      expected_cmd = [
349          test_runner.CROS_RUN_TEST_PATH,
350          '--board',
351          'eve',
352          '--cache-dir',
353          test_runner.DEFAULT_CROS_CACHE,
354          '--flash',
355          '--device',
356          'localhost:2222',
357          '--build-dir',
358          os.path.join(test_runner.CHROMIUM_SRC_PATH, 'out/Release'),
359          '--host-cmd',
360      ]
361      if is_lacros:
362        expected_cmd += [
363            '--deploy-lacros',
364            '--lacros-launcher-script',
365            test_runner.LACROS_LAUNCHER_SCRIPT_PATH,
366        ]
367      if is_ash:
368        expected_cmd += ['--mount', '--deploy']
369      if not strip_chrome:
370        expected_cmd += ['--nostrip']
371
372      expected_cmd += [
373          '--',
374          'fake_cmd',
375      ]
376
377      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
378
379
380if __name__ == '__main__':
381  unittest.main()
382