• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env vpython3
2# Copyright 2020 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import json
7import os
8import shutil
9import sys
10import tempfile
11from textwrap import dedent
12import unittest
13
14# The following non-std imports are fetched via vpython. See the list at
15# //.vpython3
16import mock  # pylint: disable=import-error
17from parameterized import parameterized  # pylint: disable=import-error
18
19import test_runner
20
21_TAST_TEST_RESULTS_JSON = {
22    "name": "login.Chrome",
23    "errors": None,
24    "start": "2020-01-01T15:41:30.799228462-08:00",
25    "end": "2020-01-01T15:41:53.318914698-08:00",
26    "skipReason": ""
27}
28
29
30class TestRunnerTest(unittest.TestCase):
31
32  def setUp(self):
33    self._tmp_dir = tempfile.mkdtemp()
34    self.mock_rdb = mock.patch.object(
35        test_runner.result_sink, 'TryInitClient', return_value=None)
36    self.mock_rdb.start()
37
38  def tearDown(self):
39    shutil.rmtree(self._tmp_dir, ignore_errors=True)
40    self.mock_rdb.stop()
41
42  def safeAssertItemsEqual(self, list1, list2):
43    """A Py3 safe version of assertItemsEqual.
44
45    See https://bugs.python.org/issue17866.
46    """
47    self.assertSetEqual(set(list1), set(list2))
48
49
50class TastTests(TestRunnerTest):
51
52  def get_common_tast_args(self, use_vm):
53    return [
54        'script_name',
55        'tast',
56        '--suite-name=chrome_all_tast_tests',
57        '--board=eve',
58        '--flash',
59        '--path-to-outdir=out_eve/Release',
60        '--logs-dir=%s' % self._tmp_dir,
61        '--use-vm' if use_vm else '--device=localhost:2222',
62    ]
63
64  def get_common_tast_expectations(self, use_vm, is_lacros=False):
65    expectation = [
66        test_runner.CROS_RUN_TEST_PATH,
67        '--board',
68        'eve',
69        '--cache-dir',
70        test_runner.DEFAULT_CROS_CACHE,
71        '--results-dest-dir',
72        '%s/system_logs' % self._tmp_dir,
73        '--flash',
74        '--build-dir',
75        'out_eve/Release',
76        '--results-dir',
77        self._tmp_dir,
78        '--tast-total-shards=1',
79        '--tast-shard-index=0',
80    ]
81    expectation.extend(['--start', '--copy-on-write']
82                       if use_vm else ['--device', 'localhost:2222'])
83    for p in test_runner.SYSTEM_LOG_LOCATIONS:
84      expectation.extend(['--results-src', p])
85
86    if not is_lacros:
87      expectation += [
88          '--mount',
89          '--deploy',
90          '--nostrip',
91      ]
92    return expectation
93
94  def test_tast_gtest_filter(self):
95    """Tests running tast tests with a gtest-style filter."""
96    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
97      json.dump(_TAST_TEST_RESULTS_JSON, f)
98
99    args = self.get_common_tast_args(False) + [
100        '--attr-expr=( "group:mainline" && "dep:chrome" && !informational)',
101        '--gtest_filter=login.Chrome:ui.WindowControl',
102    ]
103    with mock.patch.object(sys, 'argv', args),\
104         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
105      mock_popen.return_value.returncode = 0
106
107      test_runner.main()
108      # The gtest filter should cause the Tast expr to be replaced with a list
109      # of the tests in the filter.
110      expected_cmd = self.get_common_tast_expectations(False) + [
111          '--tast=("name:login.Chrome" || "name:ui.WindowControl")'
112      ]
113
114      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
115
116  @parameterized.expand([
117      [True],
118      [False],
119  ])
120  def test_tast_attr_expr(self, use_vm):
121    """Tests running a tast tests specified by an attribute expression."""
122    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
123      json.dump(_TAST_TEST_RESULTS_JSON, f)
124
125    args = self.get_common_tast_args(use_vm) + [
126        '--attr-expr=( "group:mainline" && "dep:chrome" && !informational)',
127    ]
128    with mock.patch.object(sys, 'argv', args),\
129         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
130      mock_popen.return_value.returncode = 0
131
132      test_runner.main()
133      expected_cmd = self.get_common_tast_expectations(use_vm) + [
134          '--tast=( "group:mainline" && "dep:chrome" && !informational)',
135      ]
136
137      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
138
139  @parameterized.expand([
140      [True],
141      [False],
142  ])
143  def test_tast_lacros(self, use_vm):
144    """Tests running a tast tests for Lacros."""
145    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
146      json.dump(_TAST_TEST_RESULTS_JSON, f)
147
148    args = self.get_common_tast_args(use_vm) + [
149        '-t=lacros.Basic',
150        '--deploy-lacros',
151    ]
152
153    with mock.patch.object(sys, 'argv', args),\
154         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
155      mock_popen.return_value.returncode = 0
156
157      test_runner.main()
158      expected_cmd = self.get_common_tast_expectations(
159          use_vm, is_lacros=True) + [
160              '--tast',
161              'lacros.Basic',
162              '--deploy-lacros',
163              '--lacros-launcher-script',
164              test_runner.LACROS_LAUNCHER_SCRIPT_PATH,
165          ]
166
167      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
168
169  @parameterized.expand([
170      [True],
171      [False],
172  ])
173  def test_tast_with_vars(self, use_vm):
174    """Tests running a tast tests with runtime variables."""
175    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
176      json.dump(_TAST_TEST_RESULTS_JSON, f)
177
178    args = self.get_common_tast_args(use_vm) + [
179        '-t=login.Chrome',
180        '--tast-var=key=value',
181    ]
182    with mock.patch.object(sys, 'argv', args),\
183         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
184      mock_popen.return_value.returncode = 0
185      test_runner.main()
186      expected_cmd = self.get_common_tast_expectations(use_vm) + [
187          '--tast', 'login.Chrome', '--tast-var', 'key=value'
188      ]
189
190      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
191
192  @parameterized.expand([
193      [True],
194      [False],
195  ])
196  def test_tast_retries(self, use_vm):
197    """Tests running a tast tests with retries."""
198    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
199      json.dump(_TAST_TEST_RESULTS_JSON, f)
200
201    args = self.get_common_tast_args(use_vm) + [
202        '-t=login.Chrome',
203        '--tast-retries=1',
204    ]
205    with mock.patch.object(sys, 'argv', args),\
206         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
207      mock_popen.return_value.returncode = 0
208      test_runner.main()
209      expected_cmd = self.get_common_tast_expectations(use_vm) + [
210          '--tast', 'login.Chrome', '--tast-retries=1'
211      ]
212
213      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
214
215  @parameterized.expand([
216      [True],
217      [False],
218  ])
219  def test_tast(self, use_vm):
220    """Tests running a tast tests."""
221    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
222      json.dump(_TAST_TEST_RESULTS_JSON, f)
223
224    args = self.get_common_tast_args(use_vm) + [
225        '-t=login.Chrome',
226    ]
227    with mock.patch.object(sys, 'argv', args),\
228         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
229      mock_popen.return_value.returncode = 0
230
231      test_runner.main()
232      expected_cmd = self.get_common_tast_expectations(use_vm) + [
233          '--tast', 'login.Chrome'
234      ]
235
236      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
237
238
239class GTestTest(TestRunnerTest):
240
241  @parameterized.expand([
242      [True, True, True],
243      [True, False, False],
244      [False, True, True],
245      [False, False, False],
246  ])
247  def test_gtest(self, use_vm, stop_ui, use_test_sudo_helper):
248    """Tests running a gtest."""
249    fd_mock = mock.mock_open()
250
251    args = [
252        'script_name',
253        'gtest',
254        '--test-exe=out_eve/Release/base_unittests',
255        '--board=eve',
256        '--path-to-outdir=out_eve/Release',
257        '--use-vm' if use_vm else '--device=localhost:2222',
258    ]
259    if stop_ui:
260      args.append('--stop-ui')
261    if use_test_sudo_helper:
262      args.append('--run-test-sudo-helper')
263
264    with mock.patch.object(sys, 'argv', args),\
265         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen,\
266         mock.patch.object(os, 'fdopen', fd_mock),\
267         mock.patch.object(os, 'remove') as mock_remove,\
268         mock.patch.object(tempfile, 'mkstemp',
269            return_value=(3, 'out_eve/Release/device_script.sh')),\
270         mock.patch.object(os, 'fchmod'):
271      mock_popen.return_value.returncode = 0
272
273      test_runner.main()
274      self.assertEqual(1, mock_popen.call_count)
275      expected_cmd = [
276          'vpython3', test_runner.CROS_RUN_TEST_PATH, '--board', 'eve',
277          '--cache-dir', test_runner.DEFAULT_CROS_CACHE, '--remote-cmd',
278          '--cwd', 'out_eve/Release', '--files',
279          'out_eve/Release/device_script.sh'
280      ]
281      if not stop_ui:
282        expected_cmd.append('--as-chronos')
283      expected_cmd.extend(['--start', '--copy-on-write']
284                          if use_vm else ['--device', 'localhost:2222'])
285      expected_cmd.extend(['--', './device_script.sh'])
286      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
287
288      expected_device_script = dedent("""\
289          #!/bin/sh
290          export HOME=/usr/local/tmp
291          export TMPDIR=/usr/local/tmp
292          """)
293
294      core_cmd = 'LD_LIBRARY_PATH=./ ./out_eve/Release/base_unittests'\
295          ' --test-launcher-shard-index=0 --test-launcher-total-shards=1'
296
297      if use_test_sudo_helper:
298        expected_device_script += dedent("""\
299            TEST_SUDO_HELPER_PATH=$(mktemp)
300            ./test_sudo_helper.py --socket-path=${TEST_SUDO_HELPER_PATH} &
301            TEST_SUDO_HELPER_PID=$!
302          """)
303        core_cmd += ' --test-sudo-helper-socket-path=${TEST_SUDO_HELPER_PATH}'
304
305      if stop_ui:
306        dbus_cmd = 'dbus-send --system --type=method_call'\
307          ' --dest=org.chromium.PowerManager'\
308          ' /org/chromium/PowerManager'\
309          ' org.chromium.PowerManager.HandleUserActivity int32:0'
310        expected_device_script += dedent("""\
311          stop ui
312          {0}
313          chown -R chronos: ../..
314          sudo -E -u chronos -- /bin/bash -c \"{1}\"
315          start ui
316          """).format(dbus_cmd, core_cmd)
317      else:
318        expected_device_script += core_cmd + '\n'
319
320      if use_test_sudo_helper:
321        expected_device_script += dedent("""\
322            pkill -P $TEST_SUDO_HELPER_PID
323            kill $TEST_SUDO_HELPER_PID
324            unlink ${TEST_SUDO_HELPER_PATH}
325          """)
326
327      self.assertEqual(1, fd_mock().write.call_count)
328      # Split the strings to make failure messages easier to read.
329      self.assertListEqual(
330          expected_device_script.split('\n'),
331          fd_mock().write.call_args[0][0].split('\n'))
332      mock_remove.assert_called_once_with('out_eve/Release/device_script.sh')
333
334  def test_gtest_with_vpython(self):
335    """Tests building a gtest with --vpython-dir."""
336    args = mock.MagicMock()
337    args.test_exe = 'base_unittests'
338    args.test_launcher_summary_output = None
339    args.trace_dir = None
340    args.runtime_deps_path = None
341    args.path_to_outdir = self._tmp_dir
342    args.vpython_dir = self._tmp_dir
343    args.logs_dir = self._tmp_dir
344
345    # With vpython_dir initially empty, the test_runner should error out
346    # due to missing vpython binaries.
347    gtest = test_runner.GTestTest(args, None)
348    with self.assertRaises(test_runner.TestFormatError):
349      gtest.build_test_command()
350
351    # Create the two expected tools, and the test should be ready to run.
352    with open(os.path.join(args.vpython_dir, 'vpython3'), 'w'):
353      pass  # Just touch the file.
354    os.mkdir(os.path.join(args.vpython_dir, 'bin'))
355    with open(os.path.join(args.vpython_dir, 'bin', 'python3'), 'w'):
356      pass
357    gtest = test_runner.GTestTest(args, None)
358    gtest.build_test_command()
359
360
361class HostCmdTests(TestRunnerTest):
362
363  @parameterized.expand([
364      [True, False, True],
365      [False, True, True],
366      [True, True, False],
367      [False, True, False],
368  ])
369  def test_host_cmd(self, is_lacros, is_ash, strip_chrome):
370    args = [
371        'script_name',
372        'host-cmd',
373        '--board=eve',
374        '--flash',
375        '--path-to-outdir=out/Release',
376        '--device=localhost:2222',
377    ]
378    if is_lacros:
379      args += ['--deploy-lacros']
380    if is_ash:
381      args += ['--deploy-chrome']
382    if strip_chrome:
383      args += ['--strip-chrome']
384    args += [
385        '--',
386        'fake_cmd',
387    ]
388    with mock.patch.object(sys, 'argv', args),\
389         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
390      mock_popen.return_value.returncode = 0
391
392      test_runner.main()
393      expected_cmd = [
394          test_runner.CROS_RUN_TEST_PATH,
395          '--board',
396          'eve',
397          '--cache-dir',
398          test_runner.DEFAULT_CROS_CACHE,
399          '--flash',
400          '--device',
401          'localhost:2222',
402          '--build-dir',
403          os.path.join(test_runner.CHROMIUM_SRC_PATH, 'out/Release'),
404          '--host-cmd',
405      ]
406      if is_lacros:
407        expected_cmd += [
408            '--deploy-lacros',
409            '--lacros-launcher-script',
410            test_runner.LACROS_LAUNCHER_SCRIPT_PATH,
411        ]
412      if is_ash:
413        expected_cmd += ['--mount', '--deploy']
414      if not strip_chrome:
415        expected_cmd += ['--nostrip']
416
417      expected_cmd += [
418          '--',
419          'fake_cmd',
420      ]
421
422      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
423
424
425if __name__ == '__main__':
426  unittest.main()
427