• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env vpython3
2# Copyright 2020 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import json
7import os
8import shutil
9import sys
10import tempfile
11from textwrap import dedent
12import unittest
13
14# The following non-std imports are fetched via vpython. See the list at
15# //.vpython3
16import mock  # pylint: disable=import-error
17from parameterized import parameterized  # pylint: disable=import-error
18
19import test_runner
20
21_TAST_TEST_RESULTS_JSON = {
22    "name": "login.Chrome",
23    "errors": None,
24    "start": "2020-01-01T15:41:30.799228462-08:00",
25    "end": "2020-01-01T15:41:53.318914698-08:00",
26    "skipReason": ""
27}
28
29
30class TestRunnerTest(unittest.TestCase):
31
32  def setUp(self):
33    self._tmp_dir = tempfile.mkdtemp()
34    self.mock_rdb = mock.patch.object(
35        test_runner.result_sink, 'TryInitClient', return_value=None)
36    self.mock_rdb.start()
37    self.mock_env = mock.patch.dict(
38        os.environ, {'SWARMING_BOT_ID': 'cros-chrome-chromeos8-row29'})
39    self.mock_env.start()
40
41  def tearDown(self):
42    shutil.rmtree(self._tmp_dir, ignore_errors=True)
43    self.mock_rdb.stop()
44    self.mock_env.stop()
45
46  def safeAssertItemsEqual(self, list1, list2):
47    """A Py3 safe version of assertItemsEqual.
48
49    See https://bugs.python.org/issue17866.
50    """
51    self.assertSetEqual(set(list1), set(list2))
52
53
54class TastTests(TestRunnerTest):
55
56  def get_common_tast_args(self, use_vm, fetch_cros_hostname):
57    return [
58        'script_name',
59        'tast',
60        '--suite-name=chrome_all_tast_tests',
61        '--board=eve',
62        '--flash',
63        '--path-to-outdir=out_eve/Release',
64        '--logs-dir=%s' % self._tmp_dir,
65        '--use-vm' if use_vm else
66        ('--fetch-cros-hostname'
67         if fetch_cros_hostname else '--device=localhost:2222'),
68    ]
69
70  def get_common_tast_expectations(self,
71                                   use_vm,
72                                   fetch_cros_hostname,
73                                   is_lacros=False):
74    expectation = [
75        test_runner.CROS_RUN_TEST_PATH,
76        '--board',
77        'eve',
78        '--cache-dir',
79        test_runner.DEFAULT_CROS_CACHE,
80        '--results-dest-dir',
81        '%s/system_logs' % self._tmp_dir,
82        '--flash',
83        '--build-dir',
84        'out_eve/Release',
85        '--results-dir',
86        self._tmp_dir,
87        '--tast-total-shards=1',
88        '--tast-shard-index=0',
89    ]
90    expectation.extend(['--start', '--copy-on-write'] if use_vm else (
91        ['--device', 'chrome-chromeos8-row29']
92        if fetch_cros_hostname else ['--device', 'localhost:2222']))
93    for p in test_runner.SYSTEM_LOG_LOCATIONS:
94      expectation.extend(['--results-src', p])
95
96    if not is_lacros:
97      expectation += [
98          '--mount',
99          '--deploy',
100          '--nostrip',
101      ]
102    return expectation
103
104  def test_tast_gtest_filter(self):
105    """Tests running tast tests with a gtest-style filter."""
106    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
107      json.dump(_TAST_TEST_RESULTS_JSON, f)
108
109    args = self.get_common_tast_args(False, False) + [
110        '--attr-expr=( "group:mainline" && "dep:chrome" && !informational)',
111        '--gtest_filter=login.Chrome:ui.WindowControl',
112    ]
113    with mock.patch.object(sys, 'argv', args),\
114         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
115      mock_popen.return_value.returncode = 0
116
117      test_runner.main()
118      # The gtest filter should cause the Tast expr to be replaced with a list
119      # of the tests in the filter.
120      expected_cmd = self.get_common_tast_expectations(False, False) + [
121          '--tast=("name:login.Chrome" || "name:ui.WindowControl")'
122      ]
123
124      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
125
126  @parameterized.expand([
127      [True, False],
128      [False, True],
129      [False, False],
130  ])
131  def test_tast_attr_expr(self, use_vm, fetch_cros_hostname):
132    """Tests running a tast tests specified by an attribute expression."""
133    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
134      json.dump(_TAST_TEST_RESULTS_JSON, f)
135
136    args = self.get_common_tast_args(use_vm, fetch_cros_hostname) + [
137        '--attr-expr=( "group:mainline" && "dep:chrome" && !informational)',
138    ]
139    with mock.patch.object(sys, 'argv', args),\
140         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
141      mock_popen.return_value.returncode = 0
142
143      test_runner.main()
144      expected_cmd = self.get_common_tast_expectations(
145          use_vm, fetch_cros_hostname) + [
146              '--tast=( "group:mainline" && "dep:chrome" && !informational)',
147          ]
148
149      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
150
151  @parameterized.expand([
152      [True, False],
153      [False, True],
154      [False, False],
155  ])
156  def test_tast_lacros(self, use_vm, fetch_cros_hostname):
157    """Tests running a tast tests for Lacros."""
158    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
159      json.dump(_TAST_TEST_RESULTS_JSON, f)
160
161    args = self.get_common_tast_args(use_vm, fetch_cros_hostname) + [
162        '-t=lacros.Basic',
163        '--deploy-lacros',
164    ]
165
166    with mock.patch.object(sys, 'argv', args),\
167         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
168      mock_popen.return_value.returncode = 0
169
170      test_runner.main()
171      expected_cmd = self.get_common_tast_expectations(
172          use_vm, fetch_cros_hostname, is_lacros=True) + [
173              '--tast',
174              'lacros.Basic',
175              '--deploy-lacros',
176              '--lacros-launcher-script',
177              test_runner.LACROS_LAUNCHER_SCRIPT_PATH,
178          ]
179
180      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
181
182  @parameterized.expand([
183      [True, False],
184      [False, True],
185      [False, False],
186  ])
187  def test_tast_with_vars(self, use_vm, fetch_cros_hostname):
188    """Tests running a tast tests with runtime variables."""
189    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
190      json.dump(_TAST_TEST_RESULTS_JSON, f)
191
192    args = self.get_common_tast_args(use_vm, fetch_cros_hostname) + [
193        '-t=login.Chrome',
194        '--tast-var=key=value',
195    ]
196    with mock.patch.object(sys, 'argv', args),\
197         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
198      mock_popen.return_value.returncode = 0
199      test_runner.main()
200      expected_cmd = self.get_common_tast_expectations(
201          use_vm, fetch_cros_hostname) + [
202              '--tast', 'login.Chrome', '--tast-var', 'key=value'
203          ]
204
205      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
206
207  @parameterized.expand([
208      [True, False],
209      [False, True],
210      [False, False],
211  ])
212  def test_tast_retries(self, use_vm, fetch_cros_hostname):
213    """Tests running a tast tests with retries."""
214    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
215      json.dump(_TAST_TEST_RESULTS_JSON, f)
216
217    args = self.get_common_tast_args(use_vm, fetch_cros_hostname) + [
218        '-t=login.Chrome',
219        '--tast-retries=1',
220    ]
221    with mock.patch.object(sys, 'argv', args),\
222         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
223      mock_popen.return_value.returncode = 0
224      test_runner.main()
225      expected_cmd = self.get_common_tast_expectations(
226          use_vm,
227          fetch_cros_hostname) + ['--tast', 'login.Chrome', '--tast-retries=1']
228
229      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
230
231  @parameterized.expand([
232      [True, False],
233      [False, True],
234      [False, False],
235  ])
236  def test_tast(self, use_vm, fetch_cros_hostname):
237    """Tests running a tast tests."""
238    with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f:
239      json.dump(_TAST_TEST_RESULTS_JSON, f)
240
241    args = self.get_common_tast_args(use_vm, fetch_cros_hostname) + [
242        '-t=login.Chrome',
243    ]
244    with mock.patch.object(sys, 'argv', args),\
245         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
246      mock_popen.return_value.returncode = 0
247
248      test_runner.main()
249      expected_cmd = self.get_common_tast_expectations(
250          use_vm, fetch_cros_hostname) + ['--tast', 'login.Chrome']
251
252      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
253
254
255class GTestTest(TestRunnerTest):
256
257  @parameterized.expand([
258      [True, True, True, False, True],
259      [True, False, False, False, False],
260      [False, True, True, True, True],
261      [False, False, False, True, False],
262      [False, True, True, False, True],
263      [False, False, False, False, False],
264  ])
265  def test_gtest(self, use_vm, stop_ui, use_test_sudo_helper,
266                 fetch_cros_hostname, use_deployed_dbus_configs):
267    """Tests running a gtest."""
268    fd_mock = mock.mock_open()
269
270    args = [
271        'script_name',
272        'gtest',
273        '--test-exe=out_eve/Release/base_unittests',
274        '--board=eve',
275        '--path-to-outdir=out_eve/Release',
276        '--use-vm' if use_vm else
277        ('--fetch-cros-hostname'
278         if fetch_cros_hostname else '--device=localhost:2222'),
279    ]
280    if stop_ui:
281      args.append('--stop-ui')
282    if use_test_sudo_helper:
283      args.append('--run-test-sudo-helper')
284    if use_deployed_dbus_configs:
285      args.append('--use-deployed-dbus-configs')
286
287    with mock.patch.object(sys, 'argv', args),\
288         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen,\
289         mock.patch.object(os, 'fdopen', fd_mock),\
290         mock.patch.object(os, 'remove') as mock_remove,\
291         mock.patch.object(tempfile, 'mkstemp',
292            side_effect=[(3, 'out_eve/Release/device_script.sh'),\
293                         (4, 'out_eve/Release/runtime_files.txt')]),\
294         mock.patch.object(os, 'fchmod'):
295      mock_popen.return_value.returncode = 0
296
297      test_runner.main()
298      self.assertEqual(1, mock_popen.call_count)
299      expected_cmd = [
300          'vpython3', test_runner.CROS_RUN_TEST_PATH, '--board', 'eve',
301          '--cache-dir', test_runner.DEFAULT_CROS_CACHE, '--remote-cmd',
302          '--cwd', 'out_eve/Release', '--files-from',
303          'out_eve/Release/runtime_files.txt'
304      ]
305      if not stop_ui:
306        expected_cmd.append('--as-chronos')
307      expected_cmd.extend(['--start', '--copy-on-write'] if use_vm else (
308          ['--device', 'chrome-chromeos8-row29']
309          if fetch_cros_hostname else ['--device', 'localhost:2222']))
310      expected_cmd.extend(['--', './device_script.sh'])
311      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
312
313      expected_device_script = dedent("""\
314          #!/bin/sh
315          export HOME=/usr/local/tmp
316          export TMPDIR=/usr/local/tmp
317          """)
318
319      core_cmd = 'LD_LIBRARY_PATH=./ ./out_eve/Release/base_unittests'\
320          ' --test-launcher-shard-index=0 --test-launcher-total-shards=1'
321
322      if use_test_sudo_helper:
323        expected_device_script += dedent("""\
324            TEST_SUDO_HELPER_PATH=$(mktemp)
325            ./test_sudo_helper.py --socket-path=${TEST_SUDO_HELPER_PATH} &
326            TEST_SUDO_HELPER_PID=$!
327          """)
328        core_cmd += ' --test-sudo-helper-socket-path=${TEST_SUDO_HELPER_PATH}'
329
330      if use_deployed_dbus_configs:
331        expected_device_script += dedent("""\
332            mount --bind ./dbus /opt/google/chrome/dbus
333            kill -s HUP $(pgrep dbus)
334          """)
335
336      if stop_ui:
337        dbus_cmd = 'dbus-send --system --type=method_call'\
338          ' --dest=org.chromium.PowerManager'\
339          ' /org/chromium/PowerManager'\
340          ' org.chromium.PowerManager.HandleUserActivity int32:0'
341        expected_device_script += dedent("""\
342          stop ui
343          {0}
344          chown -R chronos: ../..
345          sudo -E -u chronos -- /bin/bash -c \"{1}\"
346          TEST_RETURN_CODE=$?
347          start ui
348          """).format(dbus_cmd, core_cmd)
349      else:
350        expected_device_script += dedent("""\
351          {0}
352          TEST_RETURN_CODE=$?
353          """).format(core_cmd)
354
355      if use_test_sudo_helper:
356        expected_device_script += dedent("""\
357            pkill -P $TEST_SUDO_HELPER_PID
358            kill $TEST_SUDO_HELPER_PID
359            unlink ${TEST_SUDO_HELPER_PATH}
360          """)
361
362      if use_deployed_dbus_configs:
363        expected_device_script += dedent("""\
364            umount /opt/google/chrome/dbus
365            kill -s HUP $(pgrep dbus)
366          """)
367
368      expected_device_script += dedent("""\
369          exit $TEST_RETURN_CODE
370        """)
371
372      self.assertEqual(2, fd_mock().write.call_count)
373      write_calls = fd_mock().write.call_args_list
374
375      # Split the strings to make failure messages easier to read.
376      # Verify the first write of device script.
377      self.assertListEqual(
378          expected_device_script.split('\n'),
379          str(write_calls[0][0][0]).split('\n'))
380
381      # Verify the 2nd write of runtime files.
382      expected_runtime_files = ['out_eve/Release/device_script.sh']
383      self.assertListEqual(expected_runtime_files,
384                           str(write_calls[1][0][0]).strip().split('\n'))
385
386      mock_remove.assert_called_once_with('out_eve/Release/device_script.sh')
387
388  def test_gtest_with_vpython(self):
389    """Tests building a gtest with --vpython-dir."""
390    args = mock.MagicMock()
391    args.test_exe = 'base_unittests'
392    args.test_launcher_summary_output = None
393    args.trace_dir = None
394    args.runtime_deps_path = None
395    args.path_to_outdir = self._tmp_dir
396    args.vpython_dir = self._tmp_dir
397    args.logs_dir = self._tmp_dir
398
399    # With vpython_dir initially empty, the test_runner should error out
400    # due to missing vpython binaries.
401    gtest = test_runner.GTestTest(args, None)
402    with self.assertRaises(test_runner.TestFormatError):
403      gtest.build_test_command()
404
405    # Create the two expected tools, and the test should be ready to run.
406    with open(os.path.join(args.vpython_dir, 'vpython3'), 'w'):
407      pass  # Just touch the file.
408    os.mkdir(os.path.join(args.vpython_dir, 'bin'))
409    with open(os.path.join(args.vpython_dir, 'bin', 'python3'), 'w'):
410      pass
411    gtest = test_runner.GTestTest(args, None)
412    gtest.build_test_command()
413
414
415class HostCmdTests(TestRunnerTest):
416
417  @parameterized.expand([
418      [True, False, True],
419      [False, True, True],
420      [True, True, False],
421      [False, True, False],
422  ])
423  def test_host_cmd(self, is_lacros, is_ash, strip_chrome):
424    args = [
425        'script_name',
426        'host-cmd',
427        '--board=eve',
428        '--flash',
429        '--path-to-outdir=out/Release',
430        '--device=localhost:2222',
431    ]
432    if is_lacros:
433      args += ['--deploy-lacros']
434    if is_ash:
435      args += ['--deploy-chrome']
436    if strip_chrome:
437      args += ['--strip-chrome']
438    args += [
439        '--',
440        'fake_cmd',
441    ]
442    with mock.patch.object(sys, 'argv', args),\
443         mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen:
444      mock_popen.return_value.returncode = 0
445
446      test_runner.main()
447      expected_cmd = [
448          test_runner.CROS_RUN_TEST_PATH,
449          '--board',
450          'eve',
451          '--cache-dir',
452          test_runner.DEFAULT_CROS_CACHE,
453          '--flash',
454          '--device',
455          'localhost:2222',
456          '--build-dir',
457          os.path.join(test_runner.CHROMIUM_SRC_PATH, 'out/Release'),
458          '--host-cmd',
459      ]
460      if is_lacros:
461        expected_cmd += [
462            '--deploy-lacros',
463            '--lacros-launcher-script',
464            test_runner.LACROS_LAUNCHER_SCRIPT_PATH,
465        ]
466      if is_ash:
467        expected_cmd += ['--mount', '--deploy']
468      if not strip_chrome:
469        expected_cmd += ['--nostrip']
470
471      expected_cmd += [
472          '--',
473          'fake_cmd',
474      ]
475
476      self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0])
477
478
479if __name__ == '__main__':
480  unittest.main()
481