Lines Matching +full:test +full:- +full:path
4 # Use of this source code is governed by a BSD-style license that can be
19 # The following non-std imports are fetched via vpython. See the list at
21 import dateutil.parser # pylint: disable=import-error
22 import jsonlines # pylint: disable=import-error
23 import psutil # pylint: disable=import-error
25 CHROMIUM_SRC_PATH = os.path.abspath(
26 os.path.join(os.path.dirname(__file__), '..', '..'))
28 # Use the android test-runner's gtest results support library for generating
30 sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'android'))
31 from pylib.base import base_test_result # pylint: disable=import-error
32 from pylib.results import json_results # pylint: disable=import-error
34 sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'util'))
35 # TODO(crbug.com/1421441): Re-enable the 'no-name-in-module' check.
36 from lib.results import result_sink # pylint: disable=import-error,no-name-in-module
38 import subprocess # pylint: disable=import-error,wrong-import-order
40 DEFAULT_CROS_CACHE = os.path.abspath(
41 os.path.join(CHROMIUM_SRC_PATH, 'build', 'cros_cache'))
42 CHROMITE_PATH = os.path.abspath(
43 os.path.join(CHROMIUM_SRC_PATH, 'third_party', 'chromite'))
44 CROS_RUN_TEST_PATH = os.path.abspath(
45 os.path.join(CHROMITE_PATH, 'bin', 'cros_run_test'))
47 LACROS_LAUNCHER_SCRIPT_PATH = os.path.abspath(
48 os.path.join(CHROMIUM_SRC_PATH, 'build', 'lacros',
72 # test on the device.
78 # python binaries to ~/.vpython-root and /tmp/vpython_bootstrap).
80 # the home and temp dirs for the duration of the test.
99 # invoke the test on the device. If it's not set, we assume self._test_cmd
100 # contains the test invocation.
105 '--board',
107 '--cache-dir',
112 '--start',
114 '--copy-on-write',
118 '--device', args.device if args.device else LAB_DUT_HOSTNAME
122 self._test_cmd += ['--results-src', log]
124 '--results-dest-dir',
125 os.path.join(args.logs_dir, 'system_logs')
128 self._test_cmd += ['--flash']
130 self._test_cmd += ['--public-image']
143 # Since we're using an on_device_script to invoke the test, we'll need to
146 '--remote-cmd',
147 '--cwd',
148 os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH),
160 # caught. This will allow us to capture logs from the device if a test hangs
161 # and gets timeout-killed by swarming. See also:
162 …ooglesource.com/infra/luci/luci-py/+/main/appengine/swarming/doc/Bot.md#graceful-termination_aka-t…
166 logging.warning('Received signal %d. Killing child processes of test.',
170 logging.error('Test process not running.')
180 logging.info('Test attempt #%d', i)
189 except subprocess.TimeoutExpired: # pylint: disable=no-member
190 logging.error('Test timed out. Sending SIGTERM.')
195 except subprocess.TimeoutExpired: # pylint: disable=no-member
197 logging.error('Test did not exit in time. Sending SIGKILL.')
200 logging.info('Test exitted with %d.', test_proc.returncode)
205 # Allow post_run to override test proc return code. (Useful when the host
214 def get_artifacts(path): argument
215 """Crawls a given directory for file artifacts to attach to a test.
218 path: Path to a directory to search for artifacts.
223 for dirpath, _, filenames in os.walk(path):
225 artifact_path = os.path.join(dirpath, f)
226 artifact_id = os.path.relpath(artifact_path, path)
227 # Some artifacts will have non-Latin characters in the filename, eg:
228 # 'ui_tree_Chinese Pinyin-你好.txt'. ResultDB's API rejects such
249 # The CQ passes in '--gtest_filter' when specifying tests to skip. Store it
258 # The host-side Tast bin returns 0 when tests fail, so we need to capture
261 'When using the host-side Tast bin, "--logs-dir" must be passed in '
264 # If the first test filter is negative, it should be safe to assume all of
265 # them are, so just test the first filter.
266 if self._gtest_style_filter and self._gtest_style_filter[0] == '-':
267 raise TestFormatError('Negative test filters not supported for Tast.')
275 '--test-launcher-retry-limit',
276 '--test-launcher-batch-limit',
277 '--gtest_repeat',
292 '--deploy-lacros', '--lacros-launcher-script',
296 self._test_cmd.extend(['--deploy', '--mount'])
298 self._test_cmd.extend(['--deploy', '--mount'])
300 '--build-dir',
301 os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH)
307 '--results-dir',
311 '--tast-total-shards=%d' % self._test_launcher_total_shards,
312 '--tast-shard-index=%d' % self._test_launcher_shard_index,
314 # If we're using a test filter, replace the contents of the Tast
315 # conditional with a long list of "name:test" expressions, one for each
316 # test in the filter.
320 'Presence of --gtest_filter will cause the specified Tast expr'
321 ' or test list to be ignored.')
323 for test in self._gtest_style_filter.split(':'):
324 names.append('"name:%s"' % test)
332 self._test_cmd.append('--tast=%s' % self._attr_expr)
334 self._test_cmd.append('--tast')
338 self._test_cmd.extend(['--tast-var', v])
341 self._test_cmd.append('--tast-retries=%d' % self._tast_retries)
343 # Mounting ash-chrome gives it enough disk space to not need stripping,
345 # Lacros uses --nostrip by default, so there is no need to specify.
347 self._test_cmd.append('--nostrip')
350 tast_results_path = os.path.join(self._logs_dir, 'streamed_results.jsonl')
351 if not os.path.exists(tast_results_path):
363 for test in tast_results:
364 errors = test['errors']
365 start, end = test['start'], test['end']
368 duration = dateutil.parser.parse(end) - dateutil.parser.parse(start)
373 if bool(test['skipReason']):
387 debug_link = ("If you're unsure why this test failed, consult the steps "
390 test['name'], result, duration=duration_ms, log=error_log)
392 self._maybe_handle_perf_results(test['name'])
395 # Walk the contents of the test's "outDir" and atttach any file found
398 artifacts = self.get_artifacts(test['outDir'])
401 html_artifact = 'Test was skipped because: ' + test['skipReason']
403 test['name'],
413 # Attach artifacts from the device that don't apply to a single test.
415 os.path.join(self._logs_dir, 'system_logs'))
417 self.get_artifacts(os.path.join(self._logs_dir, 'crashes')))
436 - process_perf_results looks for top level directories containing a
439 - If a perf_results.json or results-chart.json file exists in the
442 - A trivial test_results.json file is also created to indicate that the test
444 - When process_perf_results is run, it will find the expected files in the
448 perf_results = os.path.join(self._logs_dir, 'tests', test_name,
450 # TODO(stevenjb): Remove check for crosbolt results-chart.json file.
451 if not os.path.exists(perf_results):
452 perf_results = os.path.join(self._logs_dir, 'tests', test_name,
453 'results-chart.json')
454 if os.path.exists(perf_results):
455 benchmark_dir = os.path.join(self._logs_dir, test_name)
456 if not os.path.isdir(benchmark_dir):
459 os.path.join(benchmark_dir, 'perf_results.json'))
462 with open(os.path.join(benchmark_dir, 'test_results.json'), 'w') as out:
486 self._test_cmd += ['--no-clean']
507 json_out_dir = os.path.dirname(self._test_launcher_summary_output) or '.'
508 if os.path.abspath(json_out_dir) != os.path.abspath(self._logs_dir):
510 '--test-launcher-summary-output and --logs-dir must point to '
514 result_dir, result_file = os.path.split(
523 '--results-src',
525 '--results-dest-dir',
530 trace_path = os.path.dirname(self._trace_dir) or '.'
531 if os.path.abspath(trace_path) != os.path.abspath(self._logs_dir):
533 '--trace-dir and --logs-dir must point to the same directory.')
536 trace_path, trace_dirname = os.path.split(self._trace_dir)
539 '--results-src',
541 '--results-dest-dir',
545 # Build the shell script that will be used on the device to invoke the test.
552 vpython_path = os.path.join(self._path_to_outdir, self._vpython_dir,
554 cpython_path = os.path.join(self._path_to_outdir, self._vpython_dir,
556 if not os.path.exists(vpython_path) or not os.path.exists(cpython_path):
558 '--vpython-dir must point to a dir with both '
560 vpython_spec_path = os.path.relpath(
561 os.path.join(CHROMIUM_SRC_PATH, '.vpython3'), self._path_to_outdir)
562 # Initialize the vpython cache. This can take 10-20s, and some tests
565 'export PATH=$PWD/%s:$PWD/%s/bin/:$PATH' %
567 'vpython3 -vpython-spec %s -vpython-tool install' %
571 test_invocation = ('LD_LIBRARY_PATH=./ ./%s --test-launcher-shard-index=%d '
572 '--test-launcher-total-shards=%d' %
576 test_invocation += ' --test-launcher-summary-output=%s' % (
581 'rm -rf %s' % device_trace_dir,
582 'sudo -E -u chronos -- /bin/bash -c "mkdir -p %s"' % device_trace_dir,
584 test_invocation += ' --trace-dir=%s' % device_trace_dir
589 './test_sudo_helper.py --socket-path=${TEST_SUDO_HELPER_PATH} &',
593 ' --test-sudo-helper-socket-path=${TEST_SUDO_HELPER_PATH}')
596 # line consisting of "<file-regex> <file-type> <new-label>", where '--' is
603 'echo %s -- %s > %s' % (filename, label, specfile),
604 'setfiles -F %s %s' % (specfile, filename),
616 'dbus-send --system --type=method_call'
617 ' --dest=org.chromium.PowerManager /org/chromium/PowerManager'
621 # it down as chronos kills the entire execution of the test. So we'll have
622 # to run as root up until the test invocation.
624 'sudo -E -u chronos -- /bin/bash -c "%s"' % test_invocation)
625 # And we'll need to chown everything since cros_run_test's "--as-chronos"
627 device_test_script_contents.append('chown -R chronos: ../..')
632 '--as-chronos',
648 'pkill -P $TEST_SUDO_HELPER_PID',
656 runtime_files = [os.path.relpath(self._on_device_script)]
659 # --vpython-dir is relative to the out dir, but --files expects paths
660 # relative to src dir, so fix the path up a bit.
662 os.path.relpath(
663 os.path.abspath(
664 os.path.join(self._path_to_outdir, self._vpython_dir)),
668 self._test_cmd.extend(['--files', f])
671 '--',
672 './' + os.path.relpath(self._on_device_script, self._path_to_outdir)
679 abs_runtime_deps_path = os.path.abspath(
680 os.path.join(self._path_to_outdir, self._runtime_deps_path))
685 rel_file_path = os.path.relpath(
686 os.path.abspath(os.path.join(self._path_to_outdir, f)))
709 test = TastTest(args, unknown_args)
711 test = GTestTest(args, unknown_args)
713 test.build_test_command()
715 logging.info(' '.join(test.test_cmd))
717 return test.run_test()
725 '--path-to-outdir must be specified if --deploy-chrome is passed.')
729 '--board',
731 '--cache-dir',
732 os.path.join(CHROMIUM_SRC_PATH, args.cros_cache),
736 '--start',
738 '--copy-on-write',
742 '--device', args.device if args.device else LAB_DUT_HOSTNAME
745 cros_run_test_cmd.append('--debug')
747 cros_run_test_cmd.append('--flash')
749 cros_run_test_cmd += ['--public-image']
753 cros_run_test_cmd += ['--results-src', log]
755 '--results-dest-dir',
756 os.path.join(args.logs_dir, 'system_logs')
763 '--deploy-lacros', '--lacros-launcher-script',
767 # Mounting ash-chrome gives it enough disk space to not need stripping
769 cros_run_test_cmd.extend(['--deploy', '--mount'])
771 # Mounting ash-chrome gives it enough disk space to not need stripping
773 cros_run_test_cmd.extend(['--deploy', '--mount'])
776 cros_run_test_cmd.append('--nostrip')
779 '--build-dir',
780 os.path.join(CHROMIUM_SRC_PATH, args.path_to_outdir)
784 '--host-cmd',
785 '--',
798 # Some chromite scripts expect chromite/bin to be on PATH.
799 env['PATH'] = env['PATH'] + ':' + os.path.join(CHROMITE_PATH, 'bin')
804 # TODO(crbug.com/823996): Make the GN-dependent deps controllable via cmd
815 parser.add_argument('--verbose', '-v', action='store_true')
817 '--board', type=str, required=True, help='Type of CrOS device.')
819 '--deploy-chrome',
821 help='Will deploy a locally built ash-chrome binary to the device '
822 'before running the host-cmd.')
824 '--deploy-lacros', action='store_true', help='Deploy a lacros-chrome.')
826 '--cros-cache',
829 help='Path to cros cache.')
831 '--path-to-outdir',
834 help='Path to output directory, all of whose contents will be '
837 '--runtime-deps-path',
841 '--vpython-dir',
844 'deploy to the device before the test starts. The location of '
845 'this dir will be added onto PATH in the device. WARNING: The '
849 '--logs-dir',
853 'test into the specified dir.')
856 '--test-launcher-shard-index',
861 '--test-launcher-total-shards',
866 '--flash',
869 'the test.')
871 '--no-flash',
874 help='Will not flash the device before running the test.')
876 '--public-image',
880 '--magic-vm-cache',
881 help='Path to the magic CrOS VM cache dir. See the comment above '
886 '--use-vm',
888 help='Will run the test in the VM instead of a device.')
890 '--device',
892 help='Hostname (or IP) of device to run the test on. This arg is not '
893 'required if --use-vm is set.')
899 # Host-side test args.
901 'host-cmd',
902 help='Runs a host-side test. Pass the host-side command to run after '
903 '"--". If --use-vm is passed, hostname and port for the device '
907 '--strip-chrome',
909 help='Strips symbols from ash-chrome or lacros-chrome before deploying '
913 'gtest', help='Runs a device-side gtest.')
916 '--test-exe',
919 help='Path to test executable to run inside the device.')
921 # GTest args. Some are passed down to the test binary in the device. Others
924 '--test-launcher-summary-output',
926 help='When set, will pass the same option down to the test and retrieve '
929 '--stop-ui',
931 help='Will stop the UI service in the device before running the test. '
934 '--trace-dir',
936 help='When set, will pass down to the test to generate the trace and '
939 '--env-var',
943 help='Env var to set on the device for the duration of the test. '
944 'Expected format is "--env-var SOME_VAR_NAME some_var_value". Specify '
947 '--run-test-sudo-helper',
949 help='When set, will run test_sudo_helper before the test and stop it '
950 'after test finishes.')
952 "--no-clean",
956 help="Do not clean up the deployed files after running the test. "
957 "Only supported for --remote-cmd tests")
959 '--set-selinux-label',
963 ' --set-selinux-label=<filename>=<label>\n'
965 ' --set-selinux-label=my_test=u:r:cros_foo_label:s0\n'
968 # Tast test args.
969 # pylint: disable=line-too-long
972 help='Runs a device-side set of Tast tests. For more details, see: '
977 '--suite-name',
981 'on what is executed, but is used mainly for test results reporting '
984 '--test-launcher-summary-output',
986 help='Generates a simple GTest-style JSON result file for the test run.')
988 '--attr-expr',
993 '--strip-chrome',
995 help='Strips symbols from ash-chrome before deploying to the device.')
997 '--tast-var',
1003 '--tast-retries',
1008 '--test',
1009 '-t',
1012 help='A Tast test to run in the device (eg: "login.Chrome").')
1014 '--gtest_filter',
1018 'cmd-line API, this will overwrite the value(s) of "--test" above.')
1022 # Re-add N-1 -v/--verbose flags to the args we'll pass to whatever we are
1026 verbose_flags = [a for a in sys.argv if a in ('-v', '--verbose')]
1034 'The test runner is now assuming running in the lab environment, if '
1035 'this is unintentional, please re-invoke the test runner with the '
1036 '"--use-vm" arg if using a VM, otherwise use the "--device=<DUT>" arg '
1040 # we're on a lab bot and are trying to run a test on a lab DUT. See if the
1057 full_vm_cache_path = os.path.join(CHROMIUM_SRC_PATH, args.magic_vm_cache)
1058 if os.path.exists(full_vm_cache_path):
1059 with open(os.path.join(full_vm_cache_path, 'swarming.txt'), 'w') as f:
1060 f.write('non-empty file to make swarming persist this cache')