Lines Matching +full:test +full:- +full:path
4 # Use of this source code is governed by a BSD-style license that can be
20 # The following non-std imports are fetched via vpython. See the list at
22 import dateutil.parser # pylint: disable=import-error
23 import jsonlines # pylint: disable=import-error
24 import psutil # pylint: disable=import-error
26 CHROMIUM_SRC_PATH = os.path.abspath(
27 os.path.join(os.path.dirname(__file__), '..', '..'))
29 # Use the android test-runner's gtest results support library for generating
31 sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'android'))
32 from pylib.base import base_test_result # pylint: disable=import-error
33 from pylib.results import json_results # pylint: disable=import-error
35 sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'util'))
36 # TODO(crbug.com/1421441): Re-enable the 'no-name-in-module' check.
37 from lib.results import result_sink # pylint: disable=import-error,no-name-in-module
41 import subprocess # pylint: disable=import-error,wrong-import-order
43 DEFAULT_CROS_CACHE = os.path.abspath(
44 os.path.join(CHROMIUM_SRC_PATH, 'build', 'cros_cache'))
45 CHROMITE_PATH = os.path.abspath(
46 os.path.join(CHROMIUM_SRC_PATH, 'third_party', 'chromite'))
47 CROS_RUN_TEST_PATH = os.path.abspath(
48 os.path.join(CHROMITE_PATH, 'bin', 'cros_run_test'))
50 LACROS_LAUNCHER_SCRIPT_PATH = os.path.abspath(
51 os.path.join(CHROMIUM_SRC_PATH, 'build', 'lacros',
75 # test on the device.
81 # python binaries to ~/.vpython-root and /tmp/vpython_bootstrap).
83 # the home and temp dirs for the duration of the test.
102 # invoke the test on the device. If it's not set, we assume self._test_cmd
103 # contains the test invocation.
108 '--board',
110 '--cache-dir',
115 '--start',
117 '--copy-on-write',
121 '--device', args.device if args.device else LAB_DUT_HOSTNAME
125 self._test_cmd += ['--results-src', log]
127 '--results-dest-dir',
128 os.path.join(args.logs_dir, 'system_logs')
131 self._test_cmd += ['--flash']
133 self._test_cmd += ['--public-image']
146 # Since we're using an on_device_script to invoke the test, we'll need to
149 '--remote-cmd',
150 '--cwd',
151 os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH),
163 # caught. This will allow us to capture logs from the device if a test hangs
164 # and gets timeout-killed by swarming. See also:
165 …ooglesource.com/infra/luci/luci-py/+/main/appengine/swarming/doc/Bot.md#graceful-termination_aka-t…
169 logging.warning('Received signal %d. Killing child processes of test.',
173 logging.error('Test process not running.')
183 logging.info('Test attempt #%d', i)
192 except subprocess.TimeoutExpired: # pylint: disable=no-member
193 logging.error('Test timed out. Sending SIGTERM.')
198 except subprocess.TimeoutExpired: # pylint: disable=no-member
200 logging.error('Test did not exit in time. Sending SIGKILL.')
203 logging.info('Test exitted with %d.', test_proc.returncode)
208 # Allow post_run to override test proc return code. (Useful when the host
217 def get_artifacts(path): argument
218 """Crawls a given directory for file artifacts to attach to a test.
221 path: Path to a directory to search for artifacts.
226 for dirpath, _, filenames in os.walk(path):
228 artifact_path = os.path.join(dirpath, f)
229 artifact_id = os.path.relpath(artifact_path, path)
230 # Some artifacts will have non-Latin characters in the filename, eg:
231 # 'ui_tree_Chinese Pinyin-你好.txt'. ResultDB's API rejects such
252 # The CQ passes in '--gtest_filter' when specifying tests to skip. Store it
261 # The host-side Tast bin returns 0 when tests fail, so we need to capture
264 'When using the host-side Tast bin, "--logs-dir" must be passed in '
267 # If the first test filter is negative, it should be safe to assume all of
268 # them are, so just test the first filter.
269 if self._gtest_style_filter and self._gtest_style_filter[0] == '-':
270 raise TestFormatError('Negative test filters not supported for Tast.')
278 '--test-launcher-retry-limit',
279 '--test-launcher-batch-limit',
280 '--gtest_repeat',
295 '--deploy-lacros', '--lacros-launcher-script',
299 self._test_cmd.extend(['--deploy', '--mount'])
301 self._test_cmd.extend(['--deploy', '--mount'])
303 '--build-dir',
304 os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH)
310 '--results-dir',
314 '--tast-total-shards=%d' % self._test_launcher_total_shards,
315 '--tast-shard-index=%d' % self._test_launcher_shard_index,
317 # If we're using a test filter, replace the contents of the Tast
318 # conditional with a long list of "name:test" expressions, one for each
319 # test in the filter.
323 'Presence of --gtest_filter will cause the specified Tast expr'
324 ' or test list to be ignored.')
326 for test in self._gtest_style_filter.split(':'):
327 names.append('"name:%s"' % test)
335 self._test_cmd.append('--tast=%s' % self._attr_expr)
337 self._test_cmd.append('--tast')
341 self._test_cmd.extend(['--tast-var', v])
344 self._test_cmd.append('--tast-retries=%d' % self._tast_retries)
346 # Mounting ash-chrome gives it enough disk space to not need stripping,
348 # Lacros uses --nostrip by default, so there is no need to specify.
350 self._test_cmd.append('--nostrip')
353 tast_results_path = os.path.join(self._logs_dir, 'streamed_results.jsonl')
354 if not os.path.exists(tast_results_path):
366 for test in tast_results:
367 errors = test['errors']
368 start, end = test['start'], test['end']
371 duration = dateutil.parser.parse(end) - dateutil.parser.parse(start)
376 if bool(test['skipReason']):
390 debug_link = ("If you're unsure why this test failed, consult the steps "
393 test['name'], result, duration=duration_ms, log=error_log)
395 self._maybe_handle_perf_results(test['name'])
398 # Walk the contents of the test's "outDir" and atttach any file found
401 artifacts = self.get_artifacts(test['outDir'])
403 test['name'],
413 # Attach artifacts from the device that don't apply to a single test.
415 os.path.join(self._logs_dir, 'system_logs'))
417 self.get_artifacts(os.path.join(self._logs_dir, 'crashes')))
436 - process_perf_results looks for top level directories containing a
439 - If a perf_results.json or results-chart.json file exists in the
442 - A trivial test_results.json file is also created to indicate that the test
444 - When process_perf_results is run, it will find the expected files in the
448 perf_results = os.path.join(self._logs_dir, 'tests', test_name,
450 # TODO(stevenjb): Remove check for crosbolt results-chart.json file.
451 if not os.path.exists(perf_results):
452 perf_results = os.path.join(self._logs_dir, 'tests', test_name,
453 'results-chart.json')
454 if os.path.exists(perf_results):
455 benchmark_dir = os.path.join(self._logs_dir, test_name)
456 if not os.path.isdir(benchmark_dir):
459 os.path.join(benchmark_dir, 'perf_results.json'))
462 with open(os.path.join(benchmark_dir, 'test_results.json'), 'w') as out:
501 json_out_dir = os.path.dirname(self._test_launcher_summary_output) or '.'
502 if os.path.abspath(json_out_dir) != os.path.abspath(self._logs_dir):
504 '--test-launcher-summary-output and --logs-dir must point to '
508 result_dir, result_file = os.path.split(
517 '--results-src',
519 '--results-dest-dir',
524 trace_path = os.path.dirname(self._trace_dir) or '.'
525 if os.path.abspath(trace_path) != os.path.abspath(self._logs_dir):
527 '--trace-dir and --logs-dir must point to the same directory.')
530 trace_path, trace_dirname = os.path.split(self._trace_dir)
533 '--results-src',
535 '--results-dest-dir',
539 # Build the shell script that will be used on the device to invoke the test.
546 vpython_path = os.path.join(self._path_to_outdir, self._vpython_dir,
548 cpython_path = os.path.join(self._path_to_outdir, self._vpython_dir,
550 if not os.path.exists(vpython_path) or not os.path.exists(cpython_path):
552 '--vpython-dir must point to a dir with both '
554 vpython_spec_path = os.path.relpath(
555 os.path.join(CHROMIUM_SRC_PATH, '.vpython3'), self._path_to_outdir)
556 # Initialize the vpython cache. This can take 10-20s, and some tests
559 'export PATH=$PWD/%s:$PWD/%s/bin/:$PATH' %
561 'vpython3 -vpython-spec %s -vpython-tool install' %
565 test_invocation = ('LD_LIBRARY_PATH=./ ./%s --test-launcher-shard-index=%d '
566 '--test-launcher-total-shards=%d' %
570 test_invocation += ' --test-launcher-summary-output=%s' % (
575 'rm -rf %s' % device_trace_dir,
576 'sudo -E -u chronos -- /bin/bash -c "mkdir -p %s"' % device_trace_dir,
578 test_invocation += ' --trace-dir=%s' % device_trace_dir
588 # it down as chronos kills the entire execution of the test. So we'll have
589 # to run as root up until the test invocation.
591 'sudo -E -u chronos -- /bin/bash -c "%s"' % test_invocation)
592 # And we'll need to chown everything since cros_run_test's "--as-chronos"
594 device_test_script_contents.append('chown -R chronos: ../..')
599 '--as-chronos',
607 runtime_files = [os.path.relpath(self._on_device_script)]
610 # --vpython-dir is relative to the out dir, but --files expects paths
611 # relative to src dir, so fix the path up a bit.
613 os.path.relpath(
614 os.path.abspath(
615 os.path.join(self._path_to_outdir, self._vpython_dir)),
619 self._test_cmd.extend(['--files', f])
622 '--',
623 './' + os.path.relpath(self._on_device_script, self._path_to_outdir)
630 abs_runtime_deps_path = os.path.abspath(
631 os.path.join(self._path_to_outdir, self._runtime_deps_path))
636 rel_file_path = os.path.relpath(
637 os.path.abspath(os.path.join(self._path_to_outdir, f)))
660 test = TastTest(args, unknown_args)
662 test = GTestTest(args, unknown_args)
664 test.build_test_command()
666 logging.info(' '.join(test.test_cmd))
668 return test.run_test()
676 '--path-to-outdir must be specified if --deploy-chrome is passed.')
680 '--board',
682 '--cache-dir',
683 os.path.join(CHROMIUM_SRC_PATH, args.cros_cache),
687 '--start',
689 '--copy-on-write',
693 '--device', args.device if args.device else LAB_DUT_HOSTNAME
696 cros_run_test_cmd.append('--debug')
698 cros_run_test_cmd.append('--flash')
700 cros_run_test_cmd += ['--public-image']
704 cros_run_test_cmd += ['--results-src', log]
706 '--results-dest-dir',
707 os.path.join(args.logs_dir, 'system_logs')
714 '--deploy-lacros', '--lacros-launcher-script',
718 # Mounting ash-chrome gives it enough disk space to not need stripping
720 cros_run_test_cmd.extend(['--deploy', '--mount'])
722 # Mounting ash-chrome gives it enough disk space to not need stripping
724 cros_run_test_cmd.extend(['--deploy', '--mount'])
727 cros_run_test_cmd.append('--nostrip')
730 '--build-dir',
731 os.path.join(CHROMIUM_SRC_PATH, args.path_to_outdir)
735 '--host-cmd',
736 '--',
749 # Some chromite scripts expect chromite/bin to be on PATH.
750 env['PATH'] = env['PATH'] + ':' + os.path.join(CHROMITE_PATH, 'bin')
755 # TODO(crbug.com/823996): Make the GN-dependent deps controllable via cmd
766 parser.add_argument('--verbose', '-v', action='store_true')
768 '--board', type=str, required=True, help='Type of CrOS device.')
770 '--deploy-chrome',
772 help='Will deploy a locally built ash-chrome binary to the device '
773 'before running the host-cmd.')
775 '--deploy-lacros', action='store_true', help='Deploy a lacros-chrome.')
777 '--cros-cache',
780 help='Path to cros cache.')
782 '--path-to-outdir',
785 help='Path to output directory, all of whose contents will be '
788 '--runtime-deps-path',
792 '--vpython-dir',
795 'deploy to the device before the test starts. The location of '
796 'this dir will be added onto PATH in the device. WARNING: The '
800 '--logs-dir',
804 'test into the specified dir.')
807 '--test-launcher-shard-index',
812 '--test-launcher-total-shards',
817 '--flash',
820 'the test.')
822 '--public-image',
826 '--magic-vm-cache',
827 help='Path to the magic CrOS VM cache dir. See the comment above '
832 '--use-vm',
834 help='Will run the test in the VM instead of a device.')
836 '--device',
838 help='Hostname (or IP) of device to run the test on. This arg is not '
839 'required if --use-vm is set.')
845 # Host-side test args.
847 'host-cmd',
848 help='Runs a host-side test. Pass the host-side command to run after '
849 '"--". If --use-vm is passed, hostname and port for the device '
853 '--strip-chrome',
855 help='Strips symbols from ash-chrome or lacros-chrome before deploying '
859 'gtest', help='Runs a device-side gtest.')
862 '--test-exe',
865 help='Path to test executable to run inside the device.')
867 # GTest args. Some are passed down to the test binary in the device. Others
870 '--test-launcher-summary-output',
872 help='When set, will pass the same option down to the test and retrieve '
875 '--stop-ui',
877 help='Will stop the UI service in the device before running the test.')
879 '--trace-dir',
881 help='When set, will pass down to the test to generate the trace and '
884 '--env-var',
888 help='Env var to set on the device for the duration of the test. '
889 'Expected format is "--env-var SOME_VAR_NAME some_var_value". Specify '
892 # Tast test args.
893 # pylint: disable=line-too-long
896 help='Runs a device-side set of Tast tests. For more details, see: '
901 '--suite-name',
905 'on what is executed, but is used mainly for test results reporting '
908 '--test-launcher-summary-output',
910 help='Generates a simple GTest-style JSON result file for the test run.')
912 '--attr-expr',
917 '--strip-chrome',
919 help='Strips symbols from ash-chrome before deploying to the device.')
921 '--tast-var',
927 '--tast-retries',
932 '--test',
933 '-t',
936 help='A Tast test to run in the device (eg: "login.Chrome").')
938 '--gtest_filter',
942 'cmd-line API, this will overwrite the value(s) of "--test" above.')
946 # Re-add N-1 -v/--verbose flags to the args we'll pass to whatever we are
950 verbose_flags = [a for a in sys.argv if a in ('-v', '--verbose')]
958 'The test runner is now assuming running in the lab environment, if '
959 'this is unintentional, please re-invoke the test runner with the '
960 '"--use-vm" arg if using a VM, otherwise use the "--device=<DUT>" arg '
964 # we're on a lab bot and are trying to run a test on a lab DUT. See if the
981 full_vm_cache_path = os.path.join(CHROMIUM_SRC_PATH, args.magic_vm_cache)
982 if os.path.exists(full_vm_cache_path):
983 with open(os.path.join(full_vm_cache_path, 'swarming.txt'), 'w') as f:
984 f.write('non-empty file to make swarming persist this cache')