• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2021 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""This is a library for working with test executables in a way that is
5Chromium-bot-friendly as specified by //docs/testing/test_executable_api.md
6
7Example usage:
8    import os
9    import sys
10
11    import main_program
12    import rust_main_program
13
14    if __name__ == '__main__':
15        cmdline_parser = argparse.ArgumentParser()
16        main_program.add_cmdline_args(cmdline_parser)
17        ... adding other cmdline parameter definitions ...
18        parsed_cmdline_args = cmdline_parser.parse_args()
19
20        test_executable_wrappers = []
21        test_executable_wrappers.append(
22            rust_main_program.TestExecutableWrapper(...))
23        ...
24
25        main_program.main(
26            test_executable_wrappers, parsed_cmdline_args, os.environ)
27"""
28
29import time
30
31import test_filtering
32import test_results
33
34
35def add_cmdline_args(argparse_parser):
36    """Adds test-filtering-specific cmdline parameter definitions to
37    `argparse_parser`.
38
39    Args:
40        argparse_parser: An object of argparse.ArgumentParser type.
41    """
42    test_filtering.add_cmdline_args(argparse_parser)
43    test_results.add_cmdline_args(argparse_parser)
44
45    argparse_parser.add_argument(
46        '--isolated-script-test-launcher-retry-limit',
47        dest='retry_limit',
48        default=3,
49        help='Sets the limit of test retries on failures to N.',
50        metavar='N',
51        type=int)
52    argparse_parser.add_argument('--isolated-script-test-repeat',
53                                 dest='repetitions',
54                                 default=1,
55                                 help='Repeats each test N times.',
56                                 metavar='N',
57                                 type=int)
58
59
60def _calculate_tests_to_run(argparse_parsed_args, env,
61                            test_executable_wrappers):
62    tests = []
63    for wrapper in test_executable_wrappers:
64        extra_tests = wrapper.list_all_tests()
65        for extra_test in extra_tests:
66            assert extra_test not in tests
67        tests.extend(extra_tests)
68    return test_filtering.filter_tests(argparse_parsed_args, env, tests)
69
70
71def _run_tests_and_save_results(argparse_parsed_args, list_of_tests_to_run,
72                                test_executable_wrapper):
73    start_time = time.time()
74    results = []
75    for wrapper in test_executable_wrapper:
76        results.extend(wrapper.run_tests(list_of_tests_to_run))
77    test_results.print_test_results(argparse_parsed_args, results, start_time)
78
79
80def main(test_executable_wrappers, argparse_parsed_args, env):
81    """Runs tests within `test_executable_wrappers` using cmdline arguments and
82    environment variables to figure out 1) which subset of tests to run, 2)
83    where to save the JSON file with test results.
84
85    Args:
86        test_executable_wrappers: A list of objects providing
87          list_all_tests(...) and run_tests(...) methods (see
88          rust_main_program._TestExecutableWrapper).
89        argparse_parsed_arg: A result of an earlier call to
90          argparse_parser.parse_args() call (where `argparse_parser` has been
91          populated via an even earlier call to add_cmdline_args).
92        env: a dictionary-like object (typically from `os.environ`).
93    """
94    list_of_test_names_to_run = _calculate_tests_to_run(
95        argparse_parsed_args, env, test_executable_wrappers)
96    _run_tests_and_save_results(argparse_parsed_args,
97                                list_of_test_names_to_run,
98                                test_executable_wrappers)
99    # TODO(lukasza): Repeat tests `args.repetitions` times.
100    # TODO(lukasza): Retry failing times up to `args.retry_limit` times.
101