• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017, The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""
16Aggregates test runners, groups tests by test runners and kicks off tests.
17"""
18
19import itertools
20import time
21import traceback
22
23import atest_error
24import constants
25import result_reporter
26
27from metrics import metrics
28from metrics import metrics_utils
29from test_runners import atest_tf_test_runner
30from test_runners import robolectric_test_runner
31from test_runners import suite_plan_test_runner
32from test_runners import vts_tf_test_runner
33
34# pylint: disable=line-too-long
35_TEST_RUNNERS = {
36    atest_tf_test_runner.AtestTradefedTestRunner.NAME: atest_tf_test_runner.AtestTradefedTestRunner,
37    robolectric_test_runner.RobolectricTestRunner.NAME: robolectric_test_runner.RobolectricTestRunner,
38    suite_plan_test_runner.SuitePlanTestRunner.NAME: suite_plan_test_runner.SuitePlanTestRunner,
39    vts_tf_test_runner.VtsTradefedTestRunner.NAME: vts_tf_test_runner.VtsTradefedTestRunner,
40}
41
42
43def _get_test_runners():
44    """Returns the test runners.
45
46    If external test runners are defined outside atest, they can be try-except
47    imported into here.
48
49    Returns:
50        Dict of test runner name to test runner class.
51    """
52    test_runners_dict = _TEST_RUNNERS
53    # Example import of example test runner:
54    try:
55        # pylint: disable=line-too-long
56        from test_runners import example_test_runner
57        test_runners_dict[example_test_runner.ExampleTestRunner.NAME] = example_test_runner.ExampleTestRunner
58    except ImportError:
59        pass
60    return test_runners_dict
61
62
63def group_tests_by_test_runners(test_infos):
64    """Group the test_infos by test runners
65
66    Args:
67        test_infos: List of TestInfo.
68
69    Returns:
70        List of tuples (test runner, tests).
71    """
72    tests_by_test_runner = []
73    test_runner_dict = _get_test_runners()
74    key = lambda x: x.test_runner
75    sorted_test_infos = sorted(list(test_infos), key=key)
76    for test_runner, tests in itertools.groupby(sorted_test_infos, key):
77        # groupby returns a grouper object, we want to operate on a list.
78        tests = list(tests)
79        test_runner_class = test_runner_dict.get(test_runner)
80        if test_runner_class is None:
81            raise atest_error.UnknownTestRunnerError('Unknown Test Runner %s' %
82                                                     test_runner)
83        tests_by_test_runner.append((test_runner_class, tests))
84    return tests_by_test_runner
85
86
87def get_test_runner_reqs(module_info, test_infos):
88    """Returns the requirements for all test runners specified in the tests.
89
90    Args:
91        module_info: ModuleInfo object.
92        test_infos: List of TestInfo.
93
94    Returns:
95        Set of build targets required by the test runners.
96    """
97    unused_result_dir = ''
98    test_runner_build_req = set()
99    for test_runner, _ in group_tests_by_test_runners(test_infos):
100        test_runner_build_req |= test_runner(
101            unused_result_dir,
102            module_info=module_info).get_test_runner_build_reqs()
103    return test_runner_build_req
104
105
106def run_all_tests(results_dir, test_infos, extra_args,
107                  delay_print_summary=False):
108    """Run the given tests.
109
110    Args:
111        results_dir: String directory to store atest results.
112        test_infos: List of TestInfo.
113        extra_args: Dict of extra args for test runners to use.
114
115    Returns:
116        0 if tests succeed, non-zero otherwise.
117    """
118    reporter = result_reporter.ResultReporter()
119    reporter.print_starting_text()
120    tests_ret_code = constants.EXIT_CODE_SUCCESS
121    for test_runner, tests in group_tests_by_test_runners(test_infos):
122        test_name = ' '.join([test.test_name for test in tests])
123        test_start = time.time()
124        is_success = True
125        ret_code = constants.EXIT_CODE_TEST_FAILURE
126        stacktrace = ''
127        try:
128            test_runner = test_runner(results_dir)
129            ret_code = test_runner.run_tests(tests, extra_args, reporter)
130            tests_ret_code |= ret_code
131        # pylint: disable=broad-except
132        except Exception:
133            stacktrace = traceback.format_exc()
134            reporter.runner_failure(test_runner.NAME, stacktrace)
135            tests_ret_code = constants.EXIT_CODE_TEST_FAILURE
136            is_success = False
137        metrics.RunnerFinishEvent(
138            duration=metrics_utils.convert_duration(time.time() - test_start),
139            success=is_success,
140            runner_name=test_runner.NAME,
141            test=[{'name': test_name,
142                   'result': ret_code,
143                   'stacktrace': stacktrace}])
144    if delay_print_summary:
145        return tests_ret_code, reporter
146    return reporter.print_summary() or tests_ret_code, reporter
147