• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 Google Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Runner for Mobly test suites.
15
16These is just example code to help users run a collection of Mobly test
17classes. Users can use it as is or customize it based on their requirements.
18
19There are two ways to use this runner.
20
211. Call suite_runner.run_suite() with one or more individual test classes. This
22is for users who just need to execute a collection of test classes without any
23additional steps.
24
25.. code-block:: python
26
27  from mobly import suite_runner
28
29  from my.test.lib import foo_test
30  from my.test.lib import bar_test
31  ...
32  if __name__ == '__main__':
33    suite_runner.run_suite(foo_test.FooTest, bar_test.BarTest)
34
352. Create a subclass of base_suite.BaseSuite and add the individual test
36classes. Using the BaseSuite class allows users to define their own setup
37and teardown steps on the suite level as well as custom config for each test
38class.
39
40.. code-block:: python
41
42  from mobly import base_suite
43  from mobly import suite_runner
44
45  from my.path import MyFooTest
46  from my.path import MyBarTest
47
48
49  class MySuite(base_suite.BaseSuite):
50
51    def setup_suite(self, config):
52      # Add a class with default config.
53      self.add_test_class(MyFooTest)
54      # Add a class with test selection.
55      self.add_test_class(MyBarTest,
56                          tests=['test_a', 'test_b'])
57      # Add the same class again with a custom config and suffix.
58      my_config = some_config_logic(config)
59      self.add_test_class(MyBarTest,
60                          config=my_config,
61                          name_suffix='WithCustomConfig')
62
63
64  if __name__ == '__main__':
65    suite_runner.run_suite_class()
66"""
67import argparse
68import collections
69import inspect
70import logging
71import sys
72
73from mobly import base_test
74from mobly import base_suite
75from mobly import config_parser
76from mobly import signals
77from mobly import test_runner
78
79
80class Error(Exception):
81  pass
82
83
84def _parse_cli_args(argv):
85  """Parses cli args that are consumed by Mobly.
86
87  Args:
88    argv: A list that is then parsed as cli args. If None, defaults to cli
89      input.
90
91  Returns:
92    Namespace containing the parsed args.
93  """
94  parser = argparse.ArgumentParser(description='Mobly Suite Executable.')
95  group = parser.add_mutually_exclusive_group(required=True)
96  group.add_argument('-c',
97                     '--config',
98                     type=str,
99                     metavar='<PATH>',
100                     help='Path to the test configuration file.')
101  group.add_argument(
102      '-l',
103      '--list_tests',
104      action='store_true',
105      help='Print the names of the tests defined in a script without '
106      'executing them.')
107  parser.add_argument('--tests',
108                      '--test_case',
109                      nargs='+',
110                      type=str,
111                      metavar='[ClassA[.test_a] ClassB[.test_b] ...]',
112                      help='A list of test classes and optional tests to execute.')
113  parser.add_argument('-tb',
114                      '--test_bed',
115                      nargs='+',
116                      type=str,
117                      metavar='[<TEST BED NAME1> <TEST BED NAME2> ...]',
118                      help='Specify which test beds to run tests on.')
119
120  parser.add_argument('-v',
121                      '--verbose',
122                      action='store_true',
123                      help='Set console logger level to DEBUG')
124  if not argv:
125    argv = sys.argv[1:]
126  return parser.parse_known_args(argv)[0]
127
128
129def _find_suite_class():
130  """Finds the test suite class in the current module.
131
132  Walk through module members and find the subclass of BaseSuite. Only
133  one subclass is allowed in a module.
134
135  Returns:
136      The test suite class in the test module.
137  """
138  test_suites = []
139  main_module_members = sys.modules['__main__']
140  for _, module_member in main_module_members.__dict__.items():
141    if inspect.isclass(module_member):
142      if issubclass(module_member, base_suite.BaseSuite):
143        test_suites.append(module_member)
144  if len(test_suites) != 1:
145    logging.error('Expected 1 test class per file, found %s.',
146                  [t.__name__ for t in test_suites])
147    sys.exit(1)
148  return test_suites[0]
149
150
151def _print_test_names(test_classes):
152  """Prints the names of all the tests in all test classes.
153  Args:
154    test_classes: classes, the test classes to print names from.
155  """
156  for test_class in test_classes:
157    cls = test_class(config_parser.TestRunConfig())
158    test_names = []
159    try:
160      # Executes pre-setup procedures, this is required since it might
161      # generate test methods that we want to return as well.
162      cls._pre_run()
163      if cls.tests:
164        # Specified by run list in class.
165        test_names = list(cls.tests)
166      else:
167        # No test method specified by user, list all in test class.
168        test_names = cls.get_existing_test_names()
169    except Exception:
170      logging.exception('Failed to retrieve generated tests.')
171    finally:
172      cls._clean_up()
173    print('==========> %s <==========' % cls.TAG)
174    for name in test_names:
175      print(f"{cls.TAG}.{name}")
176
177
178def run_suite_class(argv=None):
179  """Executes tests in the test suite.
180
181  Args:
182    argv: A list that is then parsed as CLI args. If None, defaults to sys.argv.
183  """
184  cli_args = _parse_cli_args(argv)
185  suite_class = _find_suite_class()
186  if cli_args.list_tests:
187    _print_test_names([suite_class])
188    sys.exit(0)
189  test_configs = config_parser.load_test_config_file(cli_args.config,
190                                                     cli_args.test_bed)
191  config_count = len(test_configs)
192  if config_count != 1:
193    logging.error('Expect exactly one test config, found %d', config_count)
194  config = test_configs[0]
195  runner = test_runner.TestRunner(
196      log_dir=config.log_path, testbed_name=config.testbed_name)
197  suite = suite_class(runner, config)
198  console_level = logging.DEBUG if cli_args.verbose else logging.INFO
199  ok = False
200  with runner.mobly_logger(console_level=console_level):
201    try:
202      suite.setup_suite(config.copy())
203      try:
204        runner.run()
205        ok = runner.results.is_all_pass
206        print(ok)
207      except signals.TestAbortAll:
208        pass
209    finally:
210      suite.teardown_suite()
211  if not ok:
212    sys.exit(1)
213
214
215def run_suite(test_classes, argv=None):
216  """Executes multiple test classes as a suite.
217
218  This is the default entry point for running a test suite script file
219  directly.
220
221  Args:
222    test_classes: List of python classes containing Mobly tests.
223    argv: A list that is then parsed as cli args. If None, defaults to cli
224      input.
225  """
226  args = _parse_cli_args(argv)
227
228  # Check the classes that were passed in
229  for test_class in test_classes:
230    if not issubclass(test_class, base_test.BaseTestClass):
231      logging.error(
232          'Test class %s does not extend '
233          'mobly.base_test.BaseTestClass', test_class)
234      sys.exit(1)
235
236  if args.list_tests:
237    _print_test_names(test_classes)
238    sys.exit(0)
239
240  # Load test config file.
241  test_configs = config_parser.load_test_config_file(args.config,
242                                                     args.test_bed)
243  # Find the full list of tests to execute
244  selected_tests = compute_selected_tests(test_classes, args.tests)
245
246  console_level = logging.DEBUG if args.verbose else logging.INFO
247  # Execute the suite
248  ok = True
249  for config in test_configs:
250    runner = test_runner.TestRunner(config.log_path, config.testbed_name)
251    with runner.mobly_logger(console_level=console_level):
252      for (test_class, tests) in selected_tests.items():
253        runner.add_test_class(config, test_class, tests)
254      try:
255        runner.run()
256        ok = runner.results.is_all_pass and ok
257      except signals.TestAbortAll:
258        pass
259      except Exception:
260        logging.exception('Exception when executing %s.', config.testbed_name)
261        ok = False
262  if not ok:
263    sys.exit(1)
264
265
266def compute_selected_tests(test_classes, selected_tests):
267  """Computes tests to run for each class from selector strings.
268
269  This function transforms a list of selector strings (such as FooTest or
270  FooTest.test_method_a) to a dict where keys are test_name classes, and
271  values are lists of selected tests in those classes. None means all tests in
272  that class are selected.
273
274  Args:
275    test_classes: list of strings, names of all the classes that are part
276      of a suite.
277    selected_tests: list of strings, list of tests to execute. If empty,
278      all classes `test_classes` are selected. E.g.
279
280      .. code-block:: python
281
282        [
283          'FooTest',
284          'BarTest',
285          'BazTest.test_method_a',
286          'BazTest.test_method_b'
287        ]
288
289  Returns:
290    dict: Identifiers for TestRunner. Keys are test class names; valures
291      are lists of test names within class. E.g. the example in
292      `selected_tests` would translate to:
293
294      .. code-block:: python
295
296        {
297          FooTest: None,
298          BarTest: None,
299          BazTest: ['test_method_a', 'test_method_b']
300        }
301
302      This dict is easy to consume for `TestRunner`.
303  """
304  class_to_tests = collections.OrderedDict()
305  if not selected_tests:
306    # No selection is needed; simply run all tests in all classes.
307    for test_class in test_classes:
308      class_to_tests[test_class] = None
309    return class_to_tests
310
311  # The user is selecting some tests to run. Parse the selectors.
312  # Dict from test_name class name to list of tests to execute (or None for all
313  # tests).
314  test_class_name_to_tests = collections.OrderedDict()
315  for test_name in selected_tests:
316    if '.' in test_name:  # Has a test method
317      (test_class_name, test_name) = test_name.split('.', maxsplit=1)
318      if test_class_name not in test_class_name_to_tests:
319        # Never seen this class before
320        test_class_name_to_tests[test_class_name] = [test_name]
321      elif test_class_name_to_tests[test_class_name] is None:
322        # Already running all tests in this class, so ignore this extra
323        # test.
324        pass
325      else:
326        test_class_name_to_tests[test_class_name].append(test_name)
327    else:  # No test method; run all tests in this class.
328      test_class_name_to_tests[test_name] = None
329
330  # Now transform class names to class objects.
331  # Dict from test_name class name to instance.
332  class_name_to_class = {cls.__name__: cls for cls in test_classes}
333  for test_class_name, tests in test_class_name_to_tests.items():
334    test_class = class_name_to_class.get(test_class_name)
335    if not test_class:
336      raise Error('Unknown test_name class %s' % test_class_name)
337    class_to_tests[test_class] = tests
338
339  return class_to_tests
340