• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2019, The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""ATest execution info generator."""
16
17
18from __future__ import print_function
19
20import argparse
21import glob
22import json
23import logging
24import os
25import pathlib
26import sys
27from typing import List
28
29from atest import atest_utils as au
30from atest import atest_utils
31from atest import constants
32from atest import feedback
33from atest.atest_enum import ExitCode
34from atest.logstorage import log_uploader
35from atest.metrics import metrics_utils
36
37_ARGS_KEY = 'args'
38_STATUS_PASSED_KEY = 'PASSED'
39_STATUS_FAILED_KEY = 'FAILED'
40_STATUS_IGNORED_KEY = 'IGNORED'
41_SUMMARY_KEY = 'summary'
42_TOTAL_SUMMARY_KEY = 'total_summary'
43_TEST_RUNNER_KEY = 'test_runner'
44_TEST_NAME_KEY = 'test_name'
45_TEST_TIME_KEY = 'test_time'
46_TEST_DETAILS_KEY = 'details'
47_TEST_RESULT_NAME = 'test_result'
48_TEST_RESULT_LINK = 'test_result_link'
49_EXIT_CODE_ATTR = 'EXIT_CODE'
50_MAIN_MODULE_KEY = '__main__'
51_UUID_LEN = 30
52_RESULT_LEN = 20
53_RESULT_URL_LEN = 35
54_COMMAND_LEN = 50
55_LOGCAT_FMT = '{}/log/invocation_*/{}*device_logcat_test*'
56
57_SUMMARY_MAP_TEMPLATE = {
58    _STATUS_PASSED_KEY: 0,
59    _STATUS_FAILED_KEY: 0,
60    _STATUS_IGNORED_KEY: 0,
61}
62
63PREPARE_END_TIME = None
64
65
66def preparation_time(start_time):
67  """Return the preparation time.
68
69  Args:
70      start_time: The time.
71
72  Returns:
73      The preparation time if PREPARE_END_TIME is set, None otherwise.
74  """
75  return PREPARE_END_TIME - start_time if PREPARE_END_TIME else None
76
77
78def symlink_latest_result(test_result_dir):
79  """Make the symbolic link to latest result.
80
81  Args:
82      test_result_dir: A string of the dir path.
83  """
84  symlink = os.path.join(constants.ATEST_RESULT_ROOT, 'LATEST')
85  if os.path.exists(symlink) or os.path.islink(symlink):
86    os.remove(symlink)
87  os.symlink(test_result_dir, symlink)
88
89
90def print_test_result(root, history_arg):
91  """Make a list of latest n test result.
92
93  Args:
94      root: A string of the test result root path.
95      history_arg: A string of an integer or uuid. If it's an integer string,
96        the number of lines of test result will be given; else it will be
97        treated a uuid and print test result accordingly in detail.
98  """
99  if not history_arg.isdigit():
100    path = os.path.join(constants.ATEST_RESULT_ROOT, history_arg, 'test_result')
101    print_test_result_by_path(path)
102    return
103  target = '%s/20*_*_*' % root
104  paths = glob.glob(target)
105  paths.sort(reverse=True)
106  if has_url_results():
107    print(
108        '{:-^{uuid_len}} {:-^{result_len}} {:-^{result_url_len}}'
109        ' {:-^{command_len}}'.format(
110            'uuid',
111            'result',
112            'result_url',
113            'command',
114            uuid_len=_UUID_LEN,
115            result_len=_RESULT_LEN,
116            result_url_len=_RESULT_URL_LEN,
117            command_len=_COMMAND_LEN,
118        )
119    )
120  else:
121    print(
122        '{:-^{uuid_len}} {:-^{result_len}} {:-^{command_len}}'.format(
123            'uuid',
124            'result',
125            'command',
126            uuid_len=_UUID_LEN,
127            result_len=_RESULT_LEN,
128            command_len=_COMMAND_LEN,
129        )
130    )
131  for path in paths[0 : int(history_arg) + 1]:
132    result_path = os.path.join(path, 'test_result')
133    result = au.load_json_safely(result_path)
134    total_summary = result.get(_TOTAL_SUMMARY_KEY, {})
135    summary_str = ', '.join(
136        [k[:1] + ':' + str(v) for k, v in total_summary.items()]
137    )
138    test_result_url = result.get(_TEST_RESULT_LINK, '')
139    if has_url_results():
140      print(
141          '{:<{uuid_len}} {:<{result_len}} '
142          '{:<{result_url_len}} atest {:<{command_len}}'.format(
143              os.path.basename(path),
144              summary_str,
145              test_result_url,
146              result.get(_ARGS_KEY, ''),
147              uuid_len=_UUID_LEN,
148              result_len=_RESULT_LEN,
149              result_url_len=_RESULT_URL_LEN,
150              command_len=_COMMAND_LEN,
151          )
152      )
153    else:
154      print(
155          '{:<{uuid_len}} {:<{result_len}} atest {:<{command_len}}'.format(
156              os.path.basename(path),
157              summary_str,
158              result.get(_ARGS_KEY, ''),
159              uuid_len=_UUID_LEN,
160              result_len=_RESULT_LEN,
161              command_len=_COMMAND_LEN,
162          )
163      )
164
165
166def print_test_result_by_path(path):
167  """Print latest test result.
168
169  Args:
170      path: A string of test result path.
171  """
172  result = au.load_json_safely(path)
173  if not result:
174    return
175  print('\natest {}'.format(result.get(_ARGS_KEY, '')))
176  test_result_url = result.get(_TEST_RESULT_LINK, '')
177  if test_result_url:
178    print('\nTest Result Link: {}'.format(test_result_url))
179  print('\nTotal Summary:\n{}'.format(au.delimiter('-')))
180  total_summary = result.get(_TOTAL_SUMMARY_KEY, {})
181  print(', '.join([(k + ':' + str(v)) for k, v in total_summary.items()]))
182  fail_num = total_summary.get(_STATUS_FAILED_KEY)
183  if fail_num > 0:
184    message = '%d test failed' % fail_num
185    print(f'\n{au.mark_red(message)}\n{"-" * len(message)}')
186    test_runner = result.get(_TEST_RUNNER_KEY, {})
187    for runner_name in test_runner.keys():
188      test_dict = test_runner.get(runner_name, {})
189      for test_name in test_dict:
190        test_details = test_dict.get(test_name, {})
191        for fail in test_details.get(_STATUS_FAILED_KEY):
192          print(au.mark_red(f'{fail.get(_TEST_NAME_KEY)}'))
193          failure_files = glob.glob(
194              _LOGCAT_FMT.format(
195                  os.path.dirname(path), fail.get(_TEST_NAME_KEY)
196              )
197          )
198          if failure_files:
199            print(
200                '{} {}'.format(
201                    au.mark_cyan('LOGCAT-ON-FAILURES:'), failure_files[0]
202                )
203            )
204          print(
205              '{} {}'.format(
206                  au.mark_cyan('STACKTRACE:\n'), fail.get(_TEST_DETAILS_KEY)
207              )
208          )
209
210
211def has_non_test_options(args: argparse.ArgumentParser):
212  """check whether non-test option in the args.
213
214  Args:
215      args: An argparse.ArgumentParser class instance holding parsed args.
216
217  Returns:
218      True, if args has at least one non-test option.
219      False, otherwise.
220  """
221  return (
222      args.collect_tests_only
223      or args.dry_run
224      or args.history
225      or args.version
226      or args.latest_result
227      or args.history
228  )
229
230
231def has_url_results():
232  """Get if contains url info."""
233  for root, _, files in os.walk(constants.ATEST_RESULT_ROOT):
234    for file in files:
235      if file != 'test_result':
236        continue
237      json_file = os.path.join(root, 'test_result')
238      result = au.load_json_safely(json_file)
239      url_link = result.get(_TEST_RESULT_LINK, '')
240      if url_link:
241        return True
242  return False
243
244
245class AtestExecutionInfo:
246  """Class that stores the whole test progress information in JSON format.
247
248  ----
249  For example, running command
250      atest hello_world_test HelloWorldTest
251
252  will result in storing the execution detail in JSON:
253  {
254    "args": "hello_world_test HelloWorldTest",
255    "test_runner": {
256        "AtestTradefedTestRunner": {
257            "hello_world_test": {
258                "FAILED": [
259                    {"test_time": "(5ms)",
260                     "details": "Hello, Wor...",
261                     "test_name": "HelloWorldTest#PrintHelloWorld"}
262                    ],
263                "summary": {"FAILED": 1, "PASSED": 0, "IGNORED": 0}
264            },
265            "HelloWorldTests": {
266                "PASSED": [
267                    {"test_time": "(27ms)",
268                     "details": null,
269                     "test_name": "...HelloWorldTest#testHalloWelt"},
270                    {"test_time": "(1ms)",
271                     "details": null,
272                     "test_name": "....HelloWorldTest#testHelloWorld"}
273                    ],
274                "summary": {"FAILED": 0, "PASSED": 2, "IGNORED": 0}
275            }
276        }
277    },
278    "total_summary": {"FAILED": 1, "PASSED": 2, "IGNORED": 0}
279  }
280  """
281
282  result_reporters = []
283
284  def __init__(
285      self, args: List[str], work_dir: str, args_ns: argparse.ArgumentParser
286  ):
287    """Initialise an AtestExecutionInfo instance.
288
289    Args:
290        args: Command line parameters.
291        work_dir: The directory for saving information.
292        args_ns: An argparse.ArgumentParser class instance holding parsed args.
293
294    Returns:
295           A json format string.
296    """
297    self.args = args
298    self.work_dir = work_dir
299    self.result_file_obj = None
300    self.args_ns = args_ns
301    self.test_result = os.path.join(self.work_dir, _TEST_RESULT_NAME)
302    logging.debug(
303        'A %s object is created with args %s, work_dir %s',
304        __class__,
305        args,
306        work_dir,
307    )
308
309  def __enter__(self):
310    """Create and return information file object."""
311    try:
312      self.result_file_obj = open(self.test_result, 'w')
313    except IOError:
314      atest_utils.print_and_log_error('Cannot open file %s', self.test_result)
315    return self.result_file_obj
316
317  def __exit__(self, exit_type, value, traceback):
318    """Write execution information and close information file."""
319    if self.result_file_obj and not has_non_test_options(self.args_ns):
320      self.result_file_obj.write(
321          AtestExecutionInfo._generate_execution_detail(self.args)
322      )
323      self.result_file_obj.close()
324      au.prompt_suggestions(self.test_result)
325      au.generate_print_result_html(self.test_result)
326      symlink_latest_result(self.work_dir)
327    main_module = sys.modules.get(_MAIN_MODULE_KEY)
328    main_exit_code = (
329        value.code
330        if isinstance(value, SystemExit)
331        else (getattr(main_module, _EXIT_CODE_ATTR, ExitCode.ERROR))
332    )
333    # Do not send stacktrace with send_exit_event when exit code is not
334    # ERROR.
335    if main_exit_code != ExitCode.ERROR:
336      logging.debug('send_exit_event:%s', main_exit_code)
337      metrics_utils.send_exit_event(main_exit_code)
338    else:
339      logging.debug('handle_exc_and_send_exit_event:%s', main_exit_code)
340      metrics_utils.handle_exc_and_send_exit_event(main_exit_code)
341
342    if log_uploader.is_uploading_logs():
343      log_uploader.upload_logs_detached(pathlib.Path(self.work_dir))
344    feedback.print_feedback_message()
345
346  @staticmethod
347  def _generate_execution_detail(args):
348    """Generate execution detail.
349
350    Args:
351        args: Command line parameters that you want to save.
352
353    Returns:
354        A json format string.
355    """
356    info_dict = {_ARGS_KEY: ' '.join(args)}
357    try:
358      AtestExecutionInfo._arrange_test_result(
359          info_dict, AtestExecutionInfo.result_reporters
360      )
361      return json.dumps(info_dict)
362    except ValueError as err:
363      atest_utils.print_and_log_warning(
364          'Parsing test result failed due to : %s', err
365      )
366    return {}
367
368  @staticmethod
369  def _arrange_test_result(info_dict, reporters):
370    """Append test result information in given dict.
371
372    Arrange test information to below
373    "test_runner": {
374        "test runner name": {
375            "test name": {
376                "FAILED": [
377                    {"test time": "",
378                     "details": "",
379                     "test name": ""}
380                ],
381            "summary": {"FAILED": 0, "PASSED": 0, "IGNORED": 0}
382            },
383        },
384    "total_summary": {"FAILED": 0, "PASSED": 0, "IGNORED": 0}
385
386    Args:
387        info_dict: A dict you want to add result information in.
388        reporters: A list of result_reporter.
389
390    Returns:
391        A dict contains test result information data.
392    """
393    info_dict[_TEST_RUNNER_KEY] = {}
394    for reporter in reporters:
395      if reporter.test_result_link:
396        info_dict[_TEST_RESULT_LINK] = reporter.test_result_link
397      for test in reporter.all_test_results:
398        runner = info_dict[_TEST_RUNNER_KEY].setdefault(test.runner_name, {})
399        group = runner.setdefault(test.group_name, {})
400        result_dict = {
401            _TEST_NAME_KEY: test.test_name,
402            _TEST_TIME_KEY: test.test_time,
403            _TEST_DETAILS_KEY: test.details,
404        }
405        group.setdefault(test.status, []).append(result_dict)
406
407    total_test_group_summary = _SUMMARY_MAP_TEMPLATE.copy()
408    for runner in info_dict[_TEST_RUNNER_KEY]:
409      for group in info_dict[_TEST_RUNNER_KEY][runner]:
410        group_summary = _SUMMARY_MAP_TEMPLATE.copy()
411        for status in info_dict[_TEST_RUNNER_KEY][runner][group]:
412          count = len(info_dict[_TEST_RUNNER_KEY][runner][group][status])
413          if status in _SUMMARY_MAP_TEMPLATE:
414            group_summary[status] = count
415            total_test_group_summary[status] += count
416        info_dict[_TEST_RUNNER_KEY][runner][group][_SUMMARY_KEY] = group_summary
417    info_dict[_TOTAL_SUMMARY_KEY] = total_test_group_summary
418    return info_dict
419