• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3# Copyright 2017 The Chromium Authors
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7
8
9import argparse
10import collections
11import contextlib
12import json
13import logging
14import tempfile
15import os
16import sys
17try:
18  from urllib.parse import urlencode
19  from urllib.request import urlopen
20except ImportError:
21  from urllib import urlencode
22  from urllib2 import urlopen
23
24
25CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
26BASE_DIR = os.path.abspath(os.path.join(
27    CURRENT_DIR, '..', '..', '..', '..', '..'))
28
29sys.path.append(os.path.join(BASE_DIR, 'build', 'android'))
30from pylib.results.presentation import standard_gtest_merge
31from pylib.utils import google_storage_helper  # pylint: disable=import-error
32
33sys.path.append(os.path.join(BASE_DIR, 'third_party'))
34import jinja2  # pylint: disable=import-error
35JINJA_ENVIRONMENT = jinja2.Environment(
36    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
37    autoescape=True)
38
39
40def cell(data, html_class='center'):
41  """Formats table cell data for processing in jinja template."""
42  return {
43    'data': data,
44    'class': html_class,
45  }
46
47
48def pre_cell(data, html_class='center'):
49  """Formats table <pre> cell data for processing in jinja template."""
50  return {
51    'cell_type': 'pre',
52    'data': data,
53    'class': html_class,
54  }
55
56
57class LinkTarget:
58  # Opens the linked document in a new window or tab.
59  NEW_TAB = '_blank'
60  # Opens the linked document in the same frame as it was clicked.
61  CURRENT_TAB = '_self'
62
63
64def link(data, href, target=LinkTarget.CURRENT_TAB):
65  """Formats <a> tag data for processing in jinja template.
66
67  Args:
68    data: String link appears as on HTML page.
69    href: URL where link goes.
70    target: Where link should be opened (e.g. current tab or new tab).
71  """
72  return {
73    'data': data,
74    'href': href,
75    'target': target,
76  }
77
78
79def links_cell(links, html_class='center', rowspan=None):
80  """Formats table cell with links for processing in jinja template.
81
82  Args:
83    links: List of link dictionaries. Use |link| function to generate them.
84    html_class: Class for table cell.
85    rowspan: Rowspan HTML attribute.
86  """
87  return {
88    'cell_type': 'links',
89    'class': html_class,
90    'links': links,
91    'rowspan': rowspan,
92  }
93
94
95def action_cell(action, data, html_class):
96  """Formats table cell with javascript actions.
97
98  Args:
99    action: Javscript action.
100    data: Data in cell.
101    class: Class for table cell.
102  """
103  return {
104    'cell_type': 'action',
105    'action': action,
106    'data': data,
107    'class': html_class,
108  }
109
110
111def flakiness_dashbord_link(test_name, suite_name, bucket):
112  # Assume the bucket will be like "foo-bar-baz", we will take "foo"
113  # as the test_project.
114  # Fallback to "chromium" if bucket is not passed, e.g. local_output=True
115  test_project = bucket.split('-')[0] if bucket else 'chromium'
116  query = '%s/%s' % (suite_name, test_name)
117  url_args = urlencode([('t', 'TESTS'), ('q', query), ('tp', test_project)])
118  return 'https://ci.chromium.org/ui/search?%s' % url_args
119
120
121def logs_cell(result, test_name, suite_name, bucket):
122  """Formats result logs data for processing in jinja template."""
123  link_list = []
124  result_link_dict = result.get('links', {})
125  result_link_dict['flakiness'] = flakiness_dashbord_link(
126      test_name, suite_name, bucket)
127  for name, href in sorted(result_link_dict.items()):
128    link_list.append(link(
129        data=name,
130        href=href,
131        target=LinkTarget.NEW_TAB))
132  if link_list:
133    return links_cell(link_list)
134  return cell('(no logs)')
135
136
137def code_search(test, cs_base_url):
138  """Returns URL for test on codesearch."""
139  search = test.replace('#', '.')
140  return '%s/search/?q=%s&type=cs' % (cs_base_url, search)
141
142
143def status_class(status):
144  """Returns HTML class for test status."""
145  if not status:
146    return 'failure unknown'
147  status = status.lower()
148  if status not in ('success', 'skipped'):
149    return 'failure %s' % status
150  return status
151
152
153def create_test_table(results_dict, cs_base_url, suite_name, bucket):
154  """Format test data for injecting into HTML table."""
155
156  header_row = [
157    cell(data='test_name', html_class='text'),
158    cell(data='status', html_class='flaky'),
159    cell(data='elapsed_time_ms', html_class='number'),
160    cell(data='logs', html_class='text'),
161    cell(data='output_snippet', html_class='text'),
162  ]
163
164  test_row_blocks = []
165  for test_name, test_results in results_dict.items():
166    test_runs = []
167    for index, result in enumerate(test_results):
168      if index == 0:
169        test_run = [links_cell(
170            links=[
171                link(href=code_search(test_name, cs_base_url),
172                     target=LinkTarget.NEW_TAB,
173                     data=test_name)],
174            rowspan=len(test_results),
175            html_class='left %s' % test_name
176        )]                                          # test_name
177      else:
178        test_run = []
179
180      test_run.extend([
181          cell(data=result['status'] or 'UNKNOWN',
182                                                    # status
183               html_class=('center %s' %
184                  status_class(result['status']))),
185          cell(data=result['elapsed_time_ms']),     # elapsed_time_ms
186          logs_cell(result, test_name, suite_name, bucket),
187                                                    # logs
188          pre_cell(data=result['output_snippet'],   # output_snippet
189                   html_class='left'),
190      ])
191      test_runs.append(test_run)
192    test_row_blocks.append(test_runs)
193  return header_row, test_row_blocks
194
195
196def create_suite_table(results_dict):
197  """Format test suite data for injecting into HTML table."""
198
199  SUCCESS_COUNT_INDEX = 1
200  FAIL_COUNT_INDEX = 2
201  ALL_COUNT_INDEX = 3
202  TIME_INDEX = 4
203
204  header_row = [
205    cell(data='suite_name', html_class='text'),
206    cell(data='number_success_tests', html_class='number'),
207    cell(data='number_fail_tests', html_class='number'),
208    cell(data='all_tests', html_class='number'),
209    cell(data='elapsed_time_ms', html_class='number'),
210  ]
211
212  footer_row = [
213    action_cell(
214          'showTestsOfOneSuiteOnlyWithNewState("TOTAL")',
215          'TOTAL',
216          'center'
217        ),         # TOTAL
218    cell(data=0),  # number_success_tests
219    cell(data=0),  # number_fail_tests
220    cell(data=0),  # all_tests
221    cell(data=0),  # elapsed_time_ms
222  ]
223
224  suite_row_dict = collections.defaultdict(lambda: [
225      # Note: |suite_name| will be given in the following for loop.
226      # It is not assigned yet here.
227      action_cell('showTestsOfOneSuiteOnlyWithNewState("%s")' % suite_name,
228                  suite_name, 'left'),  # suite_name
229      cell(data=0),  # number_success_tests
230      cell(data=0),  # number_fail_tests
231      cell(data=0),  # all_tests
232      cell(data=0),  # elapsed_time_ms
233  ])
234  for test_name, test_results in results_dict.items():
235    # TODO(mikecase): This logic doesn't work if there are multiple test runs.
236    # That is, if 'per_iteration_data' has multiple entries.
237    # Since we only care about the result of the last test run.
238    result = test_results[-1]
239
240    suite_name = (test_name.split('#')[0]
241                  if '#' in test_name else test_name.split('.')[0])
242    suite_row = suite_row_dict[suite_name]
243
244    suite_row[ALL_COUNT_INDEX]['data'] += 1
245    footer_row[ALL_COUNT_INDEX]['data'] += 1
246
247    if result['status'] == 'SUCCESS':
248      suite_row[SUCCESS_COUNT_INDEX]['data'] += 1
249      footer_row[SUCCESS_COUNT_INDEX]['data'] += 1
250    elif result['status'] != 'SKIPPED':
251      suite_row[FAIL_COUNT_INDEX]['data'] += 1
252      footer_row[FAIL_COUNT_INDEX]['data'] += 1
253
254    # Some types of crashes can have 'null' values for elapsed_time_ms.
255    if result['elapsed_time_ms'] is not None:
256      suite_row[TIME_INDEX]['data'] += result['elapsed_time_ms']
257      footer_row[TIME_INDEX]['data'] += result['elapsed_time_ms']
258
259  for suite in list(suite_row_dict.values()):
260    if suite[FAIL_COUNT_INDEX]['data'] > 0:
261      suite[FAIL_COUNT_INDEX]['class'] += ' failure'
262    else:
263      suite[FAIL_COUNT_INDEX]['class'] += ' success'
264
265  if footer_row[FAIL_COUNT_INDEX]['data'] > 0:
266    footer_row[FAIL_COUNT_INDEX]['class'] += ' failure'
267  else:
268    footer_row[FAIL_COUNT_INDEX]['class'] += ' success'
269
270  return (header_row, [[suite_row]
271                       for suite_row in list(suite_row_dict.values())],
272          footer_row)
273
274
275def feedback_url(result_details_link):
276  url_args = [
277      ('labels', 'Pri-2,Type-Bug,Restrict-View-Google'),
278      ('summary', 'Result Details Feedback:'),
279      ('components', 'Test>Android'),
280  ]
281  if result_details_link:
282    url_args.append(('comment', 'Please check out: %s' % result_details_link))
283  url_args = urlencode(url_args)
284  return 'https://bugs.chromium.org/p/chromium/issues/entry?%s' % url_args
285
286
287def results_to_html(results_dict, cs_base_url, bucket, test_name,
288                    builder_name, build_number, local_output):
289  """Convert list of test results into html format.
290
291  Args:
292    local_output: Whether this results file is uploaded to Google Storage or
293        just a local file.
294  """
295  test_rows_header, test_rows = create_test_table(
296      results_dict, cs_base_url, test_name, bucket)
297  suite_rows_header, suite_rows, suite_row_footer = create_suite_table(
298      results_dict)
299
300  suite_table_values = {
301    'table_id': 'suite-table',
302    'table_headers': suite_rows_header,
303    'table_row_blocks': suite_rows,
304    'table_footer': suite_row_footer,
305  }
306
307  test_table_values = {
308    'table_id': 'test-table',
309    'table_headers': test_rows_header,
310    'table_row_blocks': test_rows,
311  }
312
313  main_template = JINJA_ENVIRONMENT.get_template(
314      os.path.join('template', 'main.html'))
315
316  if local_output:
317    html_render = main_template.render(  #  pylint: disable=no-member
318        {
319          'tb_values': [suite_table_values, test_table_values],
320          'feedback_url': feedback_url(None),
321        })
322    return (html_render, None, None)
323  dest = google_storage_helper.unique_name(
324      '%s_%s_%s' % (test_name, builder_name, build_number))
325  result_details_link = google_storage_helper.get_url_link(
326      dest, '%s/html' % bucket)
327  html_render = main_template.render(  #  pylint: disable=no-member
328      {
329        'tb_values': [suite_table_values, test_table_values],
330        'feedback_url': feedback_url(result_details_link),
331      })
332  return (html_render, dest, result_details_link)
333
334
335def result_details(json_path, test_name, cs_base_url, bucket=None,
336                   builder_name=None, build_number=None, local_output=False):
337  """Get result details from json path and then convert results to html.
338
339  Args:
340    local_output: Whether this results file is uploaded to Google Storage or
341        just a local file.
342  """
343
344  with open(json_path) as json_file:
345    json_object = json.loads(json_file.read())
346
347  if not 'per_iteration_data' in json_object:
348    return 'Error: json file missing per_iteration_data.'
349
350  results_dict = collections.defaultdict(list)
351  for testsuite_run in json_object['per_iteration_data']:
352    for test, test_runs in testsuite_run.items():
353      results_dict[test].extend(test_runs)
354  return results_to_html(results_dict, cs_base_url, bucket, test_name,
355                         builder_name, build_number, local_output)
356
357
358def upload_to_google_bucket(html, bucket, dest):
359  with tempfile.NamedTemporaryFile(suffix='.html') as temp_file:
360    temp_file.write(html)
361    temp_file.flush()
362    return google_storage_helper.upload(
363        name=dest,
364        filepath=temp_file.name,
365        bucket='%s/html' % bucket,
366        content_type='text/html',
367        authenticated_link=True)
368
369
370def ui_screenshot_set(json_path):
371  with open(json_path) as json_file:
372    json_object = json.loads(json_file.read())
373  if not 'per_iteration_data' in json_object:
374    # This will be reported as an error by result_details, no need to duplicate.
375    return None
376  ui_screenshots = []
377  # pylint: disable=too-many-nested-blocks
378  for testsuite_run in json_object['per_iteration_data']:
379    for _, test_runs in testsuite_run.items():
380      for test_run in test_runs:
381        if 'ui screenshot' in test_run['links']:
382          screenshot_link = test_run['links']['ui screenshot']
383          if screenshot_link.startswith('file:'):
384            with contextlib.closing(urlopen(screenshot_link)) as f:
385              test_screenshots = json.load(f)
386          else:
387            # Assume anything that isn't a file link is a google storage link
388            screenshot_string = google_storage_helper.read_from_link(
389                screenshot_link)
390            if not screenshot_string:
391              logging.error('Bad screenshot link %s', screenshot_link)
392              continue
393            test_screenshots = json.loads(
394                screenshot_string)
395          ui_screenshots.extend(test_screenshots)
396  # pylint: enable=too-many-nested-blocks
397
398  if ui_screenshots:
399    return json.dumps(ui_screenshots)
400  return None
401
402
403def upload_screenshot_set(json_path, test_name, bucket, builder_name,
404                          build_number):
405  screenshot_set = ui_screenshot_set(json_path)
406  if not screenshot_set:
407    return None
408  dest = google_storage_helper.unique_name(
409    'screenshots_%s_%s_%s' % (test_name, builder_name, build_number),
410    suffix='.json')
411  with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as temp_file:
412    temp_file.write(screenshot_set)
413    temp_file.flush()
414    return google_storage_helper.upload(
415        name=dest,
416        filepath=temp_file.name,
417        bucket='%s/json' % bucket,
418        content_type='application/json',
419        authenticated_link=True)
420
421
422def main():
423  parser = argparse.ArgumentParser()
424  parser.add_argument('--json-file', help='Path of json file.')
425  parser.add_argument('--cs-base-url', help='Base url for code search.',
426                      default='http://cs.chromium.org')
427  parser.add_argument('--bucket', help='Google storage bucket.', required=True)
428  parser.add_argument('--builder-name', help='Builder name.')
429  parser.add_argument('--build-number', help='Build number.')
430  parser.add_argument('--test-name', help='The name of the test.',
431                      required=True)
432  parser.add_argument(
433      '-o', '--output-json',
434      help='(Swarming Merge Script API) '
435           'Output JSON file to create.')
436  parser.add_argument(
437      '--build-properties',
438      help='(Swarming Merge Script API) '
439           'Build property JSON file provided by recipes.')
440  parser.add_argument(
441      '--summary-json',
442      help='(Swarming Merge Script API) '
443           'Summary of shard state running on swarming. '
444           '(Output of the swarming.py collect '
445           '--task-summary-json=XXX command.)')
446  parser.add_argument(
447      '--task-output-dir',
448      help='(Swarming Merge Script API) '
449           'Directory containing all swarming task results.')
450  parser.add_argument(
451      'positional', nargs='*',
452      help='output.json from shards.')
453
454  args = parser.parse_args()
455
456  if ((args.build_properties is None) ==
457         (args.build_number is None or args.builder_name is None)):
458    raise parser.error('Exactly one of build_perperties or '
459                       '(build_number or builder_name) should be given.')
460
461  if (args.build_number is None) != (args.builder_name is None):
462    raise parser.error('args.build_number and args.builder_name '
463                       'has to be be given together'
464                       'or not given at all.')
465
466  if len(args.positional) == 0 and args.json_file is None:
467    if args.output_json:
468      with open(args.output_json, 'w') as f:
469        json.dump({}, f)
470    return
471  if len(args.positional) != 0 and args.json_file:
472    raise parser.error('Exactly one of args.positional and '
473                       'args.json_file should be given.')
474
475  if args.build_properties:
476    build_properties = json.loads(args.build_properties)
477    if ((not 'buildnumber' in build_properties) or
478        (not 'buildername' in build_properties)):
479      raise parser.error('Build number/builder name not specified.')
480    build_number = build_properties['buildnumber']
481    builder_name = build_properties['buildername']
482  elif args.build_number and args.builder_name:
483    build_number = args.build_number
484    builder_name = args.builder_name
485
486  if args.positional:
487    if len(args.positional) == 1:
488      json_file = args.positional[0]
489    else:
490      if args.output_json and args.summary_json:
491        standard_gtest_merge.standard_gtest_merge(
492            args.output_json, args.summary_json, args.positional)
493        json_file = args.output_json
494      elif not args.output_json:
495        raise Exception('output_json required by merge API is missing.')
496      else:
497        raise Exception('summary_json required by merge API is missing.')
498  elif args.json_file:
499    json_file = args.json_file
500
501  if not os.path.exists(json_file):
502    raise IOError('--json-file %s not found.' % json_file)
503
504  # Link to result details presentation page is a part of the page.
505  result_html_string, dest, result_details_link = result_details(
506      json_file, args.test_name, args.cs_base_url, args.bucket,
507      builder_name, build_number)
508
509  result_details_link_2 = upload_to_google_bucket(
510      result_html_string.encode('UTF-8'),
511      args.bucket, dest)
512  assert result_details_link == result_details_link_2, (
513      'Result details link do not match. The link returned by get_url_link'
514      ' should be the same as that returned by upload.')
515
516  ui_screenshot_set_link = upload_screenshot_set(json_file, args.test_name,
517      args.bucket, builder_name, build_number)
518
519  if ui_screenshot_set_link:
520    ui_catalog_url = 'https://chrome-ui-catalog.appspot.com/'
521    ui_catalog_query = urlencode({'screenshot_source': ui_screenshot_set_link})
522    ui_screenshot_link = '%s?%s' % (ui_catalog_url, ui_catalog_query)
523
524  if args.output_json:
525    with open(json_file) as original_json_file:
526      json_object = json.load(original_json_file)
527      json_object['links'] = {
528          'result_details (logcats, flakiness links)': result_details_link
529      }
530
531      if ui_screenshot_set_link:
532        json_object['links']['ui screenshots'] = ui_screenshot_link
533
534      with open(args.output_json, 'w') as f:
535        json.dump(json_object, f)
536  else:
537    print('Result Details: %s' % result_details_link)
538
539    if ui_screenshot_set_link:
540      print('UI Screenshots %s' % ui_screenshot_link)
541
542
543if __name__ == '__main__':
544  sys.exit(main())
545