• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2017 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import argparse
7import json
8import os
9import shutil
10import sys
11import tempfile
12import traceback
13
14from common import gtest_utils
15from slave import annotation_utils
16from slave import slave_utils
17
18
19MISSING_SHARDS_MSG = r"""Missing results from the following shard(s): %s
20
21This can happen in following cases:
22  * Test failed to start (missing *.dll/*.so dependency for example)
23  * Test crashed or hung
24  * Task expired because there are not enough bots available and are all used
25  * Swarming service experienced problems
26
27Please examine logs to figure out what happened.
28"""
29
30
31def emit_warning(title, log=None):
32  print '@@@STEP_WARNINGS@@@'
33  print title
34  if log:
35    slave_utils.WriteLogLines(title, log.split('\n'))
36
37
38def merge_shard_results(summary_json, jsons_to_merge):
39  """Reads JSON test output from all shards and combines them into one.
40
41  Returns dict with merged test output on success or None on failure. Emits
42  annotations.
43  """
44  # summary.json is produced by swarming.py itself. We are mostly interested
45  # in the number of shards.
46  try:
47    with open(summary_json) as f:
48      summary = json.load(f)
49  except (IOError, ValueError):
50    emit_warning(
51        'summary.json is missing or can not be read',
52        'Something is seriously wrong with swarming_client/ or the bot.')
53    return None
54
55  # Merge all JSON files together. Keep track of missing shards.
56  merged = {
57    'all_tests': set(),
58    'disabled_tests': set(),
59    'global_tags': set(),
60    'missing_shards': [],
61    'per_iteration_data': [],
62    'swarming_summary': summary,
63  }
64  for index, result in enumerate(summary['shards']):
65    if result is not None:
66      # Author note: this code path doesn't trigger convert_to_old_format() in
67      # client/swarming.py, which means the state enum is saved in its string
68      # name form, not in the number form.
69      state = result.get('state')
70      if state == u'BOT_DIED':
71        emit_warning('Shard #%d had a Swarming internal failure' % index)
72      elif state == u'EXPIRED':
73        emit_warning('There wasn\'t enough capacity to run your test')
74      elif state == u'TIMED_OUT':
75        emit_warning(
76            'Test runtime exceeded allocated time',
77            'Either it ran for too long (hard timeout) or it didn\'t produce '
78            'I/O for an extended period of time (I/O timeout)')
79      elif state == u'COMPLETED':
80        json_data, err_msg = load_shard_json(index, jsons_to_merge)
81        if json_data:
82          # Set-like fields.
83          for key in ('all_tests', 'disabled_tests', 'global_tags'):
84            merged[key].update(json_data.get(key), [])
85
86          # 'per_iteration_data' is a list of dicts. Dicts should be merged
87          # together, not the 'per_iteration_data' list itself.
88          merged['per_iteration_data'] = merge_list_of_dicts(
89              merged['per_iteration_data'],
90              json_data.get('per_iteration_data', []))
91          continue
92        else:
93          emit_warning('Task ran but no result was found: %s' % err_msg)
94      else:
95        emit_warning('Invalid Swarming task state: %s' % state)
96    merged['missing_shards'].append(index)
97
98  # If some shards are missing, make it known. Continue parsing anyway. Step
99  # should be red anyway, since swarming.py return non-zero exit code in that
100  # case.
101  if merged['missing_shards']:
102    as_str = ', '.join(map(str, merged['missing_shards']))
103    emit_warning(
104        'some shards did not complete: %s' % as_str,
105        MISSING_SHARDS_MSG % as_str)
106    # Not all tests run, combined JSON summary can not be trusted.
107    merged['global_tags'].add('UNRELIABLE_RESULTS')
108
109  # Convert to jsonish dict.
110  for key in ('all_tests', 'disabled_tests', 'global_tags'):
111    merged[key] = sorted(merged[key])
112  return merged
113
114
115OUTPUT_JSON_SIZE_LIMIT = 100 * 1024 * 1024  # 100 MB
116
117
118def load_shard_json(index, jsons_to_merge):
119  """Reads JSON output of the specified shard.
120
121  Args:
122    output_dir: The directory in which to look for the JSON output to load.
123    index: The index of the shard to load data for.
124
125  Returns: A tuple containing:
126    * The contents of path, deserialized into a python object.
127    * An error string.
128    (exactly one of the tuple elements will be non-None).
129  """
130  # 'output.json' is set in swarming/api.py, gtest_task method.
131  matching_json_files = [
132      j for j in jsons_to_merge
133      if (os.path.basename(j) == 'output.json'
134          and os.path.basename(os.path.dirname(j)) == str(index))]
135
136  if not matching_json_files:
137    print >> sys.stderr, 'shard %s test output missing' % index
138    return (None, 'shard %s test output was missing' % index)
139  elif len(matching_json_files) > 1:
140    print >> sys.stderr, 'duplicate test output for shard %s' % index
141    return (None, 'shard %s test output was duplicated' % index)
142
143  path = matching_json_files[0]
144
145  try:
146    filesize = os.stat(path).st_size
147    if filesize > OUTPUT_JSON_SIZE_LIMIT:
148      print >> sys.stderr, 'output.json is %d bytes. Max size is %d' % (
149           filesize, OUTPUT_JSON_SIZE_LIMIT)
150      return (None, 'shard %s test output exceeded the size limit' % index)
151
152    with open(path) as f:
153      return (json.load(f), None)
154  except (IOError, ValueError, OSError) as e:
155    print >> sys.stderr, 'Missing or invalid gtest JSON file: %s' % path
156    print >> sys.stderr, '%s: %s' % (type(e).__name__, e)
157
158    return (None, 'shard %s test output was missing or invalid' % index)
159
160
161def merge_list_of_dicts(left, right):
162  """Merges dicts left[0] with right[0], left[1] with right[1], etc."""
163  output = []
164  for i in xrange(max(len(left), len(right))):
165    left_dict = left[i] if i < len(left) else {}
166    right_dict = right[i] if i < len(right) else {}
167    merged_dict = left_dict.copy()
168    merged_dict.update(right_dict)
169    output.append(merged_dict)
170  return output
171
172
173def standard_gtest_merge(
174    output_json, summary_json, jsons_to_merge):
175
176  output = merge_shard_results(summary_json, jsons_to_merge)
177  with open(output_json, 'wb') as f:
178    json.dump(output, f)
179
180  return 0
181
182
183def main(raw_args):
184
185  parser = argparse.ArgumentParser()
186  parser.add_argument('--build-properties')
187  parser.add_argument('--summary-json')
188  parser.add_argument('-o', '--output-json', required=True)
189  parser.add_argument('jsons_to_merge', nargs='*')
190
191  args = parser.parse_args(raw_args)
192
193  return standard_gtest_merge(
194      args.output_json, args.summary_json, args.jsons_to_merge)
195
196
197if __name__ == '__main__':
198  sys.exit(main(sys.argv[1:]))
199