• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2017 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import json
7import os
8import sys
9
10import six
11
12import merge_api
13import results_merger
14
15
16def StandardIsolatedScriptMerge(output_json, summary_json, jsons_to_merge):
17  """Merge the contents of one or more results JSONs into a single JSON.
18
19  Args:
20    output_json: A path to a JSON file to which the merged results should be
21      written.
22    jsons_to_merge: A list of paths to JSON files that should be merged.
23  """
24  # summary.json is produced by swarming client itself. We are mostly interested
25  # in the number of shards.
26  try:
27    with open(summary_json) as f:
28      summary = json.load(f)
29  except (IOError, ValueError):
30    print(('summary.json is missing or can not be read',
31           'Something is seriously wrong with swarming client or the bot.'),
32          file=sys.stderr)
33    return 1
34
35  missing_shards = []
36  shard_results_list = []
37  for index, result in enumerate(summary['shards']):
38    output_path = None
39    if result:
40      output_path = find_shard_output_path(index, result.get('task_id'),
41                                           jsons_to_merge)
42    if not output_path:
43      missing_shards.append(index)
44      continue
45
46    with open(output_path) as f:
47      try:
48        json_contents = json.load(f)
49      except ValueError as e:
50        six.raise_from(ValueError('Failed to parse JSON from %s' % output_path),
51                       e)
52      shard_results_list.append(json_contents)
53
54  merged_results = results_merger.merge_test_results(shard_results_list)
55  if missing_shards:
56    merged_results['missing_shards'] = missing_shards
57    if 'global_tags' not in merged_results:
58      merged_results['global_tags'] = []
59    merged_results['global_tags'].append('UNRELIABLE_RESULTS')
60
61  with open(output_json, 'w') as f:
62    json.dump(merged_results, f)
63
64  return 0
65
66
67def find_shard_output_path(index, task_id, jsons_to_merge):
68  """Finds the shard matching the index/task-id.
69
70  Args:
71    index: The index of the shard to load data for, this is for old api.
72    task_id: The directory of the shard to load data for, this is for new api.
73    jsons_to_merge: A container of file paths for shards that emitted output.
74
75  Returns:
76    * The matching path, or None
77  """
78  # 'output.json' is set in swarming/api.py, gtest_task method.
79  matching_json_files = [
80      j for j in jsons_to_merge if (os.path.basename(j) == 'output.json' and (
81          os.path.basename(os.path.dirname(j)) == str(index)
82          or os.path.basename(os.path.dirname(j)) == task_id))
83  ]
84
85  if not matching_json_files:
86    print('shard %s test output missing' % index, file=sys.stderr)
87    return None
88  if len(matching_json_files) > 1:
89    print('duplicate test output for shard %s' % index, file=sys.stderr)
90    return None
91
92  return matching_json_files[0]
93
94
95def main(raw_args):
96  parser = merge_api.ArgumentParser()
97  args = parser.parse_args(raw_args)
98  return StandardIsolatedScriptMerge(args.output_json, args.summary_json,
99                                     args.jsons_to_merge)
100
101
102if __name__ == '__main__':
103  sys.exit(main(sys.argv[1:]))
104