1#!/usr/bin/python2 2 3# Copyright 2017 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Load generator for devserver.""" 8 9import argparse 10import itertools 11import json 12import re 13import sys 14 15import common 16 17 18# Default keys to skip displaying. 19DEFAULT_SKIP = [ 20 'build_name', 21 'devserver', 22 'name', 23 'parent', 24 'quick_provision', 25 'trigger_response', 26] 27 28# List of commandline arguments for easy filtering. 29FILTER_ARGS = [ 30 'board', 31 'build_name', 32 'devserver', 33 'name', 34 'status', 35] 36 37 38def get_parser(): 39 """Creates the argparse parser.""" 40 parser = argparse.ArgumentParser(description=__doc__) 41 parser.add_argument('infile', nargs='*', type=argparse.FileType('r'), 42 help='Path to JSON file to read.', 43 default=[sys.stdin]) 44 parser.add_argument('--boards', type=str, action='store', 45 help='Boards to show.') 46 parser.add_argument('--group', type=str, action='store', 47 help='Comma-spearated list of keys to group by.') 48 parser.add_argument('--dump', action='store_true', 49 help='Dump all filtered entries.') 50 parser.add_argument('--skip', type=str, action='store', 51 help='Comma-separated list of keys to skip displaying.', 52 default=','.join(DEFAULT_SKIP)) 53 parser.add_argument('--filter', type=str, action='store', 54 help='Filter expression to apply to each node.') 55 for arg in FILTER_ARGS: 56 parser.add_argument('--%s' % arg, type=str, action='store', 57 help='Comma-separated list of %s to filter by.' % 58 arg) 59 parser.add_argument('--no-summary', action='store_false', dest='summary', 60 help='Disable summary.') 61 62 return parser 63 64def summarize_entries(entries, skip=set()): 65 """Summarize a list of entries.""" 66 TAG_KEYS = [ 67 'board', 'build_name', 'devserver', 'name', 68 'parent', 'quick_provision', 'status' 69 ] 70 VALUE_KEYS = [ 71 'avg_active', 'elapsed', 72 ] 73 summary = { 74 'COUNT': len(entries), 75 } 76 summary.update({key: summarize_tags(entries, key) for key in TAG_KEYS 77 if key not in skip}) 78 summary.update({key: summarize_values(entries, key) for key in VALUE_KEYS 79 if key not in skip}) 80 return summary 81 82def summarize_tags(entries, key): 83 """Summarize all the different string values for a given key.""" 84 tags = {str(entry[key]) for entry in entries} 85 return list(tags) 86 87def summarize_values(entries, key): 88 """Summarize the numeric values for a given key.""" 89 if entries is None or len(entries) == 0: 90 return None 91 92 values = [entry[key] for entry in entries if key in entry] 93 summary = {} 94 num_values = len(values) 95 if num_values: 96 summary['min'] = min(values) 97 summary['max'] = max(values) 98 summary['avg'] = sum(values) / num_values 99 num_skipped = len(entries) - num_values 100 if num_skipped: 101 summary['num'] = num_values 102 summary['skipped'] = num_skipped 103 return summary 104 105def group_entries(keys, entries): 106 """Group entries based on different values of given keys. 107 108 @param keys: A list of keys to group by. 109 @param entries: A list of entries to split into groups. 110 111 @return A list of list of entries, where each list has a different key 112 value. 113 """ 114 if not keys: 115 return [entries] 116 117 # Divide the group based on the first key. 118 indexed = {} 119 for entry in entries: 120 value = str(entry[keys[0]]) 121 indexed.setdefault(value, []).append(entry) 122 groups = [indexed[value] for value in sorted(indexed.keys())] 123 124 # Recursively subdivide all the groups based on the rest of the keys. 125 subgroups = [] 126 for group in groups: 127 subgroups.extend(group_entries(keys[1:], group)) 128 return subgroups 129 130def main(argv): 131 """Load generator for a devserver.""" 132 parser = get_parser() 133 options = parser.parse_args(argv) 134 135 # Read entries from the specified file. 136 all_entries = [] 137 for f in options.infile: 138 all_entries.extend([json.loads(line) for line in f]) 139 140 # Filter entries: 141 # - Ignore non-provisions. 142 # - Filter via the specified FILTER_ARGS arguments. 143 # - Filter via explicit filter request. 144 entries = filter(lambda x: x['name'] != 'Runner', all_entries) 145 for arg in FILTER_ARGS: 146 if options.__dict__.get(arg): 147 entries = filter(lambda x: x[arg] in 148 options.__dict__[arg].split(','), 149 entries) 150 if options.filter: 151 entries = filter(lambda x: eval(options.filter, {'re': re}, x), entries) 152 153 # Group the entries based on specified keys. 154 groups = group_entries(options.group.split(',') if options.group else None, 155 entries) 156 157 # Dump all filtered entries as groups, including their parents. 158 if options.dump: 159 dump_entries = itertools.chain(*groups) 160 # Dump all entries, tracking needed parents. 161 parents = [] 162 for entry in dump_entries: 163 print(json.dumps(entry)) 164 if 'parent' in entry and entry['parent'] not in parents: 165 parents.append(entry['parent']) 166 # Dump all parents. 167 for entry in all_entries: 168 if entry['id'] in parents: 169 print(json.dumps(entry)) 170 171 # Summarize the entries, group by group. 172 if options.summary: 173 skip = options.skip.split(',') if options.skip else set() 174 summaries = [summarize_entries(group, skip) for group in groups] 175 print(json.dumps(summaries, indent=2)) 176 177if __name__ == '__main__': 178 sys.exit(main(sys.argv[1:])) 179