• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import itertools
2import operator
3from django.db import models as dbmodels
4from autotest_lib.client.common_lib import priorities
5from autotest_lib.frontend.afe import rpc_utils, model_logic
6from autotest_lib.frontend.afe import models as afe_models, readonly_connection
7from autotest_lib.frontend.tko import models, tko_rpc_utils
8from autotest_lib.frontend.tko import preconfigs
9
10
11# table/spreadsheet view support
12
13def get_test_views(**filter_data):
14    return rpc_utils.prepare_for_serialization(
15        models.TestView.list_objects(filter_data))
16
17
18def get_num_test_views(**filter_data):
19    return models.TestView.query_count(filter_data)
20
21
22def get_group_counts(group_by, header_groups=None, fixed_headers=None,
23                     extra_select_fields=None, **filter_data):
24    """
25    Queries against TestView grouping by the specified fields and computings
26    counts for each group.
27    * group_by should be a list of field names.
28    * extra_select_fields can be used to specify additional fields to select
29      (usually for aggregate functions).
30    * header_groups can be used to get lists of unique combinations of group
31      fields.  It should be a list of tuples of fields from group_by.  It's
32      primarily for use by the spreadsheet view.
33    * fixed_headers can map header fields to lists of values.  the header will
34      guaranteed to return exactly those value.  this does not work together
35      with header_groups.
36
37    Returns a dictionary with two keys:
38    * header_values contains a list of lists, one for each header group in
39      header_groups.  Each list contains all the values for the corresponding
40      header group as tuples.
41    * groups contains a list of dicts, one for each row.  Each dict contains
42      keys for each of the group_by fields, plus a 'group_count' key for the
43      total count in the group, plus keys for each of the extra_select_fields.
44      The keys for the extra_select_fields are determined by the "AS" alias of
45      the field.
46    """
47    query = models.TestView.objects.get_query_set_with_joins(filter_data)
48    # don't apply presentation yet, since we have extra selects to apply
49    query = models.TestView.query_objects(filter_data, initial_query=query,
50                                          apply_presentation=False)
51    count_alias, count_sql = models.TestView.objects.get_count_sql(query)
52    query = query.extra(select={count_alias: count_sql})
53    if extra_select_fields:
54        query = query.extra(select=extra_select_fields)
55    query = models.TestView.apply_presentation(query, filter_data)
56
57    group_processor = tko_rpc_utils.GroupDataProcessor(query, group_by,
58                                                       header_groups or [],
59                                                       fixed_headers or {})
60    group_processor.process_group_dicts()
61    return rpc_utils.prepare_for_serialization(group_processor.get_info_dict())
62
63
64def get_num_groups(group_by, **filter_data):
65    """
66    Gets the count of unique groups with the given grouping fields.
67    """
68    query = models.TestView.objects.get_query_set_with_joins(filter_data)
69    query = models.TestView.query_objects(filter_data, initial_query=query)
70    return models.TestView.objects.get_num_groups(query, group_by)
71
72
73def get_status_counts(group_by, header_groups=[], fixed_headers={},
74                      **filter_data):
75    """
76    Like get_group_counts, but also computes counts of passed, complete (and
77    valid), and incomplete tests, stored in keys "pass_count', 'complete_count',
78    and 'incomplete_count', respectively.
79    """
80    return get_group_counts(group_by, header_groups=header_groups,
81                            fixed_headers=fixed_headers,
82                            extra_select_fields=tko_rpc_utils.STATUS_FIELDS,
83                            **filter_data)
84
85
86def get_latest_tests(group_by, header_groups=[], fixed_headers={},
87                     extra_info=[], **filter_data):
88    """
89    Similar to get_status_counts, but return only the latest test result per
90    group.  It still returns the same information (i.e. with pass count etc.)
91    for compatibility.  It includes an additional field "test_idx" with each
92    group.
93    @param extra_info a list containing the field names that should be returned
94                      with each cell. The fields are returned in the extra_info
95                      field of the return dictionary.
96    """
97    # find latest test per group
98    initial_query = models.TestView.objects.get_query_set_with_joins(
99            filter_data)
100    query = models.TestView.query_objects(filter_data,
101                                          initial_query=initial_query,
102                                          apply_presentation=False)
103    query = query.exclude(status__in=tko_rpc_utils._INVALID_STATUSES)
104    query = query.extra(
105            select={'latest_test_idx' : 'MAX(%s)' %
106                    models.TestView.objects.get_key_on_this_table('test_idx')})
107    query = models.TestView.apply_presentation(query, filter_data)
108
109    group_processor = tko_rpc_utils.GroupDataProcessor(query, group_by,
110                                                       header_groups,
111                                                       fixed_headers)
112    group_processor.process_group_dicts()
113    info = group_processor.get_info_dict()
114
115    # fetch full info for these tests so we can access their statuses
116    all_test_ids = [group['latest_test_idx'] for group in info['groups']]
117    test_views = initial_query.in_bulk(all_test_ids)
118
119    for group_dict in info['groups']:
120        test_idx = group_dict.pop('latest_test_idx')
121        group_dict['test_idx'] = test_idx
122        test_view = test_views[test_idx]
123
124        tko_rpc_utils.add_status_counts(group_dict, test_view.status)
125        group_dict['extra_info'] = []
126        for field in extra_info:
127            group_dict['extra_info'].append(getattr(test_view, field))
128
129    return rpc_utils.prepare_for_serialization(info)
130
131
132def get_job_ids(**filter_data):
133    """
134    Returns AFE job IDs for all tests matching the filters.
135    """
136    query = models.TestView.query_objects(filter_data)
137    job_ids = set()
138    for test_view in query.values('job_tag').distinct():
139        # extract job ID from tag
140        first_tag_component = test_view['job_tag'].split('-')[0]
141        try:
142            job_id = int(first_tag_component)
143            job_ids.add(job_id)
144        except ValueError:
145            # a nonstandard job tag, i.e. from contributed results
146            pass
147    return list(job_ids)
148
149
150# test detail view
151
152def _attributes_to_dict(attribute_list):
153    return dict((attribute.attribute, attribute.value)
154                for attribute in attribute_list)
155
156
157def _iteration_attributes_to_dict(attribute_list):
158    iter_keyfunc = operator.attrgetter('iteration')
159    attribute_list.sort(key=iter_keyfunc)
160    iterations = {}
161    for key, group in itertools.groupby(attribute_list, iter_keyfunc):
162        iterations[key] = _attributes_to_dict(group)
163    return iterations
164
165
166def _format_iteration_keyvals(test):
167    iteration_attr = _iteration_attributes_to_dict(test.iteration_attributes)
168    iteration_perf = _iteration_attributes_to_dict(test.iteration_results)
169
170    all_iterations = iteration_attr.keys() + iteration_perf.keys()
171    max_iterations = max(all_iterations + [0])
172
173    # merge the iterations into a single list of attr & perf dicts
174    return [{'attr': iteration_attr.get(index, {}),
175             'perf': iteration_perf.get(index, {})}
176            for index in xrange(1, max_iterations + 1)]
177
178
179def _job_keyvals_to_dict(keyvals):
180    return dict((keyval.key, keyval.value) for keyval in keyvals)
181
182
183def get_detailed_test_views(**filter_data):
184    test_views = models.TestView.list_objects(filter_data)
185
186    tests_by_id = models.Test.objects.in_bulk([test_view['test_idx']
187                                               for test_view in test_views])
188    tests = tests_by_id.values()
189    models.Test.objects.populate_relationships(tests, models.TestAttribute,
190                                               'attributes')
191    models.Test.objects.populate_relationships(tests, models.IterationAttribute,
192                                               'iteration_attributes')
193    models.Test.objects.populate_relationships(tests, models.IterationResult,
194                                               'iteration_results')
195    models.Test.objects.populate_relationships(tests, models.TestLabel,
196                                               'labels')
197
198    jobs_by_id = models.Job.objects.in_bulk([test_view['job_idx']
199                                             for test_view in test_views])
200    jobs = jobs_by_id.values()
201    models.Job.objects.populate_relationships(jobs, models.JobKeyval,
202                                              'keyvals')
203
204    for test_view in test_views:
205        test = tests_by_id[test_view['test_idx']]
206        test_view['attributes'] = _attributes_to_dict(test.attributes)
207        test_view['iterations'] = _format_iteration_keyvals(test)
208        test_view['labels'] = [label.name for label in test.labels]
209
210        job = jobs_by_id[test_view['job_idx']]
211        test_view['job_keyvals'] = _job_keyvals_to_dict(job.keyvals)
212
213    return rpc_utils.prepare_for_serialization(test_views)
214
215
216def get_tests_summary(job_names):
217    """
218    Gets the count summary of all passed and failed tests per suite.
219    @param job_names: Names of the suite jobs to get the summary from.
220    @returns: A summary of all the passed and failed tests per suite job.
221    """
222    # Take advantage of Django's literal escaping to prevent SQL injection
223    sql_list = ','.join(['%s'] * len(job_names))
224    query = ('''SELECT job_name, IF (status = 'GOOD', status, 'FAIL')
225                   AS test_status, COUNT(*) num
226                 FROM tko_test_view_2
227                 WHERE job_name IN (%s)
228                   AND test_name <> 'SERVER_JOB'
229                   AND test_name NOT LIKE 'CLIENT_JOB%%%%'
230                   AND status <> 'TEST_NA'
231                 GROUP BY job_name, IF (status = 'GOOD', status, 'FAIL')'''
232            % sql_list)
233
234    cursor = readonly_connection.cursor()
235    cursor.execute(query, job_names)
236    results = rpc_utils.fetchall_as_list_of_dicts(cursor)
237
238    summaries = {}
239    for result in results:
240        status = 'passed' if result['test_status'] == 'GOOD' else 'failed'
241        summary = summaries.setdefault(result['job_name'], {})
242        summary[status] = result['num']
243
244    return summaries
245
246
247def get_tests_summary_with_wildcards(job_names):
248    """
249    Like get_tests_summary(job_names) but allowing wildcards.
250    @param job_names: Names of the suite jobs to get the summary from.
251    @returns: A summary of all the passed and failed tests per suite job.
252    """
253    query = '''SELECT IF (status = 'GOOD', status, 'FAIL')
254                   AS test_status, COUNT(*) num
255                 FROM tko_test_view_2
256                 WHERE job_name LIKE %s
257                   AND test_name <> 'SERVER_JOB'
258                   AND test_name NOT LIKE 'CLIENT_JOB%%'
259                   AND status <> 'TEST_NA'
260                 GROUP BY IF (status = 'GOOD', status, 'FAIL')'''
261
262    summaries = {}
263    cursor = readonly_connection.cursor()
264    for job_name in job_names:
265        cursor.execute(query, job_name)
266        results = rpc_utils.fetchall_as_list_of_dicts(cursor)
267        summary = summaries.setdefault(job_name, {})
268        for result in results:
269            status = 'passed' if result['test_status'] == 'GOOD' else 'failed'
270            summary[status] = result['num']
271
272    return summaries
273
274
275# graphing view support
276
277def get_hosts_and_tests():
278    """\
279    Gets every host that has had a benchmark run on it. Additionally, also
280    gets a dictionary mapping the host names to the benchmarks.
281    """
282
283    host_info = {}
284    q = (dbmodels.Q(test_name__startswith='kernbench') |
285         dbmodels.Q(test_name__startswith='dbench') |
286         dbmodels.Q(test_name__startswith='tbench') |
287         dbmodels.Q(test_name__startswith='unixbench') |
288         dbmodels.Q(test_name__startswith='iozone'))
289    test_query = models.TestView.objects.filter(q).values(
290        'test_name', 'hostname', 'machine_idx').distinct()
291    for result_dict in test_query:
292        hostname = result_dict['hostname']
293        test = result_dict['test_name']
294        machine_idx = result_dict['machine_idx']
295        host_info.setdefault(hostname, {})
296        host_info[hostname].setdefault('tests', [])
297        host_info[hostname]['tests'].append(test)
298        host_info[hostname]['id'] = machine_idx
299    return rpc_utils.prepare_for_serialization(host_info)
300
301
302def create_metrics_plot(queries, plot, invert, drilldown_callback,
303                        normalize=None):
304    raise NotImplementedError()
305
306def create_qual_histogram(query, filter_string, interval, drilldown_callback):
307    raise NotImplementedError()
308
309
310# TODO(showard) - this extremely generic RPC is used only by one place in the
311# client.  We should come up with a more opaque RPC for that place to call and
312# get rid of this.
313def execute_query_with_param(query, param):
314    cursor = readonly_connection.cursor()
315    cursor.execute(query, param)
316    return cursor.fetchall()
317
318
319def get_preconfig(name, type):
320    return preconfigs.manager.get_preconfig(name, type)
321
322
323def get_embedding_id(url_token, graph_type, params):
324    raise NotImplementedError()
325
326
327def get_embedded_query_url_token(id):
328    raise NotImplementedError()
329
330
331# test label management
332
333def add_test_label(name, description=None):
334    return models.TestLabel.add_object(name=name, description=description).id
335
336
337def modify_test_label(label_id, **data):
338    models.TestLabel.smart_get(label_id).update_object(data)
339
340
341def delete_test_label(label_id):
342    models.TestLabel.smart_get(label_id).delete()
343
344
345def get_test_labels(**filter_data):
346    return rpc_utils.prepare_for_serialization(
347        models.TestLabel.list_objects(filter_data))
348
349
350def get_test_labels_for_tests(**test_filter_data):
351    label_ids = models.TestView.objects.query_test_label_ids(test_filter_data)
352    labels = models.TestLabel.list_objects({'id__in' : label_ids})
353    return rpc_utils.prepare_for_serialization(labels)
354
355
356def test_label_add_tests(label_id, **test_filter_data):
357    test_ids = models.TestView.objects.query_test_ids(test_filter_data)
358    models.TestLabel.smart_get(label_id).tests.add(*test_ids)
359
360
361def test_label_remove_tests(label_id, **test_filter_data):
362    label = models.TestLabel.smart_get(label_id)
363
364    # only include tests that actually have this label
365    extra_where = test_filter_data.get('extra_where', '')
366    if extra_where:
367        extra_where = '(' + extra_where + ') AND '
368    extra_where += 'tko_test_labels.id = %s' % label.id
369    test_filter_data['extra_where'] = extra_where
370    test_ids = models.TestView.objects.query_test_ids(test_filter_data)
371
372    label.tests.remove(*test_ids)
373
374
375# user-created test attributes
376
377def set_test_attribute(attribute, value, **test_filter_data):
378    """
379    * attribute - string name of attribute
380    * value - string, or None to delete an attribute
381    * test_filter_data - filter data to apply to TestView to choose tests to act
382      upon
383    """
384    assert test_filter_data # disallow accidental actions on all hosts
385    test_ids = models.TestView.objects.query_test_ids(test_filter_data)
386    tests = models.Test.objects.in_bulk(test_ids)
387
388    for test in tests.itervalues():
389        test.set_or_delete_attribute(attribute, value)
390
391
392# saved queries
393
394def get_saved_queries(**filter_data):
395    return rpc_utils.prepare_for_serialization(
396        models.SavedQuery.list_objects(filter_data))
397
398
399def add_saved_query(name, url_token):
400    name = name.strip()
401    owner = afe_models.User.current_user().login
402    existing_list = list(models.SavedQuery.objects.filter(owner=owner,
403                                                          name=name))
404    if existing_list:
405        query_object = existing_list[0]
406        query_object.url_token = url_token
407        query_object.save()
408        return query_object.id
409
410    return models.SavedQuery.add_object(owner=owner, name=name,
411                                        url_token=url_token).id
412
413
414def delete_saved_queries(id_list):
415    user = afe_models.User.current_user().login
416    query = models.SavedQuery.objects.filter(id__in=id_list, owner=user)
417    if query.count() == 0:
418        raise model_logic.ValidationError('No such queries found for this user')
419    query.delete()
420
421
422# other
423def get_motd():
424    return rpc_utils.get_motd()
425
426
427def get_static_data():
428    result = {}
429    group_fields = []
430    for field in models.TestView.group_fields:
431        if field in models.TestView.extra_fields:
432            name = models.TestView.extra_fields[field]
433        else:
434            name = models.TestView.get_field_dict()[field].verbose_name
435        group_fields.append((name.capitalize(), field))
436    model_fields = [(field.verbose_name.capitalize(), field.column)
437                    for field in models.TestView._meta.fields]
438    extra_fields = [(field_name.capitalize(), field_sql)
439                    for field_sql, field_name
440                    in models.TestView.extra_fields.iteritems()]
441
442    benchmark_key = {
443        'kernbench' : 'elapsed',
444        'dbench' : 'throughput',
445        'tbench' : 'throughput',
446        'unixbench' : 'score',
447        'iozone' : '32768-4096-fwrite'
448    }
449
450    tko_perf_view = [
451        ['Test Index', 'test_idx'],
452        ['Job Index', 'job_idx'],
453        ['Test Name', 'test_name'],
454        ['Subdirectory', 'subdir'],
455        ['Kernel Index', 'kernel_idx'],
456        ['Status Index', 'status_idx'],
457        ['Reason', 'reason'],
458        ['Host Index', 'machine_idx'],
459        ['Test Started Time', 'test_started_time'],
460        ['Test Finished Time', 'test_finished_time'],
461        ['Job Tag', 'job_tag'],
462        ['Job Name', 'job_name'],
463        ['Owner', 'job_owner'],
464        ['Job Queued Time', 'job_queued_time'],
465        ['Job Started Time', 'job_started_time'],
466        ['Job Finished Time', 'job_finished_time'],
467        ['Hostname', 'hostname'],
468        ['Platform', 'platform'],
469        ['Machine Owner', 'machine_owner'],
470        ['Kernel Hash', 'kernel_hash'],
471        ['Kernel Base', 'kernel_base'],
472        ['Kernel', 'kernel'],
473        ['Status', 'status'],
474        ['Iteration Number', 'iteration'],
475        ['Performance Keyval (Key)', 'iteration_key'],
476        ['Performance Keyval (Value)', 'iteration_value'],
477    ]
478
479    result['priorities'] = priorities.Priority.choices()
480    result['group_fields'] = sorted(group_fields)
481    result['all_fields'] = sorted(model_fields + extra_fields)
482    result['test_labels'] = get_test_labels(sort_by=['name'])
483    result['current_user'] = rpc_utils.prepare_for_serialization(
484            afe_models.User.current_user().get_object_dict())
485    result['benchmark_key'] = benchmark_key
486    result['tko_perf_view'] = tko_perf_view
487    result['tko_test_view'] = model_fields
488    result['preconfigs'] = preconfigs.manager.all_preconfigs()
489    result['motd'] = rpc_utils.get_motd()
490
491    return result
492