• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2# Copyright 2015 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6# pylint: disable-msg=W0311
7
8from collections import namedtuple
9import argparse
10import glob
11import json
12import os
13import pprint
14import re
15import subprocess
16
17_EXPECTATIONS_DIR = 'expectations'
18_AUTOTEST_RESULT_ID_TEMPLATE = 'gs://chromeos-autotest-results/%s-chromeos-test/chromeos*/graphics_dEQP/debug/graphics_dEQP.DEBUG'
19#_AUTOTEST_RESULT_TAG_TEMPLATE = 'gs://chromeos-autotest-results/%s/graphics_dEQP/debug/graphics_dEQP.DEBUG'
20_AUTOTEST_RESULT_TAG_TEMPLATE = 'gs://chromeos-autotest-results/%s/debug/client.0.DEBUG'
21# Use this template for tryjob results:
22#_AUTOTEST_RESULT_TEMPLATE = 'gs://chromeos-autotest-results/%s-ihf/*/graphics_dEQP/debug/graphics_dEQP.DEBUG'
23_BOARD_REGEX = re.compile(r'ChromeOS BOARD = (.+)')
24_CPU_FAMILY_REGEX = re.compile(r'ChromeOS CPU family = (.+)')
25_GPU_FAMILY_REGEX = re.compile(r'ChromeOS GPU family = (.+)')
26_TEST_FILTER_REGEX = re.compile(r'dEQP test filter = (.+)')
27_HASTY_MODE_REGEX = re.compile(r'\'hasty\': \'True\'|Running in hasty mode.')
28
29#04/23 07:30:21.624 INFO |graphics_d:0240| TestCase: dEQP-GLES3.functional.shaders.operator.unary_operator.bitwise_not.highp_ivec3_vertex
30#04/23 07:30:21.840 INFO |graphics_d:0261| Result: Pass
31_TEST_RESULT_REGEX = re.compile(r'TestCase: (.+?)$\n.+? Result: (.+?)$',
32                                re.MULTILINE)
33_HASTY_TEST_RESULT_REGEX = re.compile(
34    r'\[stdout\] Test case \'(.+?)\'..$\n'
35    r'.+?\[stdout\]   (Pass|NotSupported|QualityWarning|CompatibilityWarning|'
36    r'Fail|ResourceError|Crash|Timeout|InternalError|Skipped) \((.+)\)', re.MULTILINE)
37Logfile = namedtuple('Logfile', 'job_id name gs_path')
38
39
40def execute(cmd_list):
41  sproc = subprocess.Popen(cmd_list, stdout=subprocess.PIPE)
42  return sproc.communicate()[0]
43
44
45def get_metadata(s):
46  cpu = re.search(_CPU_FAMILY_REGEX, s).group(1)
47  gpu = re.search(_GPU_FAMILY_REGEX, s).group(1)
48  board = re.search(_BOARD_REGEX, s).group(1)
49  filter = re.search(_TEST_FILTER_REGEX, s).group(1)
50  hasty = False
51  if re.search(_HASTY_MODE_REGEX, s):
52    hasty = True
53  print('Found results from %s for GPU = %s, filter = %s and hasty = %r.' %
54        (board, gpu, filter, hasty))
55  return board, gpu, filter, hasty
56
57
58def copy_logs_from_gs_path(autotest_result_path):
59  logs = []
60  gs_paths = execute(['gsutil', 'ls', autotest_result_path]).splitlines()
61  for gs_path in gs_paths:
62    job_id = gs_path.split('/')[3].split('-')[0]
63    # DEBUG logs have more information than INFO logs, especially for hasty.
64    name = os.path.join('logs', job_id + '_graphics_dEQP.DEBUG')
65    logs.append(Logfile(job_id, name, gs_path))
66  for log in logs:
67    execute(['gsutil', 'cp', log.gs_path, log.name])
68  return logs
69
70
71def get_local_logs():
72  logs = []
73  for name in glob.glob(os.path.join('logs', '*_graphics_dEQP.INFO')):
74    job_id = name.split('_')[0]
75    logs.append(Logfile(job_id, name, name))
76  for name in glob.glob(os.path.join('logs', '*_graphics_dEQP.DEBUG')):
77    job_id = name.split('_')[0]
78    logs.append(Logfile(job_id, name, name))
79  return logs
80
81
82def get_all_tests(text):
83  tests = []
84  for test, result in re.findall(_TEST_RESULT_REGEX, text):
85    tests.append((test, result))
86  for test, result, details in re.findall(_HASTY_TEST_RESULT_REGEX, text):
87    tests.append((test, result))
88  return tests
89
90
91def get_not_passing_tests(text):
92  not_passing = []
93  for test, result in re.findall(_TEST_RESULT_REGEX, text):
94    if not (result == 'Pass' or result == 'NotSupported' or result == 'Skipped' or
95            result == 'QualityWarning' or result == 'CompatibilityWarning'):
96      not_passing.append((test, result))
97  for test, result, details in re.findall(_HASTY_TEST_RESULT_REGEX, text):
98    if result != 'Pass':
99      not_passing.append((test, result))
100  return not_passing
101
102
103def load_expectation_dict(json_file):
104  data = {}
105  if os.path.isfile(json_file):
106    print 'Loading file ' + json_file
107    with open(json_file, 'r') as f:
108      text = f.read()
109      data = json.loads(text)
110  return data
111
112
113def load_expectations(json_file):
114  data = load_expectation_dict(json_file)
115  expectations = {}
116  # Convert from dictionary of lists to dictionary of sets.
117  for key in data:
118    expectations[key] = set(data[key])
119  return expectations
120
121
122def expectation_list_to_dict(tests):
123  data = {}
124  tests = list(set(tests))
125  for test, result in tests:
126    if data.has_key(result):
127      new_list = list(set(data[result].append(test)))
128      data.pop(result)
129      data[result] = new_list
130    else:
131      data[result] = [test]
132  return data
133
134
135def save_expectation_dict(expectation_path, expectation_dict):
136  # Clean up obsolete expectations.
137  for file_name in glob.glob(expectation_path + '.*'):
138    if not '.hasty.' in file_name or '.hasty' in expectation_path:
139      os.remove(file_name)
140  # Dump json for next iteration.
141  with open(expectation_path + '.json', 'w') as f:
142    json.dump(expectation_dict,
143              f,
144              sort_keys=True,
145              indent=4,
146              separators=(',', ': '))
147  # Dump plain text for autotest.
148  for key in expectation_dict:
149    if expectation_dict[key]:
150      with open(expectation_path + '.' + key, 'w') as f:
151        for test in expectation_dict[key]:
152          f.write(test)
153          f.write('\n')
154
155
156# Figure out duplicates and move them to Flaky result set/list.
157def process_flaky(status_dict):
158  """Figure out duplicates and move them to Flaky result set/list."""
159  clean_dict = {}
160  flaky = set([])
161  if status_dict.has_key('Flaky'):
162    flaky = status_dict['Flaky']
163
164  # FLaky tests are tests with 2 distinct results.
165  for key1 in status_dict.keys():
166    for key2 in status_dict.keys():
167      if key1 != key2:
168        flaky |= status_dict[key1] & status_dict[key2]
169
170  # Remove Flaky tests from other status and convert to dict of list.
171  for key in status_dict.keys():
172    if key != 'Flaky':
173      not_flaky = list(status_dict[key] - flaky)
174      not_flaky.sort()
175      print 'Number of "%s" is %d.' % (key, len(not_flaky))
176      clean_dict[key] = not_flaky
177
178  # And finally process flaky list/set.
179  flaky_list = list(flaky)
180  flaky_list.sort()
181  clean_dict['Flaky'] = flaky_list
182
183  return clean_dict
184
185
186def merge_expectation_list(expectation_path, tests):
187  status_dict = {}
188  expectation_json = expectation_path + '.json'
189  if os.access(expectation_json, os.R_OK):
190    status_dict = load_expectations(expectation_json)
191  else:
192    print 'Could not load', expectation_json
193  for test, result in tests:
194    if status_dict.has_key(result):
195      new_set = status_dict[result]
196      new_set.add(test)
197      status_dict.pop(result)
198      status_dict[result] = new_set
199    else:
200      status_dict[result] = set([test])
201  clean_dict = process_flaky(status_dict)
202  save_expectation_dict(expectation_path, clean_dict)
203
204
205def load_log(name):
206  """Load test log and clean it from stderr spew."""
207  with open(name) as f:
208    lines = f.read().splitlines()
209  text = ''
210  for line in lines:
211    if ('dEQP test filter =' in line or 'ChromeOS BOARD = ' in line or
212        'ChromeOS CPU family =' in line or 'ChromeOS GPU family =' in line or
213        'TestCase: ' in line or 'Result: ' in line or
214        'Test Options: ' in line or 'Running in hasty mode.' in line or
215        # For hasty logs we have:
216        'Pass (' in line or 'NotSupported (' in line or 'Skipped (' in line or
217        'QualityWarning (' in line or 'CompatibilityWarning (' in line or
218        'Fail (' in line or 'ResourceError (' in line or 'Crash (' in line or
219        'Timeout (' in line or 'InternalError (' in line or
220        ' Test case \'' in line):
221      text += line + '\n'
222  # TODO(ihf): Warn about or reject log files missing the end marker.
223  return text
224
225
226def all_passing(tests):
227  for _, result in tests:
228    if not (result == 'Pass'):
229      return False
230  return True
231
232
233def process_logs(logs):
234  for log in logs:
235    text = load_log(log.name)
236    if text:
237      print '================================================================'
238      print 'Loading %s...' % log.name
239      try:
240        _, gpu, filter, hasty = get_metadata(text)
241        tests = get_all_tests(text)
242        print 'Found %d test results.' % len(tests)
243        if all_passing(tests):
244          # Delete logs that don't contain failures.
245          os.remove(log.name)
246        else:
247          # GPU family goes first in path to simplify adding/deleting families.
248          output_path = os.path.join(_EXPECTATIONS_DIR, gpu)
249          if not os.access(output_path, os.R_OK):
250            os.makedirs(output_path)
251          expectation_path = os.path.join(output_path, filter)
252          if hasty:
253            expectation_path = os.path.join(output_path, filter + '.hasty')
254          merge_expectation_list(expectation_path, tests)
255      except:
256        print 'Error processing %s' % log.name
257
258
259JOB_TAGS_ALL = (
260'select distinct job_tag from chromeos_autotest_db.tko_test_view_2 '
261'where not job_tag like "%%hostless" and '
262'test_name LIKE "graphics_dEQP%%" and '
263'build_version>="%s" and '
264'build_version<="%s" and '
265'((status = "FAIL" and not job_name like "%%.NotPass") or '
266'job_name like "%%.functional" or '
267'job_name like "%%-master")' )
268
269JOB_TAGS_MASTER = (
270'select distinct job_tag from chromeos_autotest_db.tko_test_view_2 '
271'where not job_tag like "%%hostless" and '
272'test_name LIKE "graphics_dEQP%%" and '
273'build_version>="%s" and '
274'build_version<="%s" and '
275'job_name like "%%-master"' )
276
277def get_result_paths_from_autotest_db(host, user, password, build_from,
278                                      build_to):
279  paths = []
280  # TODO(ihf): Introduce flag to toggle between JOB_TAGS_ALL and _MASTER.
281  sql = JOB_TAGS_MASTER % (build_from, build_to)
282  cmd = ['mysql', '-u%s' % user, '-p%s' % password, '--host', host, '-e', sql]
283  p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
284  for line in p.communicate()[0].splitlines():
285    # Skip over unrelated sql spew (really first line only):
286    if line and 'chromeos-test' in line:
287      paths.append(_AUTOTEST_RESULT_TAG_TEMPLATE % line.rstrip())
288  print 'Found %d potential results in the database.' % len(paths)
289  return paths
290
291
292def copy_logs_from_gs_paths(paths):
293  i = 1
294  for gs_path in paths:
295    print '[%d/%d] %s' % (i, len(paths), gs_path)
296    copy_logs_from_gs_path(gs_path)
297    i = i+1
298
299
300argparser = argparse.ArgumentParser(
301    description='Download from GS and process dEQP logs into expectations.')
302argparser.add_argument(
303    '--host',
304    dest='host',
305    default='chromeos-server38.cbf',
306    help='Host containing autotest result DB.')
307argparser.add_argument('--user', dest='user', help='Database user account.')
308argparser.add_argument(
309    '--password',
310    dest='password',
311    help='Password for user account.')
312argparser.add_argument(
313    '--from',
314    dest='build_from',
315    help='Lowest build revision to include. Example: R51-8100.0.0')
316argparser.add_argument(
317    '--to',
318    dest='build_to',
319    help='Highest build revision to include. Example: R51-8101.0.0')
320
321args = argparser.parse_args()
322
323print pprint.pformat(args)
324# This is somewhat optional. Remove existing expectations to start clean, but
325# feel free to process them incrementally.
326execute(['rm', '-rf', _EXPECTATIONS_DIR])
327
328copy_logs_from_gs_paths(get_result_paths_from_autotest_db(
329    args.host, args.user, args.password, args.build_from, args.build_to))
330
331# This will include the just downloaded logs from GS as well.
332logs = get_local_logs()
333process_logs(logs)
334