• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2016 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6# Recipe for uploading DM results.
7
8
9import calendar
10
11
12DEPS = [
13  'recipe_engine/file',
14  'recipe_engine/json',
15  'recipe_engine/path',
16  'recipe_engine/properties',
17  'recipe_engine/step',
18  'recipe_engine/time',
19]
20
21
22DM_JSON = 'dm.json'
23UPLOAD_ATTEMPTS = 5
24VERBOSE_LOG = 'verbose.log'
25
26
27def cp(api, name, src, dst, extra_args=None):
28  cmd = ['gsutil', 'cp']
29  if extra_args:
30    cmd.extend(extra_args)
31  cmd.extend([src, dst])
32
33  name = 'upload %s' % name
34  for i in xrange(UPLOAD_ATTEMPTS):
35    step_name = name
36    if i > 0:
37      step_name += ' (attempt %d)' % (i+1)
38    try:
39      api.step(step_name, cmd=cmd)
40      break
41    except api.step.StepFailure:
42      if i == UPLOAD_ATTEMPTS - 1:
43        raise
44
45
46def RunSteps(api):
47  builder_name = api.properties['buildername']
48  revision = api.properties['revision']
49
50  results_dir = api.path['start_dir'].join('dm')
51
52  # Move dm.json and verbose.log to their own directory.
53  json_file = results_dir.join(DM_JSON)
54  log_file = results_dir.join(VERBOSE_LOG)
55  tmp_dir = api.path['start_dir'].join('tmp_upload')
56  api.file.ensure_directory('makedirs tmp dir', tmp_dir)
57  api.file.copy('copy dm.json', json_file, tmp_dir)
58  api.file.copy('copy verbose.log', log_file, tmp_dir)
59  api.file.remove('rm old dm.json', json_file)
60  api.file.remove('rm old verbose.log', log_file)
61
62  # Upload the images.
63  image_dest_path = 'gs://%s/dm-images-v1' % api.properties['gs_bucket']
64  for ext in ['.png', '.pdf']:
65    files_to_upload = api.file.glob_paths(
66        'find images',
67        results_dir,
68        '*%s' % ext,
69        test_data=['someimage.png'])
70    # For some reason, glob returns results_dir when it should return nothing.
71    files_to_upload = [f for f in files_to_upload if str(f).endswith(ext)]
72    if len(files_to_upload) > 0:
73      cp(api, 'images', results_dir.join('*%s' % ext), image_dest_path)
74
75  # Upload the JSON summary and verbose.log.
76  now = api.time.utcnow()
77  summary_dest_path = '/'.join([
78      'dm-json-v1',
79      str(now.year ).zfill(4),
80      str(now.month).zfill(2),
81      str(now.day  ).zfill(2),
82      str(now.hour ).zfill(2),
83      revision,
84      builder_name,
85      str(int(calendar.timegm(now.utctimetuple())))])
86
87  # Trybot results are further siloed by issue/patchset.
88  issue = api.properties.get('patch_issue')
89  patchset = api.properties.get('patch_set')
90  if issue and patchset:
91    summary_dest_path = '/'.join((
92        'trybot', summary_dest_path, str(issue), str(patchset)))
93
94  summary_dest_path = 'gs://%s/%s' % (api.properties['gs_bucket'],
95                                      summary_dest_path)
96
97  cp(api, 'JSON and logs', tmp_dir.join('*'), summary_dest_path,
98     ['-z', 'json,log'])
99
100
101def GenTests(api):
102  builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug'
103  yield (
104    api.test('normal_bot') +
105    api.properties(buildername=builder,
106                   gs_bucket='skia-infra-gm',
107                   revision='abc123',
108                   path_config='kitchen')
109  )
110
111  yield (
112    api.test('failed_once') +
113    api.properties(buildername=builder,
114                   gs_bucket='skia-infra-gm',
115                   revision='abc123',
116                   path_config='kitchen') +
117    api.step_data('upload images', retcode=1)
118  )
119
120  yield (
121    api.test('failed_all') +
122    api.properties(buildername=builder,
123                   gs_bucket='skia-infra-gm',
124                   revision='abc123',
125                   path_config='kitchen') +
126    api.step_data('upload images', retcode=1) +
127    api.step_data('upload images (attempt 2)', retcode=1) +
128    api.step_data('upload images (attempt 3)', retcode=1) +
129    api.step_data('upload images (attempt 4)', retcode=1) +
130    api.step_data('upload images (attempt 5)', retcode=1)
131  )
132
133  builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug'
134  yield (
135      api.test('trybot') +
136      api.properties(
137          buildername=builder,
138          gs_bucket='skia-infra-gm',
139          revision='abc123',
140          path_config='kitchen',
141          patch_storage='gerrit') +
142      api.properties.tryserver(
143          buildername=builder,
144          gerrit_project='skia',
145          gerrit_url='https://skia-review.googlesource.com/',
146      )
147  )
148