1# Copyright 2016 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6# Recipe for uploading DM results. 7 8 9import calendar 10 11 12DEPS = [ 13 'recipe_engine/file', 14 'recipe_engine/json', 15 'recipe_engine/path', 16 'recipe_engine/properties', 17 'recipe_engine/step', 18 'recipe_engine/time', 19] 20 21 22DM_JSON = 'dm.json' 23UPLOAD_ATTEMPTS = 5 24VERBOSE_LOG = 'verbose.log' 25 26 27def cp(api, name, src, dst, extra_args=None): 28 cmd = ['gsutil', 'cp'] 29 if extra_args: 30 cmd.extend(extra_args) 31 cmd.extend([src, dst]) 32 33 name = 'upload %s' % name 34 for i in xrange(UPLOAD_ATTEMPTS): 35 step_name = name 36 if i > 0: 37 step_name += ' (attempt %d)' % (i+1) 38 try: 39 api.step(step_name, cmd=cmd) 40 break 41 except api.step.StepFailure: 42 if i == UPLOAD_ATTEMPTS - 1: 43 raise 44 45 46def RunSteps(api): 47 builder_name = api.properties['buildername'] 48 revision = api.properties['revision'] 49 50 results_dir = api.path['start_dir'].join('dm') 51 52 # Move dm.json and verbose.log to their own directory. 53 json_file = results_dir.join(DM_JSON) 54 log_file = results_dir.join(VERBOSE_LOG) 55 tmp_dir = api.path['start_dir'].join('tmp_upload') 56 api.file.ensure_directory('makedirs tmp dir', tmp_dir) 57 api.file.copy('copy dm.json', json_file, tmp_dir) 58 api.file.copy('copy verbose.log', log_file, tmp_dir) 59 api.file.remove('rm old dm.json', json_file) 60 api.file.remove('rm old verbose.log', log_file) 61 62 # Upload the images. 63 image_dest_path = 'gs://%s/dm-images-v1' % api.properties['gs_bucket'] 64 for ext in ['.png', '.pdf']: 65 files_to_upload = api.file.glob_paths( 66 'find images', 67 results_dir, 68 '*%s' % ext, 69 test_data=['someimage.png']) 70 # For some reason, glob returns results_dir when it should return nothing. 71 files_to_upload = [f for f in files_to_upload if str(f).endswith(ext)] 72 if len(files_to_upload) > 0: 73 cp(api, 'images', results_dir.join('*%s' % ext), image_dest_path) 74 75 # Upload the JSON summary and verbose.log. 76 now = api.time.utcnow() 77 summary_dest_path = '/'.join([ 78 'dm-json-v1', 79 str(now.year ).zfill(4), 80 str(now.month).zfill(2), 81 str(now.day ).zfill(2), 82 str(now.hour ).zfill(2), 83 revision, 84 builder_name, 85 str(int(calendar.timegm(now.utctimetuple())))]) 86 87 # Trybot results are further siloed by issue/patchset. 88 issue = api.properties.get('patch_issue') 89 patchset = api.properties.get('patch_set') 90 if issue and patchset: 91 summary_dest_path = '/'.join(( 92 'trybot', summary_dest_path, str(issue), str(patchset))) 93 94 summary_dest_path = 'gs://%s/%s' % (api.properties['gs_bucket'], 95 summary_dest_path) 96 97 cp(api, 'JSON and logs', tmp_dir.join('*'), summary_dest_path, 98 ['-z', 'json,log']) 99 100 101def GenTests(api): 102 builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug' 103 yield ( 104 api.test('normal_bot') + 105 api.properties(buildername=builder, 106 gs_bucket='skia-infra-gm', 107 revision='abc123', 108 path_config='kitchen') 109 ) 110 111 yield ( 112 api.test('failed_once') + 113 api.properties(buildername=builder, 114 gs_bucket='skia-infra-gm', 115 revision='abc123', 116 path_config='kitchen') + 117 api.step_data('upload images', retcode=1) 118 ) 119 120 yield ( 121 api.test('failed_all') + 122 api.properties(buildername=builder, 123 gs_bucket='skia-infra-gm', 124 revision='abc123', 125 path_config='kitchen') + 126 api.step_data('upload images', retcode=1) + 127 api.step_data('upload images (attempt 2)', retcode=1) + 128 api.step_data('upload images (attempt 3)', retcode=1) + 129 api.step_data('upload images (attempt 4)', retcode=1) + 130 api.step_data('upload images (attempt 5)', retcode=1) 131 ) 132 133 builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug' 134 yield ( 135 api.test('trybot') + 136 api.properties( 137 buildername=builder, 138 gs_bucket='skia-infra-gm', 139 revision='abc123', 140 path_config='kitchen', 141 patch_storage='gerrit') + 142 api.properties.tryserver( 143 buildername=builder, 144 gerrit_project='skia', 145 gerrit_url='https://skia-review.googlesource.com/', 146 ) 147 ) 148