• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env vpython3
2
3# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
4#
5# Use of this source code is governed by a BSD-style license
6# that can be found in the LICENSE file in the root of the source
7# tree. An additional intellectual property rights grant can be found
8# in the file PATENTS.  All contributing project authors may
9# be found in the AUTHORS file in the root of the source tree.
10
11import datetime
12import json
13import subprocess
14import time
15import zlib
16
17from typing import Optional
18import dataclasses
19import httplib2
20
21from tracing.value import histogram
22from tracing.value import histogram_set
23from tracing.value.diagnostics import generic_set
24from tracing.value.diagnostics import reserved_infos
25
26
27@dataclasses.dataclass
28class UploaderOptions():
29  """Required information to upload perf metrics.
30
31    Attributes:
32      perf_dashboard_machine_group: The "master" the bots are grouped under.
33        This string is the group in the the perf dashboard path
34        group/bot/perf_id/metric/subtest.
35      bot: The bot running the test (e.g. webrtc-win-large-tests).
36      test_suite: The key for the test in the dashboard (i.e. what you select
37        in the top-level test suite selector in the dashboard
38      webrtc_git_hash: webrtc.googlesource.com commit hash.
39      commit_position: Commit pos corresponding to the git hash.
40      build_page_url: URL to the build page for this build.
41      dashboard_url: Which dashboard to use.
42      input_results_file: A HistogramSet proto file coming from WebRTC tests.
43      output_json_file: Where to write the output (for debugging).
44      wait_timeout_sec: Maximum amount of time in seconds that the script will
45        wait for the confirmation.
46      wait_polling_period_sec: Status will be requested from the Dashboard
47        every wait_polling_period_sec seconds.
48    """
49  perf_dashboard_machine_group: str
50  bot: str
51  test_suite: str
52  webrtc_git_hash: str
53  commit_position: int
54  build_page_url: str
55  dashboard_url: str
56  input_results_file: str
57  output_json_file: Optional[str] = None
58  wait_timeout_sec: datetime.timedelta = datetime.timedelta(seconds=1200)
59  wait_polling_period_sec: datetime.timedelta = datetime.timedelta(seconds=120)
60
61
62def _GenerateOauthToken():
63  args = ['luci-auth', 'token']
64  p = subprocess.Popen(args,
65                       universal_newlines=True,
66                       stdout=subprocess.PIPE,
67                       stderr=subprocess.PIPE)
68  if p.wait() == 0:
69    output = p.stdout.read()
70    return output.strip()
71  raise RuntimeError(
72      'Error generating authentication token.\nStdout: %s\nStderr:%s' %
73      (p.stdout.read(), p.stderr.read()))
74
75
76def _CreateHeaders(oauth_token):
77  return {'Authorization': 'Bearer %s' % oauth_token}
78
79
80def _SendHistogramSet(url, histograms):
81  """Make a HTTP POST with the given JSON to the Performance Dashboard.
82
83    Args:
84      url: URL of Performance Dashboard instance, e.g.
85          "https://chromeperf.appspot.com".
86      histograms: a histogram set object that contains the data to be sent.
87    """
88  headers = _CreateHeaders(_GenerateOauthToken())
89
90  serialized = json.dumps(_ApplyHacks(histograms.AsDicts()), indent=4)
91
92  if url.startswith('http://localhost'):
93    # The catapult server turns off compression in developer mode.
94    data = serialized
95  else:
96    data = zlib.compress(serialized.encode('utf-8'))
97
98  print('Sending %d bytes to %s.' % (len(data), url + '/add_histograms'))
99
100  http = httplib2.Http()
101  response, content = http.request(url + '/add_histograms',
102                                   method='POST',
103                                   body=data,
104                                   headers=headers)
105  return response, content
106
107
108def _WaitForUploadConfirmation(url, upload_token, wait_timeout,
109                               wait_polling_period):
110  """Make a HTTP GET requests to the Performance Dashboard untill upload
111    status is known or the time is out.
112
113    Args:
114      url: URL of Performance Dashboard instance, e.g.
115          "https://chromeperf.appspot.com".
116      upload_token: String that identifies Performance Dashboard and can be used
117        for the status check.
118      wait_timeout: (datetime.timedelta) Maximum time to wait for the
119        confirmation.
120      wait_polling_period: (datetime.timedelta) Performance Dashboard will be
121        polled every wait_polling_period amount of time.
122    """
123  assert wait_polling_period <= wait_timeout
124
125  headers = _CreateHeaders(_GenerateOauthToken())
126  http = httplib2.Http()
127
128  oauth_refreshed = False
129  response = None
130  resp_json = None
131  current_time = datetime.datetime.now()
132  end_time = current_time + wait_timeout
133  next_poll_time = current_time + wait_polling_period
134  while datetime.datetime.now() < end_time:
135    current_time = datetime.datetime.now()
136    if next_poll_time > current_time:
137      time.sleep((next_poll_time - current_time).total_seconds())
138    next_poll_time = datetime.datetime.now() + wait_polling_period
139
140    response, content = http.request(url + '/uploads/' + upload_token,
141                                     method='GET',
142                                     headers=headers)
143
144    print('Upload state polled. Response: %r.' % content)
145
146    if not oauth_refreshed and response.status == 403:
147      print('Oauth token refreshed. Continue polling.')
148      headers = _CreateHeaders(_GenerateOauthToken())
149      oauth_refreshed = True
150      continue
151
152    if response.status != 200:
153      break
154
155    resp_json = json.loads(content)
156    if resp_json['state'] == 'COMPLETED' or resp_json['state'] == 'FAILED':
157      break
158
159  return response, resp_json
160
161
162# Because of an issues on the Dashboard side few measurements over a large set
163# can fail to upload. That would lead to the whole upload to be marked as
164# failed. Check it, so it doesn't increase flakiness of our tests.
165# TODO(crbug.com/1145904): Remove check after fixed.
166def _CheckFullUploadInfo(url, upload_token,
167                         min_measurements_amount=50,
168                         max_failed_measurements_percent=0.03):
169  """Make a HTTP GET requests to the Performance Dashboard to get full info
170    about upload (including measurements). Checks if upload is correct despite
171    not having status "COMPLETED".
172
173    Args:
174      url: URL of Performance Dashboard instance, e.g.
175          "https://chromeperf.appspot.com".
176      upload_token: String that identifies Performance Dashboard and can be used
177        for the status check.
178      min_measurements_amount: minimal amount of measurements that the upload
179        should have to start tolerating failures in particular measurements.
180      max_failed_measurements_percent: maximal percent of failured measurements
181        to tolerate.
182    """
183  headers = _CreateHeaders(_GenerateOauthToken())
184  http = httplib2.Http()
185
186  response, content = http.request(url + '/uploads/' + upload_token +
187                                   '?additional_info=measurements',
188                                   method='GET',
189                                   headers=headers)
190
191  if response.status != 200:
192    print('Failed to reach the dashboard to get full upload info.')
193    return False
194
195  resp_json = json.loads(content)
196  print('Full upload info: %s.' % json.dumps(resp_json, indent=4))
197
198  if 'measurements' in resp_json:
199    measurements_cnt = len(resp_json['measurements'])
200    not_completed_state_cnt = len(
201        [m for m in resp_json['measurements'] if m['state'] != 'COMPLETED'])
202
203    if (measurements_cnt >= min_measurements_amount
204        and (not_completed_state_cnt /
205             (measurements_cnt * 1.0) <= max_failed_measurements_percent)):
206      print(('Not all measurements were confirmed to upload. '
207             'Measurements count: %d, failed to upload or timed out: %d' %
208             (measurements_cnt, not_completed_state_cnt)))
209      return True
210
211  return False
212
213
214# TODO(https://crbug.com/1029452): HACKHACK
215# Remove once we have doubles in the proto and handle -infinity correctly.
216def _ApplyHacks(dicts):
217  def _NoInf(value):
218    if value == float('inf'):
219      return histogram.JS_MAX_VALUE
220    if value == float('-inf'):
221      return -histogram.JS_MAX_VALUE
222    return value
223
224  for d in dicts:
225    if 'running' in d:
226      d['running'] = [_NoInf(value) for value in d['running']]
227    if 'sampleValues' in d:
228      d['sampleValues'] = [_NoInf(value) for value in d['sampleValues']]
229
230  return dicts
231
232
233def _LoadHistogramSetFromProto(options):
234  hs = histogram_set.HistogramSet()
235  with open(options.input_results_file, 'rb') as f:
236    hs.ImportProto(f.read())
237
238  return hs
239
240
241def _AddBuildInfo(histograms, options):
242  common_diagnostics = {
243      reserved_infos.MASTERS: options.perf_dashboard_machine_group,
244      reserved_infos.BOTS: options.bot,
245      reserved_infos.POINT_ID: options.commit_position,
246      reserved_infos.BENCHMARKS: options.test_suite,
247      reserved_infos.WEBRTC_REVISIONS: str(options.webrtc_git_hash),
248      reserved_infos.BUILD_URLS: options.build_page_url,
249  }
250
251  for k, v in list(common_diagnostics.items()):
252    histograms.AddSharedDiagnosticToAllHistograms(k.name,
253                                                  generic_set.GenericSet([v]))
254
255
256def _DumpOutput(histograms, output_file):
257  with open(output_file, 'w') as f:
258    json.dump(_ApplyHacks(histograms.AsDicts()), f, indent=4)
259
260
261def UploadToDashboardImpl(options):
262  histograms = _LoadHistogramSetFromProto(options)
263  _AddBuildInfo(histograms, options)
264
265  if options.output_json_file:
266    _DumpOutput(histograms, options.output_json_file)
267
268  response, content = _SendHistogramSet(options.dashboard_url, histograms)
269
270  if response.status != 200:
271    print(('Upload failed with %d: %s\n\n%s' %
272           (response.status, response.reason, content)))
273    return 1
274
275  upload_token = json.loads(content).get('token')
276  if not upload_token:
277    print(('Received 200 from dashboard. ',
278           'Not waiting for the upload status confirmation.'))
279    return 0
280
281  response, resp_json = _WaitForUploadConfirmation(
282      options.dashboard_url, upload_token, options.wait_timeout_sec,
283      options.wait_polling_period_sec)
284
285  if ((resp_json and resp_json['state'] == 'COMPLETED')
286      or _CheckFullUploadInfo(options.dashboard_url, upload_token)):
287    print('Upload completed.')
288    return 0
289
290  if response.status != 200:
291    print(('Upload status poll failed with %d: %s' %
292           (response.status, response.reason)))
293    return 1
294
295  if resp_json['state'] == 'FAILED':
296    print('Upload failed.')
297    return 1
298
299  print(('Upload wasn\'t completed in a given time: %s seconds.' %
300         options.wait_timeout_sec))
301  return 1
302
303
304def UploadToDashboard(options):
305  try:
306    exit_code = UploadToDashboardImpl(options)
307  except RuntimeError as e:
308    print(e)
309    return 1
310  return exit_code
311