• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2# Copyright 2016 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import argparse
7import json
8import os
9import urllib
10import urlparse
11import zlib
12
13
14WPT_TEST_URL = 'http://www.webpagetest.org/jsonResult.php?test={wpt_job}'
15ERROR_MISSING_WPT_JOBS = """WPT Job Ids not specified!
16
17Use --wpt_jobs to specify a list of comma separated ids.
18
19ie. python download_from_wpt.py --wpt_jobs 1,2,3 output_dir/
20"""
21
22def WriteMetadataAndTraceToFile(
23    output_path, file_name, metadata, trace_contents):
24  file_name = os.path.join(output_path, file_name)
25
26  with open(os.path.join(output_path, file_name), 'wb') as f:
27    f.write(trace_contents)
28
29  with open(os.path.join(output_path, '%s.meta' % file_name), 'w') as f:
30    json.dump(metadata, f)
31
32
33def DownloadFromWPT(wpt_job, output_path):
34  url = WPT_TEST_URL.format(wpt_job=wpt_job)
35  job_response = urllib.urlopen(url)
36  job_data = json.load(job_response)
37
38  blacklist = ['runs', 'median', 'average', 'standardDeviation']
39  metadata = dict(
40      [(k, v) for k, v in job_data['data'].iteritems() if not k in blacklist])
41
42  for k,v in job_data['data']['runs'].iteritems():
43    for a,b in v.iteritems():
44      if not 'trace' in b['rawData']:
45        continue
46      trace_url = b['rawData']['trace']
47      parsed_url = urlparse.urlparse(trace_url)
48      query = urlparse.parse_qsl(parsed_url.query)
49      file_name = '%s_%s' % (wpt_job, query[2][1])
50      print 'Downloading %s to %s ...' % (trace_url, file_name)
51
52      file_response = urllib.urlopen(trace_url)
53
54      WriteMetadataAndTraceToFile(
55          output_path, file_name, metadata, file_response.read())
56
57
58def Main():
59  parser = argparse.ArgumentParser(description='Process traces')
60  parser.add_argument('output_path', help='Output path')
61  parser.add_argument('--wpt_jobs', help='WebPageTest Job IDs, comma separated')
62  args = parser.parse_args()
63
64  output_path = os.path.abspath(args.output_path)
65
66  if not args.wpt_jobs:
67    parser.exit(1, ERROR_MISSING_WPT_JOBS)
68
69  wpt_jobs =  args.wpt_jobs.split(',')
70
71  for wpt_job in wpt_jobs:
72    DownloadFromWPT(wpt_job, output_path)
73