• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7
8""" Look through skia-autogen, searching for all checksums which should have
9corresponding files in Google Storage, and verify that those files exist. """
10
11
12import json
13import posixpath
14import re
15import subprocess
16import sys
17
18
19# TODO(borenet): Replace some/all of these with constants from gm/gm_json.py
20AUTOGEN_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual'
21GS_URL = 'gs://chromium-skia-gm/gm'
22TEST_NAME_PATTERN = re.compile('(\S+)_(\S+).png')
23
24
25def FileNameToGSURL(filename, hash_type, hash_value):
26  """ Convert a file name given in a checksum file to the URL of the
27  corresponding image file in Google Storage.
28
29  filename: string; the file name to convert. Takes the form specified by
30      TEST_NAME_PATTERN.
31  hash_type: string; the type of the checksum.
32  hash_value: string; the checksum itself.
33  """
34  test_name = TEST_NAME_PATTERN.match(filename).group(1)
35  if not test_name:
36    raise Exception('Invalid test name for file: %s' % filename)
37  return '%s/%s/%s/%s.png' % (GS_URL, hash_type, test_name, hash_value)
38
39
40def FindURLSInJSON(json_file, gs_urls):
41  """ Extract Google Storage URLs from a JSON file in svn, adding them to the
42  gs_urls dictionary.
43
44  json_file: string; URL of the JSON file.
45  gs_urls: dict; stores Google Storage URLs as keys and lists of the JSON files
46      which reference them.
47
48  Example gs_urls:
49  { 'gs://chromium-skia-gm/gm/sometest/12345.png': [
50      'http://skia-autogen.googlecode.com/svn/gm-actual/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug/actual-results.json',
51      'http://skia-autogen.googlecode.com/svn/gm-actual/Test-Mac10.8-MacMini4.1-GeForce320M-x86-Debug/actual-results.json',
52    ]
53  }
54  """
55  output = subprocess.check_output(['svn', 'cat', json_file])
56  json_content = json.loads(output)
57  for dict_type in ['actual-results']:
58    for result_type in json_content[dict_type]:
59      if json_content[dict_type][result_type]:
60        for result in json_content[dict_type][result_type].keys():
61          hash_type, hash_value = json_content[dict_type][result_type][result]
62          gs_url = FileNameToGSURL(result, hash_type, str(hash_value))
63          if gs_urls.get(gs_url):
64            gs_urls[gs_url].append(json_file)
65          else:
66            gs_urls[gs_url] = [json_file]
67
68
69def _FindJSONFiles(url, json_files):
70  """ Helper function for FindJsonFiles. Recursively explore the repository,
71  adding JSON files to a list.
72
73  url: string; URL of the repository (or subdirectory thereof) to explore.
74  json_files: list to which JSON file urls will be added.
75  """
76  proc = subprocess.Popen(['svn', 'ls', url], stdout=subprocess.PIPE,
77                          stderr=subprocess.STDOUT)
78  if proc.wait() != 0:
79    raise Exception('Failed to list svn directory.')
80  output = proc.communicate()[0].splitlines()
81  subdirs = []
82  for item in output:
83    if item.endswith(posixpath.sep):
84      subdirs.append(item)
85    elif item.endswith('.json'):
86      json_files.append(posixpath.join(url, item))
87    else:
88      print 'Warning: ignoring %s' % posixpath.join(url, item)
89  for subdir in subdirs:
90    _FindJSONFiles(posixpath.join(url, subdir), json_files)
91
92
93def FindJSONFiles(url):
94  """ Recursively explore the given repository and return a list of the JSON
95  files it contains.
96
97  url: string; URL of the repository to explore.
98  """
99  print 'Searching for JSON files in %s' % url
100  json_files = []
101  _FindJSONFiles(url, json_files)
102  return json_files
103
104
105def FindURLs(url):
106  """ Find Google Storage URLs inside of JSON files in the given repository.
107  Returns a dictionary whose keys are Google Storage URLs and values are lists
108  of the JSON files which reference them.
109
110  url: string; URL of the repository to explore.
111
112  Example output:
113  { 'gs://chromium-skia-gm/gm/sometest/12345.png': [
114      'http://skia-autogen.googlecode.com/svn/gm-actual/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug/actual-results.json',
115      'http://skia-autogen.googlecode.com/svn/gm-actual/Test-Mac10.8-MacMini4.1-GeForce320M-x86-Debug/actual-results.json',
116    ]
117  }
118  """
119  gs_urls = {}
120  for json_file in FindJSONFiles(url):
121    print 'Looking for checksums in %s' % json_file
122    FindURLSInJSON(json_file, gs_urls)
123  return gs_urls
124
125
126def VerifyURL(url):
127  """ Verify that the given URL exists.
128
129  url: string; the Google Storage URL of the image file in question.
130  """
131  proc = subprocess.Popen(['gsutil', 'ls', url], stdout=subprocess.PIPE,
132                          stderr=subprocess.STDOUT)
133  if proc.wait() != 0:
134    return False
135  return True
136
137
138def VerifyURLs(urls):
139  """ Verify that each of the given URLs exists. Return a list of which URLs do
140  not exist.
141
142  urls: dictionary; URLs of the image files in question.
143  """
144  print 'Verifying that images exist for URLs...'
145  missing = []
146  for url in urls.iterkeys():
147    if not VerifyURL(url):
148      print 'Missing: %s, referenced by: \n  %s' % (url, '\n  '.join(urls[url]))
149      missing.append(url)
150  return missing
151
152
153def Main():
154  urls = FindURLs(AUTOGEN_URL)
155  missing = VerifyURLs(urls)
156  if missing:
157    print 'Found %d Missing files.' % len(missing)
158    return 1
159
160
161if __name__ == '__main__':
162  sys.exit(Main())
163