• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2016 The Chromium Embedded Framework Authors. All rights
2# reserved. Use of this source code is governed by a BSD-style license that
3# can be found in the LICENSE file.
4
5from __future__ import absolute_import
6from __future__ import print_function
7import datetime
8import json
9import os
10import re
11import sys
12
13if sys.version_info.major == 2:
14  from urllib2 import urlopen
15else:
16  from urllib.request import urlopen
17
18# Class used to build the cefbuilds JSON file. See cef_json_builder_example.py
19# for example usage. See cef_json_builder_test.py for unit tests.
20#
21# Example JSON structure:
22# {
23#   "linux32": {
24#     "versions": [
25#       {
26#         "cef_version": "3.2704.1414.g185cd6c",
27#         "chromium_version": "51.0.2704.47"
28#         "files": [
29#           {
30#             "last_modified": "2016-05-18T22:42:14.066Z"
31#             "name": "cef_binary_3.2704.1414.g185cd6c_linux32.tar.bz2",
32#             "sha1": "47c5cfea43912a1d1771f343de35b205f388415f"
33#             "size": "48549450",
34#             "type": "standard",
35#           }, ...
36#         ],
37#       }, ...
38#     ]
39#   }, ...
40# }
41#
42# Notes:
43# - "files" in a given version will be sorted from newest to oldest based on the
44#   "last_modified" value.
45# - "versions" in a given platform will be sorted from newest to oldest based on
46#   the "last_modified" value of the first (newest) "file" sub-value.
47# - There will be at most one record at the "files" level for each "type".
48
49# This date format intentionally matches the format used in Artifactory
50# directory listings.
51_CEF_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
52
53
54def parse_date(date):
55  return datetime.datetime.strptime(date, _CEF_DATE_FORMAT)
56
57
58def format_date(date):
59  return date.strftime(_CEF_DATE_FORMAT)
60
61
62# Helpers to format datetime values on JSON read/write.
63def cef_from_json(json_object):
64  if 'last_modified' in json_object:
65    json_object['last_modified'] = parse_date(json_object['last_modified'])
66  return json_object
67
68
69class cef_json_encoder(json.JSONEncoder):
70
71  def default(self, o):
72    if isinstance(o, datetime.datetime):
73      return format_date(o)
74    return o
75
76
77_chromium_version_regex = '[1-9]{1}[0-9]{1,2}\.0\.[1-9]{1}[0-9]{2,4}\.(0|[1-9]{1}[0-9]{0,2})'
78_cef_hash_regex = 'g[0-9a-f]{7}'
79_cef_number_regex = '[0-9]{1,5}\.[0-9]{1,5}\.[0-9]{1,5}'
80
81# Example: 3.2704.1414.g185cd6c
82_cef_old_version_regex = _cef_number_regex + '\.' + _cef_hash_regex
83# Example: 74.0.1+g62d140e+chromium-74.0.3729.6
84_cef_version_release_regex = _cef_number_regex + '\+' + _cef_hash_regex + '\+chromium\-' + _chromium_version_regex
85# Example: 74.0.0-master.1920+g725ed88+chromium-74.0.3729.0
86_cef_version_dev_regex = _cef_number_regex + '\-\w+\.[0-9]{1,7}\+' + _cef_hash_regex + '\+chromium\-' + _chromium_version_regex
87
88
89class cef_json_builder:
90  """ Class used to build the cefbuilds JSON file. """
91
92  def __init__(self, prettyprint=False, silent=True):
93    """ Create a new cef_json_builder object. """
94    self._prettyprint = prettyprint
95    self._silent = silent
96    self._fatalerrors = False
97    self.clear()
98
99  @staticmethod
100  def get_platforms():
101    """ Returns the list of supported platforms. """
102    return ('linux32', 'linux64', 'linuxarm', 'linuxarm64', 'macosarm64', 'macosx64',
103            'windows32', 'windows64', 'windowsarm64')
104
105  @staticmethod
106  def get_distrib_types():
107    """ Returns the list of supported distribution types. """
108    return ('standard', 'minimal', 'client', 'release_symbols', 'debug_symbols')
109
110  @staticmethod
111  def is_valid_version(version):
112    """ Returns true if the specified CEF version is fully qualified and valid. """
113    if version is None:
114      return False
115    return bool(re.compile('^' + _cef_old_version_regex + '$').match(version)) \
116        or bool(re.compile('^' + _cef_version_release_regex + '$').match(version)) \
117        or bool(re.compile('^' + _cef_version_dev_regex + '$').match(version))
118
119  @staticmethod
120  def is_valid_chromium_version(version):
121    """ Returns true if the specified Chromium version is fully qualified and valid. """
122    if version is None:
123      return False
124    return version == 'master' or \
125        bool(re.compile('^' + _chromium_version_regex + '$').match(version))
126
127  @staticmethod
128  def get_file_name(version, platform, type, channel='stable'):
129    """ Returns the expected distribution file name excluding extension based on
130        the input parameters. """
131    type_str = '_' + type if type != 'standard' else ''
132    channel_str = '_' + channel if channel != 'stable' else ''
133    return 'cef_binary_%s_%s%s%s' % (version, platform, channel_str, type_str)
134
135  def clear(self):
136    """ Clear the contents of this object. """
137    self._data = {}
138    for platform in self.get_platforms():
139      self._data[platform] = {'versions': []}
140    self._versions = {}
141    self._queryct = 0
142
143  def __repr__(self):
144    # Return a string representation of this object.
145    self._sort_versions()
146    if self._prettyprint:
147      return json.dumps(
148          self._data,
149          cls=cef_json_encoder,
150          sort_keys=True,
151          indent=2,
152          separators=(',', ': '))
153    else:
154      return json.dumps(self._data, cls=cef_json_encoder, sort_keys=True)
155
156  def _print(self, msg):
157    if self._fatalerrors:
158      raise Exception(msg)
159    if not self._silent:
160      print(msg)
161
162  def get_query_count(self):
163    """ Returns the number of queries sent while building. """
164    return self._queryct
165
166  def _query_chromium_version(self, cef_version):
167    """ Try to remotely query the Chromium version for old-style CEF version
168        numbers. """
169    chromium_version = 'master'
170    git_hash = cef_version[-7:]
171    query_url = 'https://bitbucket.org/chromiumembedded/cef/raw/%s/CHROMIUM_BUILD_COMPATIBILITY.txt' % git_hash
172    self._queryct = self._queryct + 1
173    if not self._silent:
174      print('Reading %s' % query_url)
175
176    try:
177      # Read the remote URL contents.
178      handle = urlopen(query_url)
179      compat_value = handle.read().strip()
180      handle.close()
181
182      # Parse the contents.
183      config = eval(compat_value, {'__builtins__': None}, None)
184      if not 'chromium_checkout' in config:
185        raise Exception('Unexpected contents')
186
187      val = config['chromium_checkout']
188      if val.find('refs/tags/') == 0:
189        chromium_version = val[10:]
190    except Exception as e:
191      print('Failed to read Chromium version information')
192      raise
193
194    return chromium_version
195
196  def set_chromium_version(self, cef_version, chromium_version=None):
197    """ Set the matching Chromium version. If the specified Chromium version is
198        invalid then it will be queried remotely. """
199    if not self.is_valid_version(cef_version):
200      raise Exception('Invalid CEF version: %s' % cef_version)
201
202    if not self.is_valid_chromium_version(chromium_version):
203      if cef_version in self._versions:
204        # Keep the Chromium version that we already know about.
205        return self._versions[cef_version]
206
207      if cef_version.find('+chromium') > 0:
208        # New-style CEF version numbers include the Chromium version number.
209        # Example: 74.0.1+g62d140e+chromium-74.0.3729.6
210        chromium_version = cef_version[cef_version.rfind('-') + 1:]
211      else:
212        chromium_version = self._query_chromium_version(cef_version)
213
214    if not self.is_valid_chromium_version(chromium_version):
215      raise Exception('Invalid Chromium version: %s' % chromium_version)
216
217    self._versions[cef_version] = chromium_version
218    return chromium_version
219
220  def get_chromium_version(self, cef_version):
221    """ Return the matching Chromium version. If not currently known it will
222        be parsed from the CEF version or queried remotely. """
223    if cef_version in self._versions:
224      return self._versions[cef_version]
225    # Identify the Chromium version.
226    return self.set_chromium_version(cef_version)
227
228  def has_chromium_version(self, cef_version):
229    """ Return True if a matching Chromium version is known. """
230    return cef_version in self._versions
231
232  def load(self, json_string, fatalerrors=True):
233    """ Load new JSON into this object. Any existing contents will be cleared.
234        If |fatalerrors| is True then any errors while loading the JSON file
235        will cause an Exception to be thrown. Otherwise, malformed entries will
236        will be discarded. Unrecognized keys will always be discarded silently.
237    """
238    self.clear()
239
240    self._fatalerrors = fatalerrors
241
242    new_data = json.JSONDecoder(object_hook=cef_from_json).decode(json_string)
243
244    # Validate the new data's structure.
245    for platform in self._data.keys():
246      if not platform in new_data:
247        if not self._silent:
248          print('load: Platform %s not found' % platform)
249        continue
250      if not 'versions' in new_data[platform]:
251        self._print('load: Missing platform key(s) for %s' % platform)
252        continue
253
254      valid_versions = []
255      for version in new_data[platform]['versions']:
256        if not 'cef_version' in version or \
257            not 'chromium_version' in version or \
258            not 'files' in version:
259          self._print('load: Missing version key(s) for %s' % platform)
260          continue
261
262        valid_files = []
263        found_types = []
264        for file in version['files']:
265          if not 'type' in file or \
266              not 'name' in file or \
267              not 'size' in file or \
268              not 'last_modified' in file or \
269              not 'sha1' in file:
270            self._print('load: Missing file key(s) for %s %s' %
271                        (platform, version['cef_version']))
272            continue
273          (expected_platform, expected_version, expected_type,
274           expected_channel) = self._parse_name(file['name'])
275          if expected_platform != platform or \
276              expected_version != version['cef_version'] or \
277              expected_type != file['type']:
278            self._print('load: File name/attribute mismatch for %s %s %s' %
279                        (platform, version['cef_version'], file['name']))
280            continue
281          self._validate_args(platform, version['cef_version'], file['type'],
282                              file['size'], file['last_modified'], file['sha1'])
283          if file['type'] in found_types:
284            self._print('load: Duplicate %s type for %s %s' %
285                        (file['type'], platform, version['cef_version']))
286            continue
287          found_types.append(file['type'])
288          valid_files.append({
289              'type': file['type'],
290              'name': file['name'],
291              'size': file['size'],
292              'last_modified': file['last_modified'],
293              'sha1': file['sha1'],
294          })
295
296        if len(valid_files) > 0:
297          valid_versions.append({
298              'cef_version':
299                  version['cef_version'],
300              'chromium_version':
301                  self.set_chromium_version(version['cef_version'],
302                                            version['chromium_version']),
303              'channel':
304                  version.get('channel', 'stable'),
305              'files':
306                  self._sort_files(valid_files)
307          })
308
309      if len(valid_versions) > 0:
310        self._data[platform]['versions'] = valid_versions
311
312    self._fatalerrors = False
313
314  def _sort_versions(self):
315    # Sort version records by first (newest) file last_modified value.
316    for platform in self._data.keys():
317      for i in range(0, len(self._data[platform]['versions'])):
318        self._data[platform]['versions'] = \
319          sorted(self._data[platform]['versions'],
320                 key=lambda k: k['files'][0]['last_modified'],
321                 reverse=True)
322
323  @staticmethod
324  def _sort_files(files):
325    # Sort file records by last_modified.
326    return sorted(files, key=lambda k: k['last_modified'], reverse=True)
327
328  @staticmethod
329  def _parse_name(name):
330    # Remove file extension.
331    name_no_ext = os.path.splitext(name)[0]
332    if name_no_ext[-4:] == '.tar':
333      name_no_ext = name_no_ext[:-4]
334    name_parts = name_no_ext.split('_')
335    if len(
336        name_parts) < 4 or name_parts[0] != 'cef' or name_parts[1] != 'binary':
337      raise Exception('Invalid filename: %s' % name)
338
339    # Remove 'cef' and 'binary'.
340    del name_parts[0]
341    del name_parts[0]
342
343    type = None
344    channel = 'stable'
345
346    # Might be '<version>_<platform>_[debug|release]_symbols'.
347    if name_parts[-1] == 'symbols':
348      del name_parts[-1]
349      if name_parts[-1] == 'debug' or name_parts[-1] == 'release':
350        type = name_parts[-1] + '_symbols'
351        del name_parts[-1]
352
353    # Might be '<version>_<platform>_minimal'.
354    if name_parts[-1] == 'minimal':
355      type = 'minimal'
356      del name_parts[-1]
357
358    # Might be '<version>_<platform>_client'.
359    if name_parts[-1] == 'client':
360      type = 'client'
361      del name_parts[-1]
362
363    # Might be '<version>_<platform>_beta'.
364    if name_parts[-1] == 'beta':
365      del name_parts[-1]
366      channel = 'beta'
367
368    # Remainder must be '<version>_<platform>'.
369    if len(name_parts) != 2:
370      raise Exception('Invalid filename: %s' % name)
371
372    if type is None:
373      type = 'standard'
374
375    version = name_parts[0]
376    platform = name_parts[1]
377
378    return [platform, version, type, channel]
379
380  @staticmethod
381  def _validate_args(platform, version, type, size, last_modified, sha1):
382    # Validate input arguments.
383    if not platform in cef_json_builder.get_platforms():
384      raise Exception('Unsupported platform: %s' % platform)
385
386    if not cef_json_builder.is_valid_version(version):
387      raise Exception('Invalid version: %s' % version)
388
389    if not type in cef_json_builder.get_distrib_types():
390      raise Exception('Unsupported distribution type: %s' % type)
391
392    if int(size) <= 0:
393      raise Exception('Invalid size: %s' % size)
394
395    if not isinstance(last_modified, datetime.datetime):
396      # datetime will throw a ValueException if it doesn't parse.
397      parse_date(last_modified)
398
399    if not re.compile('^[0-9a-f]{40}$').match(sha1):
400      raise Exception('Invalid sha1: %s' % sha1)
401
402  def add_file(self, name, size, last_modified, sha1):
403    """ Add a file record with the specified attributes. Returns True if the
404        file is added or False if a file with the same |name| and |sha1|
405        already exists. """
406    # Parse the file name.
407    (platform, version, type, channel) = self._parse_name(name)
408
409    if not isinstance(size, int):
410      size = int(size)
411    if not isinstance(last_modified, datetime.datetime):
412      last_modified = parse_date(last_modified)
413
414    # Validate arguments.
415    self._validate_args(platform, version, type, size, last_modified, sha1)
416
417    # Find the existing version record.
418    version_idx = -1
419    for i in range(0, len(self._data[platform]['versions'])):
420      if self._data[platform]['versions'][i]['cef_version'] == version:
421        # Check the version record.
422        self._print('add_file: Check %s %s' % (platform, version))
423        version_idx = i
424        break
425
426    if version_idx == -1:
427      # Add a new version record.
428      self._print('add_file: Add %s %s %s' % (platform, version, channel))
429      self._data[platform]['versions'].append({
430          'cef_version': version,
431          'chromium_version': self.get_chromium_version(version),
432          'channel': channel,
433          'files': []
434      })
435      version_idx = len(self._data[platform]['versions']) - 1
436
437    # Find the existing file record with matching type.
438    file_changed = True
439    for i in range(0,
440                   len(self._data[platform]['versions'][version_idx]['files'])):
441      if self._data[platform]['versions'][version_idx]['files'][i][
442          'type'] == type:
443        existing_sha1 = self._data[platform]['versions'][version_idx]['files'][
444            i]['sha1']
445        if existing_sha1 != sha1:
446          # Remove the existing file record.
447          self._print('  Remove %s %s' % (name, existing_sha1))
448          del self._data[platform]['versions'][version_idx]['files'][i]
449        else:
450          file_changed = False
451        break
452
453    if file_changed:
454      # Add a new file record.
455      self._print('  Add %s %s' % (name, sha1))
456      self._data[platform]['versions'][version_idx]['files'].append({
457          'type': type,
458          'name': name,
459          'size': size,
460          'last_modified': last_modified,
461          'sha1': sha1
462      })
463
464      # Sort file records by last_modified.
465      # This is necessary for _sort_versions() to function correctly.
466      self._data[platform]['versions'][version_idx]['files'] = \
467          self._sort_files(self._data[platform]['versions'][version_idx]['files'])
468
469    return file_changed
470
471  def get_files(self, platform=None, version=None, type=None):
472    """ Return the files that match the input parameters.
473        All parameters are optional. Version will do partial matching. """
474    results = []
475
476    if platform is None:
477      platforms = self._data.keys()
478    else:
479      platforms = [platform]
480
481    for platform in platforms:
482      for version_obj in self._data[platform]['versions']:
483        if version is None or version_obj['cef_version'].find(version) == 0:
484          for file_obj in version_obj['files']:
485            if type is None or type == file_obj['type']:
486              result_obj = file_obj
487              # Add additional metadata.
488              result_obj['platform'] = platform
489              result_obj['cef_version'] = version_obj['cef_version']
490              result_obj['chromium_version'] = version_obj['chromium_version']
491              result_obj['channel'] = version_obj['channel']
492              results.append(result_obj)
493
494    return results
495
496  def get_versions(self, platform):
497    """ Return all versions for the specified |platform|. """
498    return self._data[platform]['versions']
499