• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import json
6import logging
7import os
8
9from py_utils import cloud_storage
10from dependency_manager import archive_info
11from dependency_manager import cloud_storage_info
12from dependency_manager import dependency_info
13from dependency_manager import exceptions
14from dependency_manager import local_path_info
15from dependency_manager import uploader
16
17
18class BaseConfig(object):
19  """A basic config class for use with the DependencyManager.
20
21  Initiated with a json file in the following format:
22
23            {  "config_type": "BaseConfig",
24               "dependencies": {
25                 "dep_name1": {
26                   "cloud_storage_base_folder": "base_folder1",
27                   "cloud_storage_bucket": "bucket1",
28                   "file_info": {
29                     "platform1": {
30                        "cloud_storage_hash": "hash_for_platform1",
31                        "download_path": "download_path111",
32                        "version_in_cs": "1.11.1.11."
33                        "local_paths": ["local_path1110", "local_path1111"]
34                      },
35                      "platform2": {
36                        "cloud_storage_hash": "hash_for_platform2",
37                        "download_path": "download_path2",
38                        "local_paths": ["local_path20", "local_path21"]
39                      },
40                      ...
41                   }
42                 },
43                 "dependency_name_2": {
44                    ...
45                 },
46                  ...
47              }
48            }
49
50    Required fields: "dependencies" and "config_type".
51                     Note that config_type must be "BaseConfig"
52
53    Assumptions:
54        "cloud_storage_base_folder" is a top level folder in the given
55          "cloud_storage_bucket" where all of the dependency files are stored
56          at "dependency_name"_"cloud_storage_hash".
57
58        "download_path" and all paths in "local_paths" are relative to the
59          config file's location.
60
61        All or none of the following cloud storage related fields must be
62          included in each platform dictionary:
63          "cloud_storage_hash", "download_path", "cs_remote_path"
64
65        "version_in_cs" is an optional cloud storage field, but is dependent
66          on the above cloud storage related fields.
67
68
69    Also note that platform names are often of the form os_architechture.
70    Ex: "win_AMD64"
71
72    More information on the fields can be found in dependencies_info.py
73  """
74  def __init__(self, file_path, writable=False):
75    """ Initialize a BaseConfig for the DependencyManager.
76
77    Args:
78        writable: False: This config will be used to lookup information.
79                  True: This config will be used to update information.
80
81        file_path: Path to a file containing a json dictionary in the expected
82                   json format for this config class. Base format expected:
83
84                   { "config_type": config_type,
85                     "dependencies": dependencies_dict }
86
87                   config_type: must match the return value of GetConfigType.
88                   dependencies: A dictionary with the information needed to
89                       create dependency_info instances for the given
90                       dependencies.
91
92                   See dependency_info.py for more information.
93    """
94    self._config_path = file_path
95    self._writable = writable
96    self._pending_uploads = []
97    if not self._config_path:
98      raise ValueError('Must supply config file path.')
99    if not os.path.exists(self._config_path):
100      if not writable:
101        raise exceptions.EmptyConfigError(file_path)
102      self._config_data = {}
103      self._WriteConfigToFile(self._config_path, dependencies=self._config_data)
104    else:
105      with open(file_path, 'r') as f:
106        config_data = json.load(f)
107      if not config_data:
108        raise exceptions.EmptyConfigError(file_path)
109      config_type = config_data.pop('config_type', None)
110      if config_type != self.GetConfigType():
111        raise ValueError(
112            'Supplied config_type (%s) is not the expected type (%s) in file '
113            '%s' % (config_type, self.GetConfigType(), file_path))
114      self._config_data = config_data.get('dependencies', {})
115
116  def IterDependencyInfo(self):
117    """ Yields a DependencyInfo for each dependency/platform pair.
118
119    Raises:
120        ReadWriteError: If called when the config is writable.
121        ValueError: If any of the dependencies contain partial information for
122            downloading from cloud_storage. (See dependency_info.py)
123    """
124    if self._writable:
125      raise exceptions.ReadWriteError(
126          'Trying to read dependency info from a  writable config. File for '
127          'config: %s' % self._config_path)
128    base_path = os.path.dirname(self._config_path)
129    for dependency in self._config_data:
130      dependency_dict = self._config_data.get(dependency)
131      platforms_dict = dependency_dict.get('file_info', {})
132      for platform in platforms_dict:
133        platform_info = platforms_dict.get(platform)
134
135        local_info = None
136        local_paths = platform_info.get('local_paths', [])
137        if local_paths:
138          paths = []
139          for path in local_paths:
140            path = self._FormatPath(path)
141            paths.append(os.path.abspath(os.path.join(base_path, path)))
142          local_info = local_path_info.LocalPathInfo(paths)
143
144        cs_info = None
145        cs_bucket = dependency_dict.get('cloud_storage_bucket')
146        cs_base_folder = dependency_dict.get('cloud_storage_base_folder', '')
147        download_path = platform_info.get('download_path')
148        if download_path:
149          download_path = self._FormatPath(download_path)
150          download_path = os.path.abspath(
151              os.path.join(base_path, download_path))
152
153          cs_hash = platform_info.get('cloud_storage_hash')
154          if not cs_hash:
155            raise exceptions.ConfigError(
156                'Dependency %s has cloud storage info on platform %s, but is '
157                'missing a cloud storage hash.', dependency, platform)
158          cs_remote_path = self._CloudStorageRemotePath(
159              dependency, cs_hash, cs_base_folder)
160          version_in_cs = platform_info.get('version_in_cs')
161
162          zip_info = None
163          path_within_archive = platform_info.get('path_within_archive')
164          if path_within_archive:
165            unzip_path = os.path.abspath(
166                os.path.join(os.path.dirname(download_path),
167                             '%s_%s_%s' % (dependency, platform, cs_hash)))
168            stale_unzip_path_glob = os.path.abspath(
169                os.path.join(os.path.dirname(download_path),
170                             '%s_%s_%s' % (dependency, platform,
171                                           '[0-9a-f]' * 40)))
172            zip_info = archive_info.ArchiveInfo(
173                download_path, unzip_path, path_within_archive,
174                stale_unzip_path_glob)
175
176          cs_info = cloud_storage_info.CloudStorageInfo(
177              cs_bucket, cs_hash, download_path, cs_remote_path,
178              version_in_cs=version_in_cs, archive_info=zip_info)
179
180        dep_info = dependency_info.DependencyInfo(
181            dependency, platform, self._config_path,
182            local_path_info=local_info, cloud_storage_info=cs_info)
183        yield dep_info
184
185  @classmethod
186  def GetConfigType(cls):
187    return 'BaseConfig'
188
189  @property
190  def config_path(self):
191    return self._config_path
192
193  def AddNewDependency(
194      self, dependency, cloud_storage_base_folder, cloud_storage_bucket):
195    self._ValidateIsConfigWritable()
196    if dependency in self:
197      raise ValueError('Config already contains dependency %s' % dependency)
198    self._config_data[dependency] = {
199        'cloud_storage_base_folder': cloud_storage_base_folder,
200        'cloud_storage_bucket': cloud_storage_bucket,
201        'file_info': {},
202    }
203
204  def SetDownloadPath(self, dependency, platform, download_path):
205    self._ValidateIsConfigWritable()
206    if not dependency in self:
207      raise ValueError('Config does not contain dependency %s' % dependency)
208    platform_dicts = self._config_data[dependency]['file_info']
209    if platform not in platform_dicts:
210      platform_dicts[platform] = {}
211    platform_dicts[platform]['download_path'] = download_path
212
213  def AddCloudStorageDependencyUpdateJob(
214      self, dependency, platform, dependency_path, version=None,
215      execute_job=True):
216    """Update the file downloaded from cloud storage for a dependency/platform.
217
218    Upload a new file to cloud storage for the given dependency and platform
219    pair and update the cloud storage hash and the version for the given pair.
220
221    Example usage:
222      The following should update the default platform for 'dep_name':
223          UpdateCloudStorageDependency('dep_name', 'default', 'path/to/file')
224
225      The following should update both the mac and win platforms for 'dep_name',
226      or neither if either update fails:
227          UpdateCloudStorageDependency(
228              'dep_name', 'mac_x86_64', 'path/to/mac/file', execute_job=False)
229          UpdateCloudStorageDependency(
230              'dep_name', 'win_AMD64', 'path/to/win/file', execute_job=False)
231          ExecuteUpdateJobs()
232
233    Args:
234      dependency: The dependency to update.
235      platform: The platform to update the dependency info for.
236      dependency_path: Path to the new dependency to be used.
237      version: Version of the updated dependency, for checking future updates
238          against.
239      execute_job: True if the config should be written to disk and the file
240          should be uploaded to cloud storage after the update. False if
241          multiple updates should be performed atomically. Must call
242          ExecuteUpdateJobs after all non-executed jobs are added to complete
243          the update.
244
245    Raises:
246      ReadWriteError: If the config was not initialized as writable, or if
247          |execute_job| is True but the config has update jobs still pending
248          execution.
249      ValueError: If no information exists in the config for |dependency| on
250          |platform|.
251    """
252    self._ValidateIsConfigUpdatable(
253        execute_job=execute_job, dependency=dependency, platform=platform)
254    cs_hash = cloud_storage.CalculateHash(dependency_path)
255    if version:
256      self._SetPlatformData(dependency, platform, 'version_in_cs', version)
257    self._SetPlatformData(dependency, platform, 'cloud_storage_hash', cs_hash)
258
259    cs_base_folder = self._GetPlatformData(
260        dependency, platform, 'cloud_storage_base_folder')
261    cs_bucket = self._GetPlatformData(
262        dependency, platform, 'cloud_storage_bucket')
263    cs_remote_path = self._CloudStorageRemotePath(
264        dependency, cs_hash, cs_base_folder)
265    self._pending_uploads.append(uploader.CloudStorageUploader(
266        cs_bucket, cs_remote_path, dependency_path))
267    if execute_job:
268      self.ExecuteUpdateJobs()
269
270  def ExecuteUpdateJobs(self, force=False):
271    """Write all config changes to the config_path specified in __init__.
272
273    Upload all files pending upload and then write the updated config to
274    file. Attempt to remove all uploaded files on failure.
275
276    Args:
277      force: True if files should be uploaded to cloud storage even if a
278          file already exists in the upload location.
279
280    Returns:
281      True: if the config was dirty and the upload succeeded.
282      False: if the config was not dirty.
283
284    Raises:
285      CloudStorageUploadConflictError: If |force| is False and the potential
286          upload location of a file already exists.
287      CloudStorageError: If copying an existing file to the backup location
288          or uploading a new file fails.
289    """
290    self._ValidateIsConfigUpdatable()
291    if not self._IsDirty():
292      logging.info('ExecuteUpdateJobs called on clean config')
293      return False
294    if not self._pending_uploads:
295      logging.debug('No files needing upload.')
296    else:
297      try:
298        for item_pending_upload in self._pending_uploads:
299          item_pending_upload.Upload(force)
300        self._WriteConfigToFile(self._config_path, self._config_data)
301        self._pending_uploads = []
302      except:
303        # Attempt to rollback the update in any instance of failure, even user
304        # interrupt via Ctrl+C; but don't consume the exception.
305        logging.error('Update failed, attempting to roll it back.')
306        for upload_item in reversed(self._pending_uploads):
307          upload_item.Rollback()
308        raise
309    return True
310
311  def GetVersion(self, dependency, platform):
312    """Return the Version information for the given dependency."""
313    return self._GetPlatformData(
314        dependency, platform, data_type='version_in_cs')
315
316  def __contains__(self, dependency):
317    """ Returns whether this config contains |dependency|
318
319    Args:
320      dependency: the string name of dependency
321    """
322    return dependency in self._config_data
323
324  def _IsDirty(self):
325    with open(self._config_path, 'r') as fstream:
326      curr_config_data = json.load(fstream)
327    curr_config_data = curr_config_data.get('dependencies', {})
328    return self._config_data != curr_config_data
329
330  def _SetPlatformData(self, dependency, platform, data_type, data):
331    self._ValidateIsConfigWritable()
332    dependency_dict = self._config_data.get(dependency, {})
333    platform_dict = dependency_dict.get('file_info', {}).get(platform)
334    if not platform_dict:
335      raise ValueError('No platform data for platform %s on dependency %s' %
336                       (platform, dependency))
337    if (data_type == 'cloud_storage_bucket' or
338        data_type == 'cloud_storage_base_folder'):
339      self._config_data[dependency][data_type] = data
340    else:
341      self._config_data[dependency]['file_info'][platform][data_type] = data
342
343  def _GetPlatformData(self, dependency, platform, data_type=None):
344    dependency_dict = self._config_data.get(dependency, {})
345    if not dependency_dict:
346      raise ValueError('Dependency %s is not in config.' % dependency)
347    platform_dict = dependency_dict.get('file_info', {}).get(platform)
348    if not platform_dict:
349      raise ValueError('No platform data for platform %s on dependency %s' %
350                       (platform, dependency))
351    if data_type:
352      if (data_type == 'cloud_storage_bucket' or
353          data_type == 'cloud_storage_base_folder'):
354        return dependency_dict.get(data_type)
355      return platform_dict.get(data_type)
356    return platform_dict
357
358  def _ValidateIsConfigUpdatable(
359      self, execute_job=False, dependency=None, platform=None):
360    self._ValidateIsConfigWritable()
361    if self._IsDirty() and execute_job:
362      raise exceptions.ReadWriteError(
363          'A change has already been made to this config. Either call without'
364          'using the execute_job option or first call ExecuteUpdateJobs().')
365    if dependency and not self._config_data.get(dependency):
366      raise ValueError('Cannot update information because dependency %s does '
367                       'not exist.' % dependency)
368    if platform and not self._GetPlatformData(dependency, platform):
369      raise ValueError('No dependency info is available for the given '
370                       'dependency: %s' % dependency)
371
372  def _ValidateIsConfigWritable(self):
373    if not self._writable:
374      raise exceptions.ReadWriteError(
375          'Trying to update the information from a read-only config. '
376          'File for config: %s' % self._config_path)
377
378  @staticmethod
379  def _CloudStorageRemotePath(dependency, cs_hash, cs_base_folder):
380    cs_remote_file = '%s_%s' % (dependency, cs_hash)
381    cs_remote_path = cs_remote_file if not cs_base_folder else (
382        '%s/%s' % (cs_base_folder, cs_remote_file))
383    return cs_remote_path
384
385  @classmethod
386  def _FormatPath(cls, file_path):
387    """ Format |file_path| for the current file system.
388
389    We may be downloading files for another platform, so paths must be
390    downloadable on the current system.
391    """
392    if not file_path:
393      return file_path
394    if os.path.sep != '\\':
395      return file_path.replace('\\', os.path.sep)
396    elif os.path.sep != '/':
397      return file_path.replace('/', os.path.sep)
398    return file_path
399
400  @classmethod
401  def _WriteConfigToFile(cls, file_path, dependencies=None):
402    json_dict = cls._GetJsonDict(dependencies)
403    file_dir = os.path.dirname(file_path)
404    if not os.path.exists(file_dir):
405      os.makedirs(file_dir)
406    with open(file_path, 'w') as outfile:
407      json.dump(
408          json_dict, outfile, indent=2, sort_keys=True, separators=(',', ': '))
409    return json_dict
410
411  @classmethod
412  def _GetJsonDict(cls, dependencies=None):
413    dependencies = dependencies or {}
414    json_dict = {'config_type': cls.GetConfigType(),
415                 'dependencies': dependencies}
416    return json_dict
417