# Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import json import logging import os from py_utils import cloud_storage from dependency_manager import archive_info from dependency_manager import cloud_storage_info from dependency_manager import dependency_info from dependency_manager import exceptions from dependency_manager import local_path_info from dependency_manager import uploader class BaseConfig(object): """A basic config class for use with the DependencyManager. Initiated with a json file in the following format: { "config_type": "BaseConfig", "dependencies": { "dep_name1": { "cloud_storage_base_folder": "base_folder1", "cloud_storage_bucket": "bucket1", "file_info": { "platform1": { "cloud_storage_hash": "hash_for_platform1", "download_path": "download_path111", "version_in_cs": "1.11.1.11." "local_paths": ["local_path1110", "local_path1111"] }, "platform2": { "cloud_storage_hash": "hash_for_platform2", "download_path": "download_path2", "local_paths": ["local_path20", "local_path21"] }, ... } }, "dependency_name_2": { ... }, ... } } Required fields: "dependencies" and "config_type". Note that config_type must be "BaseConfig" Assumptions: "cloud_storage_base_folder" is a top level folder in the given "cloud_storage_bucket" where all of the dependency files are stored at "dependency_name"_"cloud_storage_hash". "download_path" and all paths in "local_paths" are relative to the config file's location. All or none of the following cloud storage related fields must be included in each platform dictionary: "cloud_storage_hash", "download_path", "cs_remote_path" "version_in_cs" is an optional cloud storage field, but is dependent on the above cloud storage related fields. Also note that platform names are often of the form os_architechture. Ex: "win_AMD64" More information on the fields can be found in dependencies_info.py """ def __init__(self, file_path, writable=False): """ Initialize a BaseConfig for the DependencyManager. Args: writable: False: This config will be used to lookup information. True: This config will be used to update information. file_path: Path to a file containing a json dictionary in the expected json format for this config class. Base format expected: { "config_type": config_type, "dependencies": dependencies_dict } config_type: must match the return value of GetConfigType. dependencies: A dictionary with the information needed to create dependency_info instances for the given dependencies. See dependency_info.py for more information. """ self._config_path = file_path self._writable = writable self._pending_uploads = [] if not self._config_path: raise ValueError('Must supply config file path.') if not os.path.exists(self._config_path): if not writable: raise exceptions.EmptyConfigError(file_path) self._config_data = {} self._WriteConfigToFile(self._config_path, dependencies=self._config_data) else: with open(file_path, 'r') as f: config_data = json.load(f) if not config_data: raise exceptions.EmptyConfigError(file_path) config_type = config_data.pop('config_type', None) if config_type != self.GetConfigType(): raise ValueError( 'Supplied config_type (%s) is not the expected type (%s) in file ' '%s' % (config_type, self.GetConfigType(), file_path)) self._config_data = config_data.get('dependencies', {}) def IterDependencyInfo(self): """ Yields a DependencyInfo for each dependency/platform pair. Raises: ReadWriteError: If called when the config is writable. ValueError: If any of the dependencies contain partial information for downloading from cloud_storage. (See dependency_info.py) """ if self._writable: raise exceptions.ReadWriteError( 'Trying to read dependency info from a writable config. File for ' 'config: %s' % self._config_path) base_path = os.path.dirname(self._config_path) for dependency in self._config_data: dependency_dict = self._config_data.get(dependency) platforms_dict = dependency_dict.get('file_info', {}) for platform in platforms_dict: platform_info = platforms_dict.get(platform) local_info = None local_paths = platform_info.get('local_paths', []) if local_paths: paths = [] for path in local_paths: path = self._FormatPath(path) paths.append(os.path.abspath(os.path.join(base_path, path))) local_info = local_path_info.LocalPathInfo(paths) cs_info = None cs_bucket = dependency_dict.get('cloud_storage_bucket') cs_base_folder = dependency_dict.get('cloud_storage_base_folder', '') download_path = platform_info.get('download_path') if download_path: download_path = self._FormatPath(download_path) download_path = os.path.abspath( os.path.join(base_path, download_path)) cs_hash = platform_info.get('cloud_storage_hash') if not cs_hash: raise exceptions.ConfigError( 'Dependency %s has cloud storage info on platform %s, but is ' 'missing a cloud storage hash.', dependency, platform) cs_remote_path = self._CloudStorageRemotePath( dependency, cs_hash, cs_base_folder) version_in_cs = platform_info.get('version_in_cs') zip_info = None path_within_archive = platform_info.get('path_within_archive') if path_within_archive: unzip_path = os.path.abspath( os.path.join(os.path.dirname(download_path), '%s_%s_%s' % (dependency, platform, cs_hash))) stale_unzip_path_glob = os.path.abspath( os.path.join(os.path.dirname(download_path), '%s_%s_%s' % (dependency, platform, '[0-9a-f]' * 40))) zip_info = archive_info.ArchiveInfo( download_path, unzip_path, path_within_archive, stale_unzip_path_glob) cs_info = cloud_storage_info.CloudStorageInfo( cs_bucket, cs_hash, download_path, cs_remote_path, version_in_cs=version_in_cs, archive_info=zip_info) dep_info = dependency_info.DependencyInfo( dependency, platform, self._config_path, local_path_info=local_info, cloud_storage_info=cs_info) yield dep_info @classmethod def GetConfigType(cls): return 'BaseConfig' @property def config_path(self): return self._config_path def AddCloudStorageDependencyUpdateJob( self, dependency, platform, dependency_path, version=None, execute_job=True): """Update the file downloaded from cloud storage for a dependency/platform. Upload a new file to cloud storage for the given dependency and platform pair and update the cloud storage hash and the version for the given pair. Example usage: The following should update the default platform for 'dep_name': UpdateCloudStorageDependency('dep_name', 'default', 'path/to/file') The following should update both the mac and win platforms for 'dep_name', or neither if either update fails: UpdateCloudStorageDependency( 'dep_name', 'mac_x86_64', 'path/to/mac/file', execute_job=False) UpdateCloudStorageDependency( 'dep_name', 'win_AMD64', 'path/to/win/file', execute_job=False) ExecuteUpdateJobs() Args: dependency: The dependency to update. platform: The platform to update the dependency info for. dependency_path: Path to the new dependency to be used. version: Version of the updated dependency, for checking future updates against. execute_job: True if the config should be written to disk and the file should be uploaded to cloud storage after the update. False if multiple updates should be performed atomically. Must call ExecuteUpdateJobs after all non-executed jobs are added to complete the update. Raises: ReadWriteError: If the config was not initialized as writable, or if |execute_job| is True but the config has update jobs still pending execution. ValueError: If no information exists in the config for |dependency| on |platform|. """ self._ValidateIsConfigUpdatable( execute_job=execute_job, dependency=dependency, platform=platform) cs_hash = cloud_storage.CalculateHash(dependency_path) if version: self._SetPlatformData(dependency, platform, 'version_in_cs', version) self._SetPlatformData(dependency, platform, 'cloud_storage_hash', cs_hash) cs_base_folder = self._GetPlatformData( dependency, platform, 'cloud_storage_base_folder') cs_bucket = self._GetPlatformData( dependency, platform, 'cloud_storage_bucket') cs_remote_path = self._CloudStorageRemotePath( dependency, cs_hash, cs_base_folder) self._pending_uploads.append(uploader.CloudStorageUploader( cs_bucket, cs_remote_path, dependency_path)) if execute_job: self.ExecuteUpdateJobs() def ExecuteUpdateJobs(self, force=False): """Write all config changes to the config_path specified in __init__. Upload all files pending upload and then write the updated config to file. Attempt to remove all uploaded files on failure. Args: force: True if files should be uploaded to cloud storage even if a file already exists in the upload location. Returns: True: if the config was dirty and the upload succeeded. False: if the config was not dirty. Raises: CloudStorageUploadConflictError: If |force| is False and the potential upload location of a file already exists. CloudStorageError: If copying an existing file to the backup location or uploading a new file fails. """ self._ValidateIsConfigUpdatable() if not self._IsDirty(): logging.info('ExecuteUpdateJobs called on clean config') return False if not self._pending_uploads: logging.debug('No files needing upload.') else: try: for item_pending_upload in self._pending_uploads: item_pending_upload.Upload(force) self._WriteConfigToFile(self._config_path, self._config_data) self._pending_uploads = [] except: # Attempt to rollback the update in any instance of failure, even user # interrupt via Ctrl+C; but don't consume the exception. logging.error('Update failed, attempting to roll it back.') for upload_item in reversed(self._pending_uploads): upload_item.Rollback() raise return True def GetVersion(self, dependency, platform): """Return the Version information for the given dependency.""" return self._GetPlatformData( dependency, platform, data_type='version_in_cs') def _IsDirty(self): with open(self._config_path, 'r') as fstream: curr_config_data = json.load(fstream) curr_config_data = curr_config_data.get('dependencies', {}) return self._config_data != curr_config_data def _SetPlatformData(self, dependency, platform, data_type, data): self._ValidateIsConfigWritable() dependency_dict = self._config_data.get(dependency, {}) platform_dict = dependency_dict.get('file_info', {}).get(platform) if not platform_dict: raise ValueError('No platform data for platform %s on dependency %s' % (platform, dependency)) if (data_type == 'cloud_storage_bucket' or data_type == 'cloud_storage_base_folder'): self._config_data[dependency][data_type] = data else: self._config_data[dependency]['file_info'][platform][data_type] = data def _GetPlatformData(self, dependency, platform, data_type=None): dependency_dict = self._config_data.get(dependency, {}) if not dependency_dict: raise ValueError('Dependency %s is not in config.' % dependency) platform_dict = dependency_dict.get('file_info', {}).get(platform) if not platform_dict: raise ValueError('No platform data for platform %s on dependency %s' % (platform, dependency)) if data_type: if (data_type == 'cloud_storage_bucket' or data_type == 'cloud_storage_base_folder'): return dependency_dict.get(data_type) return platform_dict.get(data_type) return platform_dict def _ValidateIsConfigUpdatable( self, execute_job=False, dependency=None, platform=None): self._ValidateIsConfigWritable() if self._IsDirty() and execute_job: raise exceptions.ReadWriteError( 'A change has already been made to this config. Either call without' 'using the execute_job option or first call ExecuteUpdateJobs().') if dependency and not self._config_data.get(dependency): raise ValueError('Cannot update information because dependency %s does ' 'not exist.' % dependency) if platform and not self._GetPlatformData(dependency, platform): raise ValueError('No dependency info is available for the given ' 'dependency: %s' % dependency) def _ValidateIsConfigWritable(self): if not self._writable: raise exceptions.ReadWriteError( 'Trying to update the information from a read-only config. ' 'File for config: %s' % self._config_path) @staticmethod def _CloudStorageRemotePath(dependency, cs_hash, cs_base_folder): cs_remote_file = '%s_%s' % (dependency, cs_hash) cs_remote_path = cs_remote_file if not cs_base_folder else ( '%s/%s' % (cs_base_folder, cs_remote_file)) return cs_remote_path @classmethod def _FormatPath(cls, file_path): """ Format |file_path| for the current file system. We may be downloading files for another platform, so paths must be downloadable on the current system. """ if not file_path: return file_path if os.path.sep != '\\': return file_path.replace('\\', os.path.sep) elif os.path.sep != '/': return file_path.replace('/', os.path.sep) return file_path @classmethod def _WriteConfigToFile(cls, file_path, dependencies=None): json_dict = cls._GetJsonDict(dependencies) file_dir = os.path.dirname(file_path) if not os.path.exists(file_dir): os.makedirs(file_dir) with open(file_path, 'w') as outfile: json.dump( json_dict, outfile, indent=2, sort_keys=True, separators=(',', ': ')) return json_dict @classmethod def _GetJsonDict(cls, dependencies=None): dependencies = dependencies or {} json_dict = {'config_type': cls.GetConfigType(), 'dependencies': dependencies} return json_dict