1#!/usr/bin/env python 2# 3# Copyright (C) 2017 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18import argparse 19import glob 20import logging 21import os 22import shutil 23import subprocess 24import tempfile 25import xml.etree.ElementTree as xml_tree 26 27import utils 28 29 30class GPLChecker(object): 31 """Checks that all GPL projects in a VNDK snapshot have released sources. 32 33 Makes sure that the current source tree have the sources for all GPL 34 prebuilt libraries in a specified VNDK snapshot version. 35 """ 36 MANIFEST_XML = utils.MANIFEST_FILE_NAME 37 MODULE_PATHS_TXT = utils.MODULE_PATHS_FILE_NAME 38 39 def __init__(self, install_dir, android_build_top, temp_artifact_dir, 40 remote_git): 41 """GPLChecker constructor. 42 43 Args: 44 install_dir: string, absolute path to the prebuilts/vndk/v{version} 45 directory where the build files will be generated. 46 android_build_top: string, absolute path to ANDROID_BUILD_TOP 47 temp_artifact_dir: string, temp directory to hold build artifacts 48 fetched from Android Build server. 49 remote_git: string, remote name to fetch and check if the revision of 50 VNDK snapshot is included in the source if it is not in the current 51 git repository. 52 """ 53 self._android_build_top = android_build_top 54 self._install_dir = install_dir 55 self._remote_git = remote_git 56 self._manifest_file = os.path.join(temp_artifact_dir, 57 self.MANIFEST_XML) 58 self._notice_files_dir = os.path.join(install_dir, 59 utils.NOTICE_FILES_DIR_PATH) 60 61 if not os.path.isfile(self._manifest_file): 62 raise RuntimeError( 63 '{manifest} not found at {manifest_file}'.format( 64 manifest=self.MANIFEST_XML, 65 manifest_file=self._manifest_file)) 66 67 def _parse_module_paths(self): 68 """Parses the module_paths.txt files into a dictionary, 69 70 Returns: 71 module_paths: dict, e.g. {libfoo.so: some/path/here} 72 """ 73 module_paths = dict() 74 for file in utils.find(self._install_dir, [self.MODULE_PATHS_TXT]): 75 file_path = os.path.join(self._install_dir, file) 76 with open(file_path, 'r') as f: 77 for line in f.read().strip().split('\n'): 78 paths = line.split(' ') 79 if len(paths) > 1: 80 if paths[0] not in module_paths: 81 module_paths[paths[0]] = paths[1] 82 return module_paths 83 84 def _parse_manifest(self): 85 """Parses manifest.xml file and returns list of 'project' tags.""" 86 87 root = xml_tree.parse(self._manifest_file).getroot() 88 return root.findall('project') 89 90 def _get_revision(self, module_path, manifest_projects): 91 """Returns revision value recorded in manifest.xml for given project. 92 93 Args: 94 module_path: string, project path relative to ANDROID_BUILD_TOP 95 manifest_projects: list of xml_tree.Element, list of 'project' tags 96 """ 97 revision = None 98 for project in manifest_projects: 99 path = project.get('path') 100 if module_path.startswith(path): 101 revision = project.get('revision') 102 break 103 return revision 104 105 def _check_revision_exists(self, revision, git_project_path): 106 """Checks whether a revision is found in a git project of current tree. 107 108 Args: 109 revision: string, revision value recorded in manifest.xml 110 git_project_path: string, path relative to ANDROID_BUILD_TOP 111 """ 112 path = utils.join_realpath(self._android_build_top, git_project_path) 113 114 def _check_rev_list(revision): 115 """Checks whether revision is reachable from HEAD of git project.""" 116 117 logging.info('Checking if revision {rev} exists in {proj}'.format( 118 rev=revision, proj=git_project_path)) 119 try: 120 cmd = [ 121 'git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision) 122 ] 123 output = utils.check_output(cmd).strip() 124 except subprocess.CalledProcessError as error: 125 logging.error('Error: {}'.format(error)) 126 return False 127 else: 128 if output: 129 logging.debug( 130 '{proj} does not have the following revisions: {rev}'. 131 format(proj=git_project_path, rev=output)) 132 return False 133 else: 134 logging.info( 135 'Found revision {rev} in project {proj}'.format( 136 rev=revision, proj=git_project_path)) 137 return True 138 139 if not _check_rev_list(revision): 140 # VNDK snapshots built from a *-release branch will have merge 141 # CLs in the manifest because the *-dev branch is merged to the 142 # *-release branch periodically. In order to extract the 143 # revision relevant to the source of the git_project_path, 144 # we fetch the *-release branch and get the revision of the 145 # parent commit with FETCH_HEAD^2. 146 logging.info( 147 'Checking if the parent of revision {rev} exists in {proj}'. 148 format(rev=revision, proj=git_project_path)) 149 try: 150 cmd = ['git', '-C', path, 'fetch', self._remote_git, revision] 151 utils.check_call(cmd) 152 cmd = ['git', '-C', path, 'rev-parse', 'FETCH_HEAD^2'] 153 parent_revision = utils.check_output(cmd).strip() 154 except subprocess.CalledProcessError as error: 155 logging.error( 156 'Failed to get parent of revision {rev} from "{remote}": ' 157 '{err}'.format( 158 rev=revision, remote=self._remote_git, err=error)) 159 logging.error('Try --remote to manually set remote name') 160 raise 161 else: 162 if not _check_rev_list(parent_revision): 163 return False 164 165 return True 166 167 def check_gpl_projects(self): 168 """Checks that all GPL projects have released sources. 169 170 Raises: 171 ValueError: There are GPL projects with unreleased sources. 172 """ 173 logging.info('Starting license check for GPL projects...') 174 175 notice_files = glob.glob('{}/*'.format(self._notice_files_dir)) 176 if len(notice_files) == 0: 177 raise RuntimeError('No license files found in {}'.format( 178 self._notice_files_dir)) 179 180 gpl_projects = [] 181 pattern = 'GENERAL PUBLIC LICENSE' 182 for notice_file_path in notice_files: 183 with open(notice_file_path, 'r') as notice_file: 184 if pattern in notice_file.read(): 185 lib_name = os.path.splitext( 186 os.path.basename(notice_file_path))[0] 187 gpl_projects.append(lib_name) 188 189 if not gpl_projects: 190 logging.info('No GPL projects found.') 191 return 192 193 logging.info('GPL projects found: {}'.format(', '.join(gpl_projects))) 194 195 module_paths = self._parse_module_paths() 196 manifest_projects = self._parse_manifest() 197 released_projects = [] 198 unreleased_projects = [] 199 200 for lib in gpl_projects: 201 if lib in module_paths: 202 module_path = module_paths[lib] 203 revision = self._get_revision(module_path, manifest_projects) 204 if not revision: 205 raise RuntimeError( 206 'No project found for {path} in {manifest}'.format( 207 path=module_path, manifest=self.MANIFEST_XML)) 208 revision_exists = self._check_revision_exists( 209 revision, module_path) 210 if not revision_exists: 211 unreleased_projects.append((lib, module_path)) 212 else: 213 released_projects.append((lib, module_path)) 214 else: 215 raise RuntimeError( 216 'No module path was found for {lib} in {module_paths}'. 217 format(lib=lib, module_paths=self.MODULE_PATHS_TXT)) 218 219 if released_projects: 220 logging.info('Released GPL projects: {}'.format(released_projects)) 221 222 if unreleased_projects: 223 raise ValueError( 224 ('FAIL: The following GPL projects have NOT been released in ' 225 'current tree: {}'.format(unreleased_projects))) 226 227 logging.info('PASS: All GPL projects have source in current tree.') 228 229 230def get_args(): 231 parser = argparse.ArgumentParser() 232 parser.add_argument( 233 'vndk_version', 234 type=int, 235 help='VNDK snapshot version to check, e.g. "27".') 236 parser.add_argument('-b', '--branch', help='Branch to pull manifest from.') 237 parser.add_argument('--build', help='Build number to pull manifest from.') 238 parser.add_argument( 239 '--remote', 240 default='aosp', 241 help=('Remote name to fetch and check if the revision of VNDK snapshot ' 242 'is included in the source to conform GPL license. default=aosp')) 243 parser.add_argument( 244 '-v', 245 '--verbose', 246 action='count', 247 default=0, 248 help='Increase output verbosity, e.g. "-v", "-vv".') 249 return parser.parse_args() 250 251 252def main(): 253 """For local testing purposes. 254 255 Note: VNDK snapshot must be already installed under 256 prebuilts/vndk/v{version}. 257 """ 258 ANDROID_BUILD_TOP = utils.get_android_build_top() 259 PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 260 'prebuilts/vndk') 261 262 args = get_args() 263 vndk_version = args.vndk_version 264 install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) 265 remote = args.remote 266 if not os.path.isdir(install_dir): 267 raise ValueError( 268 'Please provide valid VNDK version. {} does not exist.' 269 .format(install_dir)) 270 utils.set_logging_config(args.verbose) 271 272 temp_artifact_dir = tempfile.mkdtemp() 273 os.chdir(temp_artifact_dir) 274 manifest_pattern = 'manifest_{}.xml'.format(args.build) 275 manifest_dest = os.path.join(temp_artifact_dir, utils.MANIFEST_FILE_NAME) 276 logging.info('Fetching {file} from {branch} (bid: {build})'.format( 277 file=manifest_pattern, branch=args.branch, build=args.build)) 278 utils.fetch_artifact(args.branch, args.build, manifest_pattern, 279 manifest_dest) 280 281 license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, 282 temp_artifact_dir, remote) 283 try: 284 license_checker.check_gpl_projects() 285 except ValueError as error: 286 logging.error('Error: {}'.format(error)) 287 raise 288 finally: 289 logging.info( 290 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir)) 291 shutil.rmtree(temp_artifact_dir) 292 293 logging.info('Done.') 294 295 296if __name__ == '__main__': 297 main() 298