1#!/usr/bin/env python3 2# 3# Copyright (C) 2017 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18import argparse 19import glob 20import logging 21import os 22import shutil 23import subprocess 24import tempfile 25import xml.etree.ElementTree as xml_tree 26 27import utils 28 29 30class GPLChecker(object): 31 """Checks that all GPL projects in a VNDK snapshot have released sources. 32 33 Makes sure that the current source tree have the sources for all GPL 34 prebuilt libraries in a specified VNDK snapshot version. 35 """ 36 MANIFEST_XML = utils.MANIFEST_FILE_NAME 37 MODULE_PATHS_TXT = utils.MODULE_PATHS_FILE_NAME 38 39 def __init__(self, install_dir, android_build_top, temp_artifact_dir, 40 remote_git): 41 """GPLChecker constructor. 42 43 Args: 44 install_dir: string, absolute path to the prebuilts/vndk/v{version} 45 directory where the build files will be generated. 46 android_build_top: string, absolute path to ANDROID_BUILD_TOP 47 temp_artifact_dir: string, temp directory to hold build artifacts 48 fetched from Android Build server. 49 remote_git: string, remote name to fetch and check if the revision of 50 VNDK snapshot is included in the source if it is not in the current 51 git repository. 52 """ 53 self._android_build_top = android_build_top 54 self._install_dir = install_dir 55 self._remote_git = remote_git 56 self._manifest_file = os.path.join(temp_artifact_dir, 57 self.MANIFEST_XML) 58 self._notice_files_dir = os.path.join(install_dir, 59 utils.NOTICE_FILES_DIR_PATH) 60 61 if not os.path.isfile(self._manifest_file): 62 raise RuntimeError( 63 '{manifest} not found at {manifest_file}'.format( 64 manifest=self.MANIFEST_XML, 65 manifest_file=self._manifest_file)) 66 67 def _parse_module_paths(self): 68 """Parses the module_paths.txt files into a dictionary, 69 70 Returns: 71 module_paths: dict, e.g. {libfoo.so: some/path/here} 72 """ 73 module_paths = dict() 74 for file in utils.find(self._install_dir, [self.MODULE_PATHS_TXT]): 75 file_path = os.path.join(self._install_dir, file) 76 with open(file_path, 'r') as f: 77 for line in f.read().strip().split('\n'): 78 paths = line.split(' ') 79 if len(paths) > 1: 80 if paths[0] not in module_paths: 81 module_paths[paths[0]] = paths[1] 82 return module_paths 83 84 def _parse_manifest(self): 85 """Parses manifest.xml file and returns list of 'project' tags.""" 86 87 root = xml_tree.parse(self._manifest_file).getroot() 88 return root.findall('project') 89 90 def _get_revision(self, module_path, manifest_projects): 91 """Returns revision value recorded in manifest.xml for given project. 92 93 Args: 94 module_path: string, project path relative to ANDROID_BUILD_TOP 95 manifest_projects: list of xml_tree.Element, list of 'project' tags 96 """ 97 revision = None 98 for project in manifest_projects: 99 path = project.get('path') 100 if module_path.startswith(path): 101 revision = project.get('revision') 102 break 103 return revision 104 105 def _check_revision_exists(self, revision, git_project_path): 106 """Checks whether a revision is found in a git project of current tree. 107 108 Args: 109 revision: string, revision value recorded in manifest.xml 110 git_project_path: string, path relative to ANDROID_BUILD_TOP 111 """ 112 path = utils.join_realpath(self._android_build_top, git_project_path) 113 114 def _check_rev_list(revision): 115 """Checks whether revision is reachable from HEAD of git project.""" 116 117 logging.info('Checking if revision {rev} exists in {proj}'.format( 118 rev=revision, proj=git_project_path)) 119 try: 120 cmd = [ 121 'git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision) 122 ] 123 output = utils.check_output(cmd).strip() 124 except subprocess.CalledProcessError as error: 125 logging.error('Error: {}'.format(error)) 126 return False 127 else: 128 if output: 129 logging.debug( 130 '{proj} does not have the following revisions: {rev}'. 131 format(proj=git_project_path, rev=output)) 132 return False 133 else: 134 logging.info( 135 'Found revision {rev} in project {proj}'.format( 136 rev=revision, proj=git_project_path)) 137 return True 138 139 def _get_2nd_parent_if_merge_commit(revision): 140 """Checks if the commit is merge commit. 141 142 Returns: 143 revision: string, the 2nd parent which is the merged commit. 144 If the commit is not a merge commit, returns None. 145 """ 146 logging.info( 147 'Checking if the parent of revision {rev} exists in {proj}'. 148 format(rev=revision, proj=git_project_path)) 149 try: 150 cmd = [ 151 'git', '-C', path, 'rev-parse', '--verify', 152 '{}^2'.format(revision)] 153 parent_revision = utils.check_output(cmd).strip() 154 except subprocess.CalledProcessError as error: 155 logging.error( 156 'Failed to get parent of revision {rev} from "{remote}": ' 157 '{err}'.format( 158 rev=revision, remote=self._remote_git, err=error)) 159 logging.error('{} is not a merge commit and must be included ' 160 'in the current branch'.format(revision)) 161 return None 162 else: 163 return parent_revision 164 165 if _check_rev_list(revision): 166 return True 167 168 # VNDK snapshots built from a *-release branch will have merge 169 # CLs in the manifest because the *-dev branch is merged to the 170 # *-release branch periodically. In order to extract the 171 # revision relevant to the source of the git_project_path, 172 # we find the parent of the merge commit. 173 try: 174 cmd = ['git', '-C', path, 'fetch', self._remote_git, revision] 175 utils.check_call(cmd) 176 except subprocess.CalledProcessError as error: 177 logging.error( 178 'Failed to fetch revision {rev} from "{remote}": ' 179 '{err}'.format( 180 rev=revision, remote=self._remote_git, err=error)) 181 logging.error('Try --remote to manually set remote name') 182 raise 183 184 parent_revision = _get_2nd_parent_if_merge_commit(revision) 185 while True: 186 if not parent_revision: 187 return False 188 if _check_rev_list(parent_revision): 189 return True 190 parent_revision = _get_2nd_parent_if_merge_commit(parent_revision) 191 192 def check_gpl_projects(self): 193 """Checks that all GPL projects have released sources. 194 195 Raises: 196 ValueError: There are GPL projects with unreleased sources. 197 """ 198 logging.info('Starting license check for GPL projects...') 199 200 notice_files = glob.glob('{}/*'.format(self._notice_files_dir)) 201 if len(notice_files) == 0: 202 raise RuntimeError('No license files found in {}'.format( 203 self._notice_files_dir)) 204 205 gpl_projects = [] 206 pattern = 'GENERAL PUBLIC LICENSE' 207 for notice_file_path in notice_files: 208 with open(notice_file_path, 'r') as notice_file: 209 if pattern in notice_file.read(): 210 lib_name = os.path.splitext( 211 os.path.basename(notice_file_path))[0] 212 gpl_projects.append(lib_name) 213 214 if not gpl_projects: 215 logging.info('No GPL projects found.') 216 return 217 218 logging.info('GPL projects found: {}'.format(', '.join(gpl_projects))) 219 220 module_paths = self._parse_module_paths() 221 manifest_projects = self._parse_manifest() 222 released_projects = [] 223 unreleased_projects = [] 224 225 for lib in gpl_projects: 226 if lib in module_paths: 227 module_path = module_paths[lib] 228 revision = self._get_revision(module_path, manifest_projects) 229 if not revision: 230 raise RuntimeError( 231 'No project found for {path} in {manifest}'.format( 232 path=module_path, manifest=self.MANIFEST_XML)) 233 revision_exists = self._check_revision_exists( 234 revision, module_path) 235 if not revision_exists: 236 unreleased_projects.append((lib, module_path)) 237 else: 238 released_projects.append((lib, module_path)) 239 else: 240 raise RuntimeError( 241 'No module path was found for {lib} in {module_paths}'. 242 format(lib=lib, module_paths=self.MODULE_PATHS_TXT)) 243 244 if released_projects: 245 logging.info('Released GPL projects: {}'.format(released_projects)) 246 247 if unreleased_projects: 248 raise ValueError( 249 ('FAIL: The following GPL projects have NOT been released in ' 250 'current tree: {}'.format(unreleased_projects))) 251 252 logging.info('PASS: All GPL projects have source in current tree.') 253 254 255def get_args(): 256 parser = argparse.ArgumentParser() 257 parser.add_argument( 258 'vndk_version', 259 type=utils.vndk_version_int, 260 help='VNDK snapshot version to check, e.g. "{}".'.format( 261 utils.MINIMUM_VNDK_VERSION)) 262 parser.add_argument('-b', '--branch', help='Branch to pull manifest from.') 263 parser.add_argument('--build', help='Build number to pull manifest from.') 264 parser.add_argument( 265 '--remote', 266 default='aosp', 267 help=('Remote name to fetch and check if the revision of VNDK snapshot ' 268 'is included in the source to conform GPL license. default=aosp')) 269 parser.add_argument( 270 '-v', 271 '--verbose', 272 action='count', 273 default=0, 274 help='Increase output verbosity, e.g. "-v", "-vv".') 275 return parser.parse_args() 276 277 278def main(): 279 """For local testing purposes. 280 281 Note: VNDK snapshot must be already installed under 282 prebuilts/vndk/v{version}. 283 """ 284 ANDROID_BUILD_TOP = utils.get_android_build_top() 285 PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 286 'prebuilts/vndk') 287 288 args = get_args() 289 vndk_version = args.vndk_version 290 install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) 291 remote = args.remote 292 if not os.path.isdir(install_dir): 293 raise ValueError( 294 'Please provide valid VNDK version. {} does not exist.' 295 .format(install_dir)) 296 utils.set_logging_config(args.verbose) 297 298 temp_artifact_dir = tempfile.mkdtemp() 299 os.chdir(temp_artifact_dir) 300 manifest_pattern = 'manifest_{}.xml'.format(args.build) 301 manifest_dest = os.path.join(temp_artifact_dir, utils.MANIFEST_FILE_NAME) 302 logging.info('Fetching {file} from {branch} (bid: {build})'.format( 303 file=manifest_pattern, branch=args.branch, build=args.build)) 304 utils.fetch_artifact(args.branch, args.build, manifest_pattern, 305 manifest_dest) 306 307 license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, 308 temp_artifact_dir, remote) 309 try: 310 license_checker.check_gpl_projects() 311 except ValueError as error: 312 logging.error('Error: {}'.format(error)) 313 raise 314 finally: 315 logging.info( 316 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir)) 317 shutil.rmtree(temp_artifact_dir) 318 319 logging.info('Done.') 320 321 322if __name__ == '__main__': 323 main() 324