1# !/usr/bin/env python3 2# 3# Copyright (C) 2024 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Generate NOTICE.xml.gz of a partition. 19Usage example: 20 gen_notice_xml.py --output_file out/soong/.intermediate/.../NOTICE.xml.gz \ 21 --metadata out/soong/compliance-metadata/aosp_cf_x86_64_phone/compliance-metadata.db \ 22 --partition system \ 23 --product_out out/target/vsoc_x86_64 \ 24 --soong_out out/soong 25""" 26 27import argparse 28import compliance_metadata 29import google.protobuf.text_format as text_format 30import gzip 31import hashlib 32import metadata_file_pb2 33import os 34import queue 35import xml.sax.saxutils 36 37 38FILE_HEADER = '''\ 39<?xml version="1.0" encoding="utf-8"?> 40<licenses> 41''' 42FILE_FOOTER = '''\ 43</licenses> 44''' 45 46 47def get_args(): 48 parser = argparse.ArgumentParser() 49 parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.') 50 parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') 51 parser.add_argument('--output_file', required=True, help='The path of the generated NOTICE.xml.gz file.') 52 parser.add_argument('--partition', required=True, help='The name of partition for which the NOTICE.xml.gz is generated.') 53 parser.add_argument('--metadata', required=True, help='The path of compliance metadata DB file.') 54 parser.add_argument('--product_out', required=True, help='The path of PRODUCT_OUT, e.g. out/target/product/vsoc_x86_64.') 55 parser.add_argument('--soong_out', required=True, help='The path of Soong output directory, e.g. out/soong') 56 57 return parser.parse_args() 58 59 60def log(*info): 61 if args.verbose: 62 for i in info: 63 print(i) 64 65 66def new_file_name_tag(file_metadata, package_name, content_id): 67 file_path = file_metadata['installed_file'].removeprefix(args.product_out) 68 lib = 'Android' 69 if package_name: 70 lib = package_name 71 return f'<file-name contentId="{content_id}" lib="{lib}">{file_path}</file-name>\n' 72 73 74def new_file_content_tag(content_id, license_text): 75 escaped_license_text = xml.sax.saxutils.escape(license_text, {'\t': '	', '\n': '
', '\r': '
'}) 76 return f'<file-content contentId="{content_id}"><![CDATA[{escaped_license_text}]]></file-content>\n\n' 77 78def get_metadata_file_path(file_metadata): 79 """Search for METADATA file of a package and return its path.""" 80 metadata_path = '' 81 if file_metadata['module_path']: 82 metadata_path = file_metadata['module_path'] 83 elif file_metadata['kernel_module_copy_files']: 84 metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0]) 85 86 while metadata_path and not os.path.exists(metadata_path + '/METADATA'): 87 metadata_path = os.path.dirname(metadata_path) 88 89 return metadata_path 90 91def md5_file_content(filepath): 92 h = hashlib.md5() 93 with open(filepath, 'rb') as f: 94 h.update(f.read()) 95 return h.hexdigest() 96 97def get_transitive_static_dep_modules(installed_file_metadata, db): 98 # Find all transitive static dep files of the installed files 99 q = queue.Queue() 100 if installed_file_metadata['static_dep_files']: 101 for f in installed_file_metadata['static_dep_files'].split(' '): 102 q.put(f) 103 if installed_file_metadata['whole_static_dep_files']: 104 for f in installed_file_metadata['whole_static_dep_files'].split(' '): 105 q.put(f) 106 107 static_dep_files = {} 108 while not q.empty(): 109 dep_file = q.get() 110 if dep_file in static_dep_files: 111 # It has been processed 112 continue 113 114 soong_module = db.get_soong_module_of_built_file(dep_file) 115 if not soong_module: 116 continue 117 118 static_dep_files[dep_file] = soong_module 119 120 if soong_module['static_dep_files']: 121 for f in soong_module['static_dep_files'].split(' '): 122 if f not in static_dep_files: 123 q.put(f) 124 if soong_module['whole_static_dep_files']: 125 for f in soong_module['whole_static_dep_files'].split(' '): 126 if f not in static_dep_files: 127 q.put(f) 128 129 return static_dep_files.values() 130 131def main(): 132 global args 133 args = get_args() 134 log('Args:', vars(args)) 135 136 global db 137 db = compliance_metadata.MetadataDb(args.metadata) 138 if args.debug: 139 db.dump_debug_db(os.path.dirname(args.output_file) + '/compliance-metadata-debug.db') 140 141 # NOTICE.xml 142 notice_xml_file_path = os.path.dirname(args.output_file) + '/NOTICE.xml' 143 with open(notice_xml_file_path, 'w', encoding="utf-8") as notice_xml_file: 144 notice_xml_file.write(FILE_HEADER) 145 146 all_license_files = {} 147 for metadata in db.get_installed_file_in_dir(args.product_out + '/' + args.partition): 148 soong_module = db.get_soong_module_of_installed_file(metadata['installed_file']) 149 if soong_module: 150 metadata.update(soong_module) 151 else: 152 # For make modules soong_module_type should be empty 153 metadata['soong_module_type'] = '' 154 metadata['static_dep_files'] = '' 155 metadata['whole_static_dep_files'] = '' 156 157 installed_file_metadata_list = [metadata] 158 if args.partition in ('vendor', 'product', 'system_ext'): 159 # For transitive static dependencies of an installed file, make it as if an installed file are 160 # also created from static dependency modules whose licenses are also collected 161 static_dep_modules = get_transitive_static_dep_modules(metadata, db) 162 for dep in static_dep_modules: 163 dep['installed_file'] = metadata['installed_file'] 164 installed_file_metadata_list.append(dep) 165 166 for installed_file_metadata in installed_file_metadata_list: 167 package_name = 'Android' 168 licenses = {} 169 if installed_file_metadata['module_path']: 170 metadata_file_path = get_metadata_file_path(installed_file_metadata) 171 if metadata_file_path: 172 proto = metadata_file_pb2.Metadata() 173 with open(metadata_file_path + '/METADATA', 'rt') as f: 174 text_format.Parse(f.read(), proto) 175 if proto.name: 176 package_name = proto.name 177 if proto.third_party and proto.third_party.version: 178 if proto.third_party.version.startswith('v'): 179 package_name = package_name + '_' + proto.third_party.version 180 else: 181 package_name = package_name + '_v_' + proto.third_party.version 182 else: 183 package_name = metadata_file_path 184 if metadata_file_path.startswith('external/'): 185 package_name = metadata_file_path.removeprefix('external/') 186 187 # Every license file is in a <file-content> element 188 licenses = db.get_module_licenses(installed_file_metadata.get('name', ''), installed_file_metadata['module_path']) 189 190 # Installed file is from PRODUCT_COPY_FILES 191 elif metadata['product_copy_files']: 192 licenses['unused_name'] = metadata['license_text'] 193 194 # Installed file is generated by the platform in builds 195 elif metadata['is_platform_generated']: 196 licenses['unused_name'] = metadata['license_text'] 197 198 if licenses: 199 # Each value is a space separated filepath list 200 for license_files in licenses.values(): 201 if not license_files: 202 continue 203 for filepath in license_files.split(' '): 204 if filepath not in all_license_files: 205 all_license_files[filepath] = md5_file_content(filepath) 206 md5 = all_license_files[filepath] 207 notice_xml_file.write(new_file_name_tag(installed_file_metadata, package_name, md5)) 208 209 # Licenses 210 processed_md5 = [] 211 for filepath, md5 in all_license_files.items(): 212 if md5 not in processed_md5: 213 processed_md5.append(md5) 214 with open(filepath, 'rt', errors='backslashreplace') as f: 215 notice_xml_file.write(new_file_content_tag(md5, f.read())) 216 217 notice_xml_file.write(FILE_FOOTER) 218 219 # NOTICE.xml.gz 220 with open(notice_xml_file_path, 'rb') as notice_xml_file, gzip.open(args.output_file, 'wb') as gz_file: 221 gz_file.writelines(notice_xml_file) 222 223if __name__ == '__main__': 224 main() 225