# !/usr/bin/env python3 # # Copyright (C) 2024 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Generate NOTICE.xml.gz of a partition. Usage example: gen_notice_xml.py --output_file out/soong/.intermediate/.../NOTICE.xml.gz \ --metadata out/soong/compliance-metadata/aosp_cf_x86_64_phone/compliance-metadata.db \ --partition system \ --product_out out/target/vsoc_x86_64 \ --soong_out out/soong """ import argparse import compliance_metadata import google.protobuf.text_format as text_format import gzip import hashlib import metadata_file_pb2 import os import queue import xml.sax.saxutils FILE_HEADER = '''\ ''' FILE_FOOTER = '''\ ''' def get_args(): parser = argparse.ArgumentParser() parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.') parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') parser.add_argument('--output_file', required=True, help='The path of the generated NOTICE.xml.gz file.') parser.add_argument('--partition', required=True, help='The name of partition for which the NOTICE.xml.gz is generated.') parser.add_argument('--metadata', required=True, help='The path of compliance metadata DB file.') parser.add_argument('--product_out', required=True, help='The path of PRODUCT_OUT, e.g. out/target/product/vsoc_x86_64.') parser.add_argument('--soong_out', required=True, help='The path of Soong output directory, e.g. out/soong') return parser.parse_args() def log(*info): if args.verbose: for i in info: print(i) def new_file_name_tag(file_metadata, package_name, content_id): file_path = file_metadata['installed_file'].removeprefix(args.product_out) lib = 'Android' if package_name: lib = package_name return f'{file_path}\n' def new_file_content_tag(content_id, license_text): escaped_license_text = xml.sax.saxutils.escape(license_text, {'\t': ' ', '\n': ' ', '\r': ' '}) return f'\n\n' def get_metadata_file_path(file_metadata): """Search for METADATA file of a package and return its path.""" metadata_path = '' if file_metadata['module_path']: metadata_path = file_metadata['module_path'] elif file_metadata['kernel_module_copy_files']: metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0]) while metadata_path and not os.path.exists(metadata_path + '/METADATA'): metadata_path = os.path.dirname(metadata_path) return metadata_path def md5_file_content(filepath): h = hashlib.md5() with open(filepath, 'rb') as f: h.update(f.read()) return h.hexdigest() def get_transitive_static_dep_modules(installed_file_metadata, db): # Find all transitive static dep files of the installed files q = queue.Queue() if installed_file_metadata['static_dep_files']: for f in installed_file_metadata['static_dep_files'].split(' '): q.put(f) if installed_file_metadata['whole_static_dep_files']: for f in installed_file_metadata['whole_static_dep_files'].split(' '): q.put(f) static_dep_files = {} while not q.empty(): dep_file = q.get() if dep_file in static_dep_files: # It has been processed continue soong_module = db.get_soong_module_of_built_file(dep_file) if not soong_module: continue static_dep_files[dep_file] = soong_module if soong_module['static_dep_files']: for f in soong_module['static_dep_files'].split(' '): if f not in static_dep_files: q.put(f) if soong_module['whole_static_dep_files']: for f in soong_module['whole_static_dep_files'].split(' '): if f not in static_dep_files: q.put(f) return static_dep_files.values() def main(): global args args = get_args() log('Args:', vars(args)) global db db = compliance_metadata.MetadataDb(args.metadata) if args.debug: db.dump_debug_db(os.path.dirname(args.output_file) + '/compliance-metadata-debug.db') # NOTICE.xml notice_xml_file_path = os.path.dirname(args.output_file) + '/NOTICE.xml' with open(notice_xml_file_path, 'w', encoding="utf-8") as notice_xml_file: notice_xml_file.write(FILE_HEADER) all_license_files = {} for metadata in db.get_installed_file_in_dir(args.product_out + '/' + args.partition): soong_module = db.get_soong_module_of_installed_file(metadata['installed_file']) if soong_module: metadata.update(soong_module) else: # For make modules soong_module_type should be empty metadata['soong_module_type'] = '' metadata['static_dep_files'] = '' metadata['whole_static_dep_files'] = '' installed_file_metadata_list = [metadata] if args.partition in ('vendor', 'product', 'system_ext'): # For transitive static dependencies of an installed file, make it as if an installed file are # also created from static dependency modules whose licenses are also collected static_dep_modules = get_transitive_static_dep_modules(metadata, db) for dep in static_dep_modules: dep['installed_file'] = metadata['installed_file'] installed_file_metadata_list.append(dep) for installed_file_metadata in installed_file_metadata_list: package_name = 'Android' licenses = {} if installed_file_metadata['module_path']: metadata_file_path = get_metadata_file_path(installed_file_metadata) if metadata_file_path: proto = metadata_file_pb2.Metadata() with open(metadata_file_path + '/METADATA', 'rt') as f: text_format.Parse(f.read(), proto) if proto.name: package_name = proto.name if proto.third_party and proto.third_party.version: if proto.third_party.version.startswith('v'): package_name = package_name + '_' + proto.third_party.version else: package_name = package_name + '_v_' + proto.third_party.version else: package_name = metadata_file_path if metadata_file_path.startswith('external/'): package_name = metadata_file_path.removeprefix('external/') # Every license file is in a element licenses = db.get_module_licenses(installed_file_metadata.get('name', ''), installed_file_metadata['module_path']) # Installed file is from PRODUCT_COPY_FILES elif metadata['product_copy_files']: licenses['unused_name'] = metadata['license_text'] # Installed file is generated by the platform in builds elif metadata['is_platform_generated']: licenses['unused_name'] = metadata['license_text'] if licenses: # Each value is a space separated filepath list for license_files in licenses.values(): if not license_files: continue for filepath in license_files.split(' '): if filepath not in all_license_files: all_license_files[filepath] = md5_file_content(filepath) md5 = all_license_files[filepath] notice_xml_file.write(new_file_name_tag(installed_file_metadata, package_name, md5)) # Licenses processed_md5 = [] for filepath, md5 in all_license_files.items(): if md5 not in processed_md5: processed_md5.append(md5) with open(filepath, 'rt', errors='backslashreplace') as f: notice_xml_file.write(new_file_content_tag(md5, f.read())) notice_xml_file.write(FILE_FOOTER) # NOTICE.xml.gz with open(notice_xml_file_path, 'rb') as notice_xml_file, gzip.open(args.output_file, 'wb') as gz_file: gz_file.writelines(notice_xml_file) if __name__ == '__main__': main()