1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3# Copyright (c) 2021 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15""" 16 17Usage: gen_notice_file --output-image-name system \ 18 --notice-file-root xx/NOTICE_FILE \ 19 --notice-file-install-path xx/system \ 20 --output-title notice_title_string 21 22Generate the project notice files, including both text and xml files. 23 24""" 25from collections import defaultdict 26import argparse 27import hashlib 28import os 29import os.path 30import sys 31import gzip 32import shutil 33import glob 34import re 35 36sys.path.append( 37 os.path.dirname(os.path.dirname(os.path.dirname( 38 os.path.abspath(__file__))))) 39from scripts.util import build_utils # noqa: E402 40from scripts.util.file_utils import write_json_file, read_json_file # noqa: E402 41 42XML_ESCAPE_TABLE = { 43 "&": "&", 44 '"': """, 45 "'": "'", 46 ">": ">", 47 "<": "<", 48} 49 50 51def copy_static_library_notices(options, depfiles: list): 52 valid_notices = [] 53 basenames = [] 54 for file in build_utils.get_all_files(options.static_library_notice_dir): 55 if os.stat(file).st_size == 0: 56 continue 57 if not file.endswith('.a.txt'): 58 continue 59 notice_file_name = os.path.basename(file) 60 if file not in basenames: 61 basenames.append(notice_file_name) 62 valid_notices.append(file) 63 depfiles.append(file) 64 65 for file in valid_notices: 66 if options.image_name == "system": 67 if options.target_cpu == "arm64" or options.target_cpu == "x64": 68 install_dir = "system/lib64" 69 elif options.target_cpu == "arm": 70 install_dir = "system/lib" 71 else: 72 continue 73 elif options.image_name == "sdk": 74 install_dir = "toolchains/lib" 75 elif options.image_name == "ndk": 76 install_dir = "sysroot/usr/lib" 77 else: 78 continue 79 dest = os.path.join(options.notice_root_dir, install_dir, 80 os.path.basename(file)) 81 os.makedirs(os.path.dirname(dest), exist_ok=True) 82 shutil.copyfile(file, dest) 83 if os.path.isfile("{}.json".format(file)): 84 os.makedirs(os.path.dirname("{}.json".format(dest)), exist_ok=True) 85 shutil.copyfile("{}.json".format(file), "{}.json".format(dest)) 86 87 88def write_file(file: str, string: str): 89 print(string, file=file) 90 91 92def compute_hash(file: str): 93 sha256 = hashlib.sha256() 94 with open(file, 'rb') as file_fd: 95 for line in file_fd: 96 sha256.update(line) 97 return sha256.hexdigest() 98 99 100def get_entity(text: str): 101 return "".join(XML_ESCAPE_TABLE.get(c, c) for c in text) 102 103 104def generate_txt_notice_files(file_hash: str, input_dir: str, output_filename: str, 105 notice_title: str): 106 with open(output_filename, "w") as output_file: 107 write_file(output_file, notice_title) 108 for value in file_hash: 109 write_file(output_file, '=' * 60) 110 write_file(output_file, "Notices for file(s):") 111 for filename in value: 112 write_file( 113 output_file, '/{}'.format( 114 re.sub('.txt.*', '', 115 os.path.relpath(filename, input_dir)))) 116 write_file(output_file, '-' * 60) 117 write_file(output_file, "Notices for software(s):") 118 software_list = [] 119 for filename in value: 120 json_filename = '{}.json'.format(filename) 121 contents = read_json_file(json_filename) 122 if contents is not None and contents not in software_list: 123 software_list.append(contents) 124 for contens_value in software_list: 125 notice_source_path = contens_value[0].get('Path').strip() 126 software_name = contens_value[0].get('Software').strip() 127 128 write_file(output_file, "Software: {}".format(software_name)) 129 write_file(output_file, "Path: {}".format(notice_source_path)) 130 write_file(output_file, '-' * 60) 131 with open(value[0], errors='ignore') as temp_file_hd: 132 write_file(output_file, temp_file_hd.read()) 133 134 135def generate_xml_notice_files(files_with_same_hash: dict, input_dir: str, 136 output_filename: str): 137 id_table = {} 138 for file_key in files_with_same_hash.keys(): 139 for filename in files_with_same_hash[file_key]: 140 id_table[filename] = file_key 141 142 with open(output_filename, "w") as output_file: 143 write_file(output_file, '<?xml version="1.0" encoding="utf-8"?>') 144 write_file(output_file, "<licenses>") 145 146 # Flatten the lists into a single filename list 147 sorted_filenames = sorted(id_table.keys()) 148 149 # write out a table of contents 150 for filename in sorted_filenames: 151 stripped_filename = re.sub('.txt.*', '', 152 os.path.relpath(filename, input_dir)) 153 write_file( 154 output_file, '<file-name contentId="%s">%s</file-name>' % 155 (id_table.get(filename), stripped_filename)) 156 157 write_file(output_file, '') 158 write_file(output_file, '') 159 160 processed_file_keys = [] 161 # write the notice file lists 162 for filename in sorted_filenames: 163 file_key = id_table.get(filename) 164 if file_key in processed_file_keys: 165 continue 166 processed_file_keys.append(file_key) 167 168 with open(filename, errors='ignore') as temp_file_hd: 169 write_file( 170 output_file, 171 '<file-content contentId="{}"><![CDATA[{}]]></file-content>' 172 .format(file_key, get_entity(temp_file_hd.read()))) 173 write_file(output_file, '') 174 175 # write the file complete node. 176 write_file(output_file, "</licenses>") 177 178 179def compress_file_to_gz(src_file_name: str, gz_file_name: str): 180 with open(src_file_name, mode='rb') as src_file_fd: 181 with gzip.open(gz_file_name, mode='wb') as gz_file_fd: 182 gz_file_fd.writelines(src_file_fd) 183 184 185def handle_zipfile_notices(zip_file: str): 186 notice_file = '{}.txt'.format(zip_file[:-4]) 187 with build_utils.temp_dir() as tmp_dir: 188 build_utils.extract_all(zip_file, tmp_dir, no_clobber=False) 189 files = build_utils.get_all_files(tmp_dir) 190 contents = [] 191 for file in files: 192 with open(file, 'r') as fd: 193 data = fd.read() 194 if data not in contents: 195 contents.append(data) 196 with open(notice_file, 'w') as merged_notice: 197 merged_notice.write('\n\n'.join(contents)) 198 return notice_file 199 200 201def main(): 202 parser = argparse.ArgumentParser() 203 parser.add_argument('--image-name') 204 parser.add_argument('--collected-notice-zipfile', 205 action='append', 206 help='zipfile stors collected notice files') 207 parser.add_argument('--notice-root-dir', help='where notice files store') 208 parser.add_argument('--output-notice-txt', help='output notice.txt') 209 parser.add_argument('--output-notice-gz', help='output notice.txt') 210 parser.add_argument('--notice-title', help='title of notice.txt') 211 parser.add_argument('--static-library-notice-dir', 212 help='path to static library notice files') 213 parser.add_argument('--target-cpu', help='cpu arch') 214 parser.add_argument('--depfile', help='depfile') 215 parser.add_argument('--notice-module-info', 216 help='module info file for notice target') 217 parser.add_argument('--notice-install-dir', 218 help='install directories of notice file') 219 220 args = parser.parse_args() 221 222 notice_dir = args.notice_root_dir 223 depfiles = [] 224 if args.collected_notice_zipfile: 225 for zip_file in args.collected_notice_zipfile: 226 build_utils.extract_all(zip_file, notice_dir, no_clobber=False) 227 else: 228 depfiles += build_utils.get_all_files(notice_dir) 229 # Copy notice of static targets to notice_root_dir 230 if args.static_library_notice_dir: 231 copy_static_library_notices(args, depfiles) 232 233 zipfiles = glob.glob('{}/**/*.zip'.format(notice_dir), recursive=True) 234 235 txt_files = glob.glob('{}/**/*.txt'.format(notice_dir), recursive=True) 236 txt_files += glob.glob('{}/**/*.txt.?'.format(notice_dir), recursive=True) 237 238 outputs = [args.output_notice_txt, args.output_notice_gz] 239 if args.notice_module_info: 240 outputs.append(args.notice_module_info) 241 build_utils.call_and_write_depfile_if_stale( 242 lambda: do_merge_notice(args, zipfiles, txt_files), 243 args, 244 depfile_deps=depfiles, 245 input_paths=depfiles, 246 input_strings=args.notice_title + args.target_cpu, 247 output_paths=(outputs)) 248 249 250def do_merge_notice(args, zipfiles: str, txt_files: str): 251 notice_dir = args.notice_root_dir 252 notice_txt = args.output_notice_txt 253 notice_gz = args.output_notice_gz 254 notice_title = args.notice_title 255 256 if not notice_txt.endswith('.txt'): 257 raise Exception( 258 'Error: input variable output_notice_txt must ends with .txt') 259 if not notice_gz.endswith('.xml.gz'): 260 raise Exception( 261 'Error: input variable output_notice_gz must ends with .xml.gz') 262 263 notice_xml = notice_gz.replace('.gz', '') 264 265 files_with_same_hash = defaultdict(list) 266 for file in zipfiles: 267 txt_files.append(handle_zipfile_notices(file)) 268 269 for file in txt_files: 270 if os.stat(file).st_size == 0: 271 continue 272 file_hash = compute_hash(file) 273 files_with_same_hash[file_hash].append(file) 274 275 file_sets = [ 276 sorted(files_with_same_hash[hash]) 277 for hash in sorted(files_with_same_hash.keys()) 278 ] 279 280 if file_sets is not None: 281 generate_txt_notice_files(file_sets, notice_dir, notice_txt, 282 notice_title) 283 284 if files_with_same_hash is not None: 285 generate_xml_notice_files(files_with_same_hash, notice_dir, notice_xml) 286 compress_file_to_gz(notice_xml, args.output_notice_gz) 287 288 if args.notice_module_info: 289 module_install_info_list = [] 290 module_install_info = {} 291 module_install_info['type'] = 'notice' 292 module_install_info['source'] = args.output_notice_txt 293 module_install_info['install_enable'] = True 294 module_install_info['dest'] = [ 295 os.path.join(args.notice_install_dir, 296 os.path.basename(args.output_notice_txt)) 297 ] 298 module_install_info_list.append(module_install_info) 299 write_json_file(args.notice_module_info, module_install_info_list) 300 301 302if __name__ == "__main__": 303 main() 304