1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3# Copyright (c) 2021 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16from collections import defaultdict 17import argparse 18import hashlib 19import os 20import os.path 21import sys 22import gzip 23import shutil 24import glob 25import re 26import subprocess 27 28sys.path.append( 29 os.path.dirname(os.path.dirname(os.path.dirname( 30 os.path.abspath(__file__))))) 31from scripts.util import build_utils # noqa: E402 32from scripts.util.file_utils import write_json_file, read_json_file # noqa: E402 33 34xml_escape_table = { 35 "&": "&", 36 '"': """, 37 "'": "'", 38 ">": ">", 39 "<": "<", 40} 41 42 43def move_static_library_notices(options): 44 files = [] 45 dest = os.path.join(options.notice_root_dir, 'libs') 46 os.makedirs(dest, exist_ok=True) 47 static_dir = os.path.join(options.notice_root_dir, "static") 48 static_subdir = os.path.join(options.static_library_notice_dir, "libs") 49 if os.path.exists(static_dir): 50 files = build_utils.get_all_files(static_dir) 51 files.sort() 52 if os.path.exists(static_subdir): 53 other_files = build_utils.get_all_files(static_subdir) 54 other_files.sort() 55 files.extend(other_files) 56 if files: 57 for file in files: 58 file_name = os.path.basename(file) 59 if not file_name.startswith("lib"): 60 dest_file = os.path.join(dest, f"lib{file_name}") 61 else: 62 dest_file = os.path.join(dest, file_name) 63 shutil.copyfile(file, dest_file) 64 if os.path.isfile("{}.json".format(dest_file)): 65 os.makedirs(os.path.dirname("{}.json".format(dest_file)), exist_ok=True) 66 shutil.copyfile("{}.json".format(file), "{}.json".format(dest_file)) 67 if os.path.exists(static_dir): 68 shutil.rmtree(static_dir) 69 if os.path.exists(static_subdir): 70 shutil.rmtree(static_subdir) 71 72 73def copy_static_library_notices(options, depfiles: list): 74 valid_notices = [] 75 basenames = [] 76 # add sort method 77 files = build_utils.get_all_files(options.static_library_notice_dir) 78 files.sort() 79 for file in files: 80 if os.stat(file).st_size == 0: 81 continue 82 if not options.lite_product: 83 if not file.endswith('.a.txt'): 84 continue 85 elif not file.endswith('.txt'): 86 continue 87 notice_file_name = os.path.basename(file) 88 if options.lite_product: 89 if not notice_file_name.startswith("lib"): 90 file_dir = os.path.dirname(file) 91 lib_file = os.path.join(file_dir, f"lib{notice_file_name}") 92 os.rename(file, lib_file) 93 file = lib_file 94 if file not in basenames: 95 basenames.append(notice_file_name) 96 valid_notices.append(file) 97 depfiles.append(file) 98 99 for file in valid_notices: 100 if options.image_name == "system": 101 if options.target_cpu == "arm64" or options.target_cpu == "x64": 102 install_dir = "system/lib64" 103 elif options.target_cpu == "arm": 104 install_dir = "system/lib" 105 else: 106 continue 107 elif options.image_name == "sdk": 108 install_dir = "toolchains/lib" 109 elif options.image_name == "ndk": 110 install_dir = "sysroot/usr/lib" 111 else: 112 continue 113 dest = os.path.join(options.notice_root_dir, install_dir, 114 os.path.basename(file)) 115 os.makedirs(os.path.dirname(dest), exist_ok=True) 116 shutil.copyfile(file, dest) 117 if os.path.isfile("{}.json".format(file)): 118 os.makedirs(os.path.dirname("{}.json".format(dest)), exist_ok=True) 119 shutil.copyfile("{}.json".format(file), "{}.json".format(dest)) 120 121 122def write_file(file: str, string: str): 123 print(string, file=file) 124 125 126def compute_hash(file: str): 127 sha256 = hashlib.sha256() 128 with open(file, 'rb') as file_fd: 129 for line in file_fd: 130 sha256.update(line) 131 return sha256.hexdigest() 132 133 134def get_entity(text: str): 135 return "".join(xml_escape_table.get(c, c) for c in text) 136 137 138def generate_txt_notice_files(file_hash: str, input_dir: str, output_filename: str, 139 notice_title: str): 140 with open(output_filename, "w") as output_file: 141 write_file(output_file, notice_title) 142 for value in file_hash: 143 write_file(output_file, '=' * 60) 144 write_file(output_file, "Notices for file(s):") 145 for filename in value: 146 write_file( 147 output_file, '/{}'.format( 148 re.sub('.txt.*', '', 149 os.path.relpath(filename, input_dir)))) 150 write_file(output_file, '-' * 60) 151 write_file(output_file, "Notices for software(s):") 152 software_list = [] 153 for filename in value: 154 json_filename = '{}.json'.format(filename) 155 contents = read_json_file(json_filename) 156 if contents is not None and contents not in software_list: 157 software_list.append(contents) 158 software_dict = {} 159 for contents_value in software_list: 160 if len(contents_value) > 0: 161 for val in contents_value: 162 if val.get('Software'): 163 software_name = val.get('Software').strip() 164 if software_name not in software_dict: 165 software_dict[software_name] = {"_version": "", "_path": []} 166 else: 167 write_file(output_file, "Software: ") 168 if val.get('Version'): 169 version = val.get('Version').strip() 170 software_dict[software_name]["_version"] = version 171 else: 172 write_file(output_file, "Version: ") 173 if val.get('Path'): 174 notice_source_path = val.get('Path').strip() 175 software_dict[software_name]["_path"].append(notice_source_path) 176 for software, software_value in software_dict.items(): 177 write_file(output_file, f"Software: {software}") 178 write_file(output_file, f"Version: {software_value.get('_version')}") 179 if software_value.get("_path"): 180 for path in software_value.get("_path"): 181 write_file(output_file, f"Path: {path}") 182 write_file(output_file, '-' * 60) 183 with open(value[0], errors='ignore') as temp_file_hd: 184 write_file(output_file, temp_file_hd.read()) 185 186 187def generate_xml_notice_files(files_with_same_hash: dict, input_dir: str, 188 output_filename: str): 189 id_table = {} 190 for file_key in files_with_same_hash.keys(): 191 for filename in files_with_same_hash[file_key]: 192 id_table[filename] = file_key 193 with open(output_filename, "w") as output_file: 194 write_file(output_file, '<?xml version="1.0" encoding="utf-8"?>') 195 write_file(output_file, "<licenses>") 196 197 # Flatten the lists into a single filename list 198 sorted_filenames = sorted(id_table.keys()) 199 200 # write out a table of contents 201 for filename in sorted_filenames: 202 stripped_filename = re.sub('.txt.*', '', 203 os.path.relpath(filename, input_dir)) 204 write_file( 205 output_file, '<file-name content_id="%s">%s</file-name>' % 206 (id_table.get(filename), stripped_filename)) 207 208 write_file(output_file, '') 209 write_file(output_file, '') 210 211 processed_file_keys = [] 212 # write the notice file lists 213 for filename in sorted_filenames: 214 file_key = id_table.get(filename) 215 if file_key in processed_file_keys: 216 continue 217 processed_file_keys.append(file_key) 218 219 with open(filename, errors='ignore') as temp_file_hd: 220 write_file( 221 output_file, 222 '<file-content content_id="{}"><![CDATA[{}]]></file-content>' 223 .format(file_key, get_entity(temp_file_hd.read()))) 224 write_file(output_file, '') 225 226 # write the file complete node. 227 write_file(output_file, "</licenses>") 228 229 230def compress_file_to_gz(src_file_name: str, gz_file_name: str): 231 with open(src_file_name, mode='rb') as src_file_fd: 232 with gzip.open(gz_file_name, mode='wb') as gz_file_fd: 233 gz_file_fd.writelines(src_file_fd) 234 235 236def handle_zipfile_notices(zip_file: str): 237 notice_file = '{}.txt'.format(zip_file[:-4]) 238 with build_utils.temp_dir() as tmp_dir: 239 build_utils.extract_all(zip_file, tmp_dir, no_clobber=False) 240 files = build_utils.get_all_files(tmp_dir) 241 contents = [] 242 for file in files: 243 with open(file, 'r') as fd: 244 data = fd.read() 245 if data not in contents: 246 contents.append(data) 247 with open(notice_file, 'w') as merged_notice: 248 merged_notice.write('\n\n'.join(contents)) 249 return notice_file 250 251 252def do_merge_notice(args, zipfiles: str, txt_files: str): 253 notice_dir = args.notice_root_dir 254 notice_txt = args.output_notice_txt 255 notice_gz = args.output_notice_gz 256 notice_title = args.notice_title 257 258 if not notice_txt.endswith('.txt'): 259 raise Exception( 260 'Error: input variable output_notice_txt must ends with .txt') 261 if not notice_gz.endswith('.xml.gz'): 262 raise Exception( 263 'Error: input variable output_notice_gz must ends with .xml.gz') 264 265 notice_xml = notice_gz.replace('.gz', '') 266 267 files_with_same_hash = defaultdict(list) 268 for file in zipfiles: 269 txt_files.append(handle_zipfile_notices(file)) 270 271 for file in txt_files: 272 if os.stat(file).st_size == 0: 273 continue 274 file_hash = compute_hash(file) 275 files_with_same_hash[file_hash].append(file) 276 277 file_sets = [ 278 sorted(files_with_same_hash[hash]) 279 for hash in sorted(files_with_same_hash.keys()) 280 ] 281 282 if file_sets is not None: 283 generate_txt_notice_files(file_sets, notice_dir, notice_txt, 284 notice_title) 285 286 if files_with_same_hash is not None: 287 generate_xml_notice_files(files_with_same_hash, notice_dir, notice_xml) 288 compress_file_to_gz(notice_xml, args.output_notice_gz) 289 290 if args.notice_module_info: 291 module_install_info_list = [] 292 module_install_info = {} 293 module_install_info['type'] = 'notice' 294 module_install_info['source'] = args.output_notice_txt 295 module_install_info['install_enable'] = True 296 module_install_info['dest'] = [ 297 os.path.join(args.notice_install_dir, 298 os.path.basename(args.output_notice_txt)) 299 ] 300 module_install_info_list.append(module_install_info) 301 write_json_file(args.notice_module_info, module_install_info_list) 302 303 if args.lite_product: 304 current_dir_cmd = ['pwd'] 305 process = subprocess.Popen(current_dir_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 306 stdout, stderr = process.communicate(timeout=600) 307 current_dir = stdout.decode().strip() 308 dest = f"{current_dir}/system/etc/NOTICE.txt" 309 if os.path.isfile(notice_txt): 310 os.makedirs(os.path.dirname(dest), exist_ok=True) 311 shutil.copyfile(notice_txt, dest) 312 313 314def parse_args(): 315 """Parses command-line arguments.""" 316 parser = argparse.ArgumentParser() 317 parser.add_argument('--image-name') 318 parser.add_argument('--collected-notice-zipfile', 319 action='append', 320 help='zipfile stors collected notice files') 321 parser.add_argument('--notice-root-dir', help='where notice files store') 322 parser.add_argument('--output-notice-txt', help='output notice.txt') 323 parser.add_argument('--output-notice-gz', help='output notice.txt') 324 parser.add_argument('--notice-title', help='title of notice.txt') 325 parser.add_argument('--static-library-notice-dir', 326 help='path to static library notice files') 327 parser.add_argument('--target-cpu', help='cpu arch') 328 parser.add_argument('--depfile', help='depfile') 329 parser.add_argument('--notice-module-info', 330 help='module info file for notice target') 331 parser.add_argument('--notice-install-dir', 332 help='install directories of notice file') 333 parser.add_argument('--lite-product', help='', default="") 334 335 336 return parser.parse_args() 337 338 339def main(): 340 """Main function to merge and generate notice files.""" 341 args = parse_args() 342 343 notice_dir = args.notice_root_dir 344 depfiles = [] 345 if args.collected_notice_zipfile: 346 for zip_file in args.collected_notice_zipfile: 347 build_utils.extract_all(zip_file, notice_dir, no_clobber=False) 348 else: 349 depfiles += build_utils.get_all_files(notice_dir) 350 # Copy notice of static targets to notice_root_dir 351 if args.lite_product: 352 move_static_library_notices(args) 353 if args.static_library_notice_dir: 354 copy_static_library_notices(args, depfiles) 355 356 zipfiles = glob.glob('{}/**/*.zip'.format(notice_dir), recursive=True) 357 358 txt_files = glob.glob('{}/**/*.txt'.format(notice_dir), recursive=True) 359 txt_files += glob.glob('{}/**/*.txt.?'.format(notice_dir), recursive=True) 360 361 outputs = [args.output_notice_txt, args.output_notice_gz] 362 if args.notice_module_info: 363 outputs.append(args.notice_module_info) 364 build_utils.call_and_write_depfile_if_stale( 365 lambda: do_merge_notice(args, zipfiles, txt_files), 366 args, 367 depfile_deps=depfiles, 368 input_paths=depfiles, 369 input_strings=args.notice_title + args.target_cpu, 370 output_paths=(outputs)) 371 372 373if __name__ == "__main__": 374 main() 375 376