1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3# Copyright (c) 2021 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15""" 16 17Usage: gen_notice_file --output-image-name system \ 18 --notice-file-root xx/NOTICE_FILE \ 19 --notice-file-install-path xx/system \ 20 --output-title notice_title_string 21 22Generate the project notice files, including both text and xml files. 23 24""" 25from collections import defaultdict 26import argparse 27import hashlib 28import os 29import os.path 30import sys 31import gzip 32import shutil 33import glob 34import re 35 36sys.path.append( 37 os.path.dirname(os.path.dirname(os.path.dirname( 38 os.path.abspath(__file__))))) 39from scripts.util import build_utils # noqa: E402 40from scripts.util.file_utils import write_json_file, read_json_file # noqa: E402 41 42XML_ESCAPE_TABLE = { 43 "&": "&", 44 '"': """, 45 "'": "'", 46 ">": ">", 47 "<": "<", 48} 49 50 51def copy_static_library_notices(options, depfiles): 52 valid_notices = [] 53 basenames = [] 54 for file in build_utils.get_all_files(options.static_library_notice_dir): 55 if os.stat(file).st_size == 0: 56 continue 57 if not file.endswith('.a.txt'): 58 continue 59 notice_file_name = os.path.basename(file) 60 if file not in basenames: 61 basenames.append(notice_file_name) 62 valid_notices.append(file) 63 depfiles.append(file) 64 65 for file in valid_notices: 66 if options.image_name == "system": 67 if options.target_cpu == "arm64" or options.target_cpu == "x64": 68 install_dir = "system/lib64" 69 elif options.target_cpu == "arm": 70 install_dir = "system/lib" 71 else: 72 continue 73 elif options.image_name == "sdk": 74 install_dir = "toolchains/lib" 75 elif options.image_name == "ndk": 76 install_dir = "sysroot/usr/lib" 77 else: 78 continue 79 dest = os.path.join(options.notice_root_dir, install_dir, 80 os.path.basename(file)) 81 os.makedirs(os.path.dirname(dest), exist_ok=True) 82 shutil.copyfile(file, dest) 83 if os.path.isfile("{}.json".format(file)): 84 os.makedirs(os.path.dirname("{}.json".format(dest)), exist_ok=True) 85 shutil.copyfile("{}.json".format(file), "{}.json".format(dest)) 86 87 88def write_file(file, string): 89 print(string, file=file) 90 91 92def compute_hash(file): 93 sha256 = hashlib.sha256() 94 with open(file, 'rb') as file_fd: 95 for line in file_fd: 96 sha256.update(line) 97 return sha256.hexdigest() 98 99 100def get_entity(text): 101 return "".join(XML_ESCAPE_TABLE.get(c, c) for c in text) 102 103 104def generate_txt_notice_files(file_hash, input_dir, output_filename, 105 notice_title): 106 with open(output_filename, "w") as output_file: 107 write_file(output_file, notice_title) 108 for value in file_hash: 109 write_file(output_file, '=' * 60) 110 write_file(output_file, "Notices for file(s):") 111 for filename in value: 112 write_file( 113 output_file, '/{}'.format( 114 re.sub('.txt.*', '', 115 os.path.relpath(filename, input_dir)))) 116 write_file(output_file, '-' * 60) 117 write_file(output_file, "Notices for software(s):") 118 software_list = [] 119 for filename in value: 120 json_filename = '{}.json'.format(filename) 121 contents = read_json_file(json_filename) 122 if contents is not None and contents not in software_list: 123 software_list.append(contents) 124 for contens_value in software_list: 125 notice_source_path = contens_value[0].get('Path').strip() 126 software_name = contens_value[0].get('Software').strip() 127 128 write_file(output_file, "Software: {}".format(software_name)) 129 write_file(output_file, "Path: {}".format(notice_source_path)) 130 write_file(output_file, '-' * 60) 131 with open(value[0], errors='ignore') as temp_file_hd: 132 write_file(output_file, temp_file_hd.read()) 133 134 135def generate_xml_notice_files(files_with_same_hash, input_dir, 136 output_filename): 137 id_table = {} 138 for file_key in files_with_same_hash.keys(): 139 for filename in files_with_same_hash[file_key]: 140 id_table[filename] = file_key 141 142 with open(output_filename, "w") as output_file: 143 write_file(output_file, '<?xml version="1.0" encoding="utf-8"?>') 144 write_file(output_file, "<licenses>") 145 146 # Flatten the lists into a single filename list 147 sorted_filenames = sorted(id_table.keys()) 148 149 # write out a table of contents 150 for filename in sorted_filenames: 151 stripped_filename = re.sub('.txt.*', '', 152 os.path.relpath(filename, input_dir)) 153 write_file( 154 output_file, '<file-name contentId="%s">%s</file-name>' % 155 (id_table.get(filename), stripped_filename)) 156 157 write_file(output_file, '') 158 write_file(output_file, '') 159 160 processed_file_keys = [] 161 # write the notice file lists 162 for filename in sorted_filenames: 163 file_key = id_table.get(filename) 164 if file_key in processed_file_keys: 165 continue 166 processed_file_keys.append(file_key) 167 168 with open(filename, errors='ignore') as temp_file_hd: 169 write_file( 170 output_file, 171 '<file-content contentId="{}"><![CDATA[{}]]></file-content>' 172 .format(file_key, get_entity(temp_file_hd.read()))) 173 write_file(output_file, '') 174 175 # write the file complete node. 176 write_file(output_file, "</licenses>") 177 178 179def compress_file_to_gz(src_file_name, gz_file_name): 180 with open(src_file_name, mode='rb') as src_file_fd: 181 with gzip.open(gz_file_name, mode='wb') as gz_file_fd: 182 gz_file_fd.writelines(src_file_fd) 183 184 185def handle_zipfile_notices(zip_file): 186 notice_file = '{}.txt'.format(zip_file[:-4]) 187 with build_utils.temp_dir() as tmp_dir: 188 build_utils.extract_all(zip_file, tmp_dir, no_clobber=False) 189 files = build_utils.get_all_files(tmp_dir) 190 contents = [] 191 for file in files: 192 with open(file, 'r') as fd: 193 data = fd.read() 194 if data not in contents: 195 contents.append(data) 196 with open(notice_file, 'w') as merged_notice: 197 merged_notice.write('\n\n'.join(contents)) 198 return notice_file 199 200 201def main(): 202 parser = argparse.ArgumentParser() 203 parser.add_argument('--image-name') 204 parser.add_argument('--collected-notice-zipfile', 205 action='append', 206 help='zipfile stors collected notice files') 207 parser.add_argument('--notice-root-dir', help='where notice files store') 208 parser.add_argument('--output-notice-txt', help='output notice.txt') 209 parser.add_argument('--output-notice-gz', help='output notice.txt') 210 parser.add_argument('--notice-title', help='title of notice.txt') 211 parser.add_argument('--static-library-notice-dir', 212 help='path to static library notice files') 213 parser.add_argument('--target-cpu', help='cpu arch') 214 parser.add_argument('--depfile', help='depfile') 215 parser.add_argument('--notice-module-info', 216 help='module info file for notice target') 217 parser.add_argument('--notice-install-dir', 218 help='install directories of notice file') 219 220 args = parser.parse_args() 221 222 notice_dir = args.notice_root_dir 223 depfiles = [] 224 if args.collected_notice_zipfile: 225 for zip_file in args.collected_notice_zipfile: 226 build_utils.extract_all(zip_file, notice_dir, no_clobber=False) 227 else: 228 depfiles += build_utils.get_all_files(notice_dir) 229 # Copy notice of static targets to notice_root_dir 230 copy_static_library_notices(args, depfiles) 231 232 zipfiles = glob.glob('{}/**/*.zip'.format(notice_dir), recursive=True) 233 234 txt_files = glob.glob('{}/**/*.txt'.format(notice_dir), recursive=True) 235 txt_files += glob.glob('{}/**/*.txt.?'.format(notice_dir), recursive=True) 236 237 outputs = [args.output_notice_txt, args.output_notice_gz] 238 if args.notice_module_info: 239 outputs.append(args.notice_module_info) 240 build_utils.call_and_write_depfile_if_stale( 241 lambda: do_merge_notice(args, zipfiles, txt_files), 242 args, 243 depfile_deps=depfiles, 244 input_paths=depfiles, 245 input_strings=args.notice_title + args.target_cpu, 246 output_paths=(outputs)) 247 248 249def do_merge_notice(args, zipfiles, txt_files): 250 notice_dir = args.notice_root_dir 251 notice_txt = args.output_notice_txt 252 notice_gz = args.output_notice_gz 253 notice_title = args.notice_title 254 255 if not notice_txt.endswith('.txt'): 256 raise Exception( 257 'Error: input variable output_notice_txt must ends with .txt') 258 if not notice_gz.endswith('.xml.gz'): 259 raise Exception( 260 'Error: input variable output_notice_gz must ends with .xml.gz') 261 262 notice_xml = notice_gz.replace('.gz', '') 263 264 files_with_same_hash = defaultdict(list) 265 for file in zipfiles: 266 txt_files.append(handle_zipfile_notices(file)) 267 268 for file in txt_files: 269 if os.stat(file).st_size == 0: 270 continue 271 file_hash = compute_hash(file) 272 files_with_same_hash[file_hash].append(file) 273 274 file_sets = [ 275 sorted(files_with_same_hash[hash]) 276 for hash in sorted(files_with_same_hash.keys()) 277 ] 278 279 if file_sets is not None: 280 generate_txt_notice_files(file_sets, notice_dir, notice_txt, 281 notice_title) 282 283 if files_with_same_hash is not None: 284 generate_xml_notice_files(files_with_same_hash, notice_dir, notice_xml) 285 compress_file_to_gz(notice_xml, args.output_notice_gz) 286 287 if args.notice_module_info: 288 module_install_info_list = [] 289 module_install_info = {} 290 module_install_info['type'] = 'notice' 291 module_install_info['source'] = args.output_notice_txt 292 module_install_info['install_enable'] = True 293 module_install_info['dest'] = [ 294 os.path.join(args.notice_install_dir, 295 os.path.basename(args.output_notice_txt)) 296 ] 297 module_install_info_list.append(module_install_info) 298 write_json_file(args.notice_module_info, module_install_info_list) 299 300 301if __name__ == "__main__": 302 main() 303