1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3# Copyright (c) 2021 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15""" 16 17Usage: gen_notice_file --output-image-name system \ 18 --notice-file-root xx/NOTICE_FILE \ 19 --notice-file-install-path xx/system \ 20 --output-title notice_title_string 21 22Generate the project notice files, including both text and xml files. 23 24""" 25from collections import defaultdict 26import argparse 27import hashlib 28import os 29import os.path 30import sys 31import gzip 32import shutil 33import glob 34import re 35 36sys.path.append( 37 os.path.dirname(os.path.dirname(os.path.dirname( 38 os.path.abspath(__file__))))) 39from scripts.util import build_utils # noqa: E402 40from scripts.util.file_utils import write_json_file # noqa: E402 41 42XML_ESCAPE_TABLE = { 43 "&": "&", 44 '"': """, 45 "'": "'", 46 ">": ">", 47 "<": "<", 48} 49 50 51def copy_static_library_notices(options, depfiles): 52 valid_notices = [] 53 basenames = [] 54 for file in build_utils.get_all_files(options.static_library_notice_dir): 55 if os.stat(file).st_size == 0: 56 continue 57 if not file.endswith('.a.txt'): 58 continue 59 notice_file_name = os.path.basename(file) 60 if file not in basenames: 61 basenames.append(notice_file_name) 62 valid_notices.append(file) 63 depfiles.append(file) 64 65 for file in valid_notices: 66 if options.image_name == "system": 67 if options.target_cpu == "arm64" or options.target_cpu == "x64": 68 install_dir = "system/lib64" 69 elif options.target_cpu == "arm": 70 install_dir = "system/lib" 71 else: 72 continue 73 elif options.image_name == "sdk": 74 install_dir = "toolchains/lib" 75 elif options.image_name == "ndk": 76 install_dir = "sysroot/usr/lib" 77 else: 78 continue 79 dest = os.path.join(options.notice_root_dir, install_dir, 80 os.path.basename(file)) 81 os.makedirs(os.path.dirname(dest), exist_ok=True) 82 shutil.copyfile(file, dest) 83 84 85def write_file(file, string): 86 print(string, file=file) 87 88 89def compute_hash(file): 90 sha256 = hashlib.sha256() 91 with open(file, 'rb') as file_fd: 92 for line in file_fd: 93 sha256.update(line) 94 return sha256.hexdigest() 95 96 97def get_entity(text): 98 return "".join(XML_ESCAPE_TABLE.get(c, c) for c in text) 99 100 101def generate_txt_notice_files(file_hash, input_dir, output_filename, 102 notice_title): 103 with open(output_filename, "w") as output_file: 104 write_file(output_file, notice_title) 105 for value in file_hash: 106 write_file(output_file, '=' * 60) 107 write_file(output_file, "Notices for file(s):") 108 for filename in value: 109 write_file( 110 output_file, '/{}'.format( 111 re.sub('.txt.*', '', 112 os.path.relpath(filename, input_dir)))) 113 write_file(output_file, '-' * 60) 114 with open(value[0], errors='ignore') as temp_file_hd: 115 write_file(output_file, temp_file_hd.read()) 116 117 118def generate_xml_notice_files(files_with_same_hash, input_dir, 119 output_filename): 120 id_table = {} 121 for file_key in files_with_same_hash.keys(): 122 for filename in files_with_same_hash[file_key]: 123 id_table[filename] = file_key 124 125 with open(output_filename, "w") as output_file: 126 write_file(output_file, '<?xml version="1.0" encoding="utf-8"?>') 127 write_file(output_file, "<licenses>") 128 129 # Flatten the lists into a single filename list 130 sorted_filenames = sorted(id_table.keys()) 131 132 # write out a table of contents 133 for filename in sorted_filenames: 134 stripped_filename = re.sub('.txt.*', '', 135 os.path.relpath(filename, input_dir)) 136 write_file( 137 output_file, '<file-name contentId="%s">%s</file-name>' % 138 (id_table.get(filename), stripped_filename)) 139 140 write_file(output_file, '') 141 write_file(output_file, '') 142 143 processed_file_keys = [] 144 # write the notice file lists 145 for filename in sorted_filenames: 146 file_key = id_table.get(filename) 147 if file_key in processed_file_keys: 148 continue 149 processed_file_keys.append(file_key) 150 151 with open(filename, errors='ignore') as temp_file_hd: 152 write_file( 153 output_file, 154 '<file-content contentId="{}"><![CDATA[{}]]></file-content>' 155 .format(file_key, get_entity(temp_file_hd.read()))) 156 write_file(output_file, '') 157 158 # write the file complete node. 159 write_file(output_file, "</licenses>") 160 161 162def compress_file_to_gz(src_file_name, gz_file_name): 163 with open(src_file_name, mode='rb') as src_file_fd: 164 with gzip.open(gz_file_name, mode='wb') as gz_file_fd: 165 gz_file_fd.writelines(src_file_fd) 166 167 168def handle_zipfile_notices(zip_file): 169 notice_file = '{}.txt'.format(zip_file[:-4]) 170 with build_utils.temp_dir() as tmp_dir: 171 build_utils.extract_all(zip_file, tmp_dir, no_clobber=False) 172 files = build_utils.get_all_files(tmp_dir) 173 contents = [] 174 for file in files: 175 with open(file, 'r') as fd: 176 data = fd.read() 177 if data not in contents: 178 contents.append(data) 179 with open(notice_file, 'w') as merged_notice: 180 merged_notice.write('\n\n'.join(contents)) 181 return notice_file 182 183 184def main(): 185 parser = argparse.ArgumentParser() 186 parser.add_argument('--image-name') 187 parser.add_argument('--collected-notice-zipfile', 188 action='append', 189 help='zipfile stors collected notice files') 190 parser.add_argument('--notice-root-dir', help='where notice files store') 191 parser.add_argument('--output-notice-txt', help='output notice.txt') 192 parser.add_argument('--output-notice-gz', help='output notice.txt') 193 parser.add_argument('--notice-title', help='title of notice.txt') 194 parser.add_argument('--static-library-notice-dir', 195 help='path to static library notice files') 196 parser.add_argument('--target-cpu', help='cpu arch') 197 parser.add_argument('--depfile', help='depfile') 198 parser.add_argument('--notice-module-info', 199 help='module info file for notice target') 200 parser.add_argument('--notice-install-dir', 201 help='install directories of notice file') 202 203 args = parser.parse_args() 204 205 notice_dir = args.notice_root_dir 206 depfiles = [] 207 if args.collected_notice_zipfile: 208 for zip_file in args.collected_notice_zipfile: 209 build_utils.extract_all(zip_file, notice_dir, no_clobber=False) 210 else: 211 depfiles += build_utils.get_all_files(notice_dir) 212 # Copy notice of static targets to notice_root_dir 213 copy_static_library_notices(args, depfiles) 214 215 zipfiles = glob.glob('{}/**/*.zip'.format(notice_dir), recursive=True) 216 217 txt_files = glob.glob('{}/**/*.txt'.format(notice_dir), recursive=True) 218 txt_files += glob.glob('{}/**/*.txt.?'.format(notice_dir), recursive=True) 219 220 outputs = [args.output_notice_txt, args.output_notice_gz] 221 if args.notice_module_info: 222 outputs.append(args.notice_module_info) 223 build_utils.call_and_write_depfile_if_stale( 224 lambda: do_merge_notice(args, zipfiles, txt_files), 225 args, 226 depfile_deps=depfiles, 227 input_paths=depfiles, 228 input_strings=args.notice_title + args.target_cpu, 229 output_paths=(outputs)) 230 231 232def do_merge_notice(args, zipfiles, txt_files): 233 notice_dir = args.notice_root_dir 234 notice_txt = args.output_notice_txt 235 notice_gz = args.output_notice_gz 236 notice_title = args.notice_title 237 238 if not notice_txt.endswith('.txt'): 239 raise Exception( 240 'Error: input variable output_notice_txt must ends with .txt') 241 if not notice_gz.endswith('.xml.gz'): 242 raise Exception( 243 'Error: input variable output_notice_gz must ends with .xml.gz') 244 245 notice_xml = notice_gz.replace('.gz', '') 246 247 files_with_same_hash = defaultdict(list) 248 for file in zipfiles: 249 txt_files.append(handle_zipfile_notices(file)) 250 251 for file in txt_files: 252 if os.stat(file).st_size == 0: 253 continue 254 file_hash = compute_hash(file) 255 files_with_same_hash[file_hash].append(file) 256 257 file_sets = [ 258 sorted(files_with_same_hash[hash]) 259 for hash in sorted(files_with_same_hash.keys()) 260 ] 261 262 if file_sets is not None: 263 generate_txt_notice_files(file_sets, notice_dir, notice_txt, 264 notice_title) 265 266 if files_with_same_hash is not None: 267 generate_xml_notice_files(files_with_same_hash, notice_dir, notice_xml) 268 compress_file_to_gz(notice_xml, args.output_notice_gz) 269 270 if args.notice_module_info: 271 module_install_info_list = [] 272 module_install_info = {} 273 module_install_info['type'] = 'notice' 274 module_install_info['source'] = args.output_notice_txt 275 module_install_info['install_enable'] = True 276 module_install_info['dest'] = [ 277 os.path.join(args.notice_install_dir, 278 os.path.basename(args.output_notice_txt)) 279 ] 280 module_install_info_list.append(module_install_info) 281 write_json_file(args.notice_module_info, module_install_info_list) 282 283 284if __name__ == "__main__": 285 main() 286