1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2022 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import os 17import sys 18import argparse 19import subprocess 20import ssl 21import shutil 22import importlib 23import time 24import pathlib 25from multiprocessing import cpu_count 26from concurrent.futures import ThreadPoolExecutor, as_completed 27from functools import partial 28from urllib.request import urlopen 29import urllib.error 30from scripts.util.file_utils import read_json_file 31 32def _run_cmd(cmd: str): 33 res = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, 34 stderr=subprocess.PIPE) 35 sout, serr = res.communicate() 36 return sout.rstrip().decode('utf-8'), serr, res.returncode 37 38def _check_sha256(check_url: str, local_file: str) -> bool: 39 check_sha256_cmd = 'curl -s -k ' + check_url + '.sha256' 40 local_sha256_cmd = 'sha256sum ' + local_file + "|cut -d ' ' -f1" 41 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 42 local_sha256, err, returncode = _run_cmd(local_sha256_cmd) 43 if check_sha256 != local_sha256: 44 print('remote file {}.sha256 is not found, begin check SHASUMS256.txt'.format(check_url)) 45 check_sha256 = _obtain_sha256_by_sha_sums256(check_url) 46 return check_sha256 == local_sha256 47 48def _check_sha256_by_mark(args, check_url: str, code_dir: str, unzip_dir: str, unzip_filename: str) -> bool: 49 check_sha256_cmd = 'curl -s -k ' + check_url + '.sha256' 50 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 51 mark_file_dir = os.path.join(code_dir, unzip_dir) 52 mark_file_name = check_sha256 + '.' + unzip_filename + '.mark' 53 mark_file_path = os.path.join(mark_file_dir, mark_file_name) 54 args.mark_file_path = mark_file_path 55 return os.path.exists(mark_file_path) 56 57def _obtain_sha256_by_sha_sums256(check_url: str) -> str: 58 sha_sums256 = 'SHASUMS256.txt' 59 sha_sums256_path = os.path.join(os.path.dirname(check_url), sha_sums256) 60 file_name = os.path.basename(check_url) 61 cmd = 'curl -s -k ' + sha_sums256_path 62 data_sha_sums256, err, returncode = _run_cmd(cmd) 63 check_sha256 = None 64 for line in data_sha_sums256.split('\n'): 65 if file_name in line: 66 check_sha256 = line.split(' ')[0] 67 return check_sha256 68 69def _config_parse(config: dict, tool_repo: str) -> dict: 70 parse_dict = dict() 71 parse_dict['unzip_dir'] = config.get('unzip_dir') 72 parse_dict['huaweicloud_url'] = tool_repo + config.get('file_path') 73 parse_dict['unzip_filename'] = config.get('unzip_filename') 74 md5_huaweicloud_url_cmd = 'echo ' + parse_dict.get('huaweicloud_url') + "|md5sum|cut -d ' ' -f1" 75 parse_dict['md5_huaweicloud_url'], err, returncode = _run_cmd(md5_huaweicloud_url_cmd) 76 parse_dict['bin_file'] = os.path.basename(parse_dict.get('huaweicloud_url')) 77 return parse_dict 78 79def _uncompress(args, src_file: str, code_dir: str, unzip_dir: str, unzip_filename: str, mark_file_path: str): 80 dest_dir = os.path.join(code_dir, unzip_dir) 81 if src_file[-3:] == 'zip': 82 cmd = 'unzip -o {} -d {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 83 elif src_file[-6:] == 'tar.gz': 84 cmd = 'tar -xvzf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 85 else: 86 cmd = 'tar -xvf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 87 _run_cmd(cmd) 88 89 90def _copy_url(args, task_id: int, url: str, local_file: str, code_dir: str, unzip_dir: str, 91 unzip_filename: str, mark_file_path: str, progress): 92 retry_times = 0 93 max_retry_times = 3 94 while retry_times < max_retry_times: 95 # download files 96 download_buffer_size = 32768 97 progress.console.log('Requesting {}'.format(url)) 98 try: 99 response = urlopen(url) 100 except urllib.error.HTTPError as e: 101 progress.console.log("Failed to open {}, HTTPError: {}".format(url, e.code), style='red') 102 progress.update(task_id, total=int(response.info()["Content-length"])) 103 with open(local_file, "wb") as dest_file: 104 progress.start_task(task_id) 105 for data in iter(partial(response.read, download_buffer_size), b""): 106 dest_file.write(data) 107 progress.update(task_id, advance=len(data)) 108 progress.console.log("Downloaded {}".format(local_file)) 109 110 if os.path.exists(local_file): 111 if _check_sha256(url, local_file): 112 # decompressing files 113 progress.console.log("Decompressing {}".format(local_file)) 114 _uncompress(args, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path) 115 progress.console.log("Decompressed {}".format(local_file)) 116 break 117 else: 118 os.remove(local_file) 119 retry_times += 1 120 if retry_times == max_retry_times: 121 print('{}, download failed with three times retry, please check network status. Prebuilts download exit.'.format(local_file)) 122 # todo, merge with copy_url_disable_rich 123 sys.exit(1) 124 125 126def _copy_url_disable_rich(args, url: str, local_file: str, code_dir: str, unzip_dir: str, 127 unzip_filename: str, mark_file_path: str): 128 # download files 129 download_buffer_size = 32768 130 print('Requesting {}, please wait'.format(url)) 131 try: 132 response = urlopen(url) 133 except urllib.error.HTTPError as e: 134 print("Failed to open {}, HTTPError: {}".format(url, e.code)) 135 with open(local_file, "wb") as dest_file: 136 for data in iter(partial(response.read, download_buffer_size), b""): 137 dest_file.write(data) 138 print("Downloaded {}".format(local_file)) 139 140 # decompressing files 141 print("Decompressing {}, please wait".format(local_file)) 142 _uncompress(args, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path) 143 print("Decompressed {}".format(local_file)) 144 145def _is_system_component() -> bool: 146 root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 147 if pathlib.Path(root_dir + "/interface/sdk-js").exists() or pathlib.Path(root_dir + "/foundation/arkui").exists() or pathlib.Path(root_dir + "/arkcompiler").exists(): 148 return True 149 else: 150 return False 151 152def _hwcloud_download(args, config: dict, bin_dir: str, code_dir: str): 153 try: 154 cnt = cpu_count() 155 except: 156 cnt = 1 157 with ThreadPoolExecutor(max_workers=cnt) as pool: 158 tasks = dict() 159 for config_info in config: 160 parse_dict = _config_parse(config_info, args.tool_repo) 161 unzip_dir = parse_dict.get('unzip_dir') 162 huaweicloud_url = parse_dict.get('huaweicloud_url') 163 unzip_filename = parse_dict.get('unzip_filename') 164 md5_huaweicloud_url = parse_dict.get('md5_huaweicloud_url') 165 bin_file = parse_dict.get('bin_file') 166 abs_unzip_dir = os.path.join(code_dir, unzip_dir) 167 if not os.path.exists(abs_unzip_dir): 168 os.makedirs(abs_unzip_dir, exist_ok=True) 169 if _check_sha256_by_mark(args, huaweicloud_url, code_dir, unzip_dir, unzip_filename): 170 if not args.disable_rich: 171 args.progress.console.log('{}, Sha256 markword check OK.'.format(huaweicloud_url), style='green') 172 else: 173 print('{}, Sha256 markword check OK.'.format(huaweicloud_url)) 174 else: 175 _run_cmd(('rm -rf {}/{}/*.{}.mark').format(code_dir, unzip_dir, unzip_filename)) 176 _run_cmd(('rm -rf {}/{}/{}').format(code_dir, unzip_dir, unzip_filename)) 177 local_file = os.path.join(bin_dir, '{}.{}'.format(md5_huaweicloud_url, bin_file)) 178 if os.path.exists(local_file): 179 if _check_sha256(huaweicloud_url, local_file): 180 if not args.disable_rich: 181 args.progress.console.log('{}, Sha256 check download OK.'.format(local_file), style='green') 182 else: 183 print('{}, Sha256 check download OK. Start decompression, please wait'.format(local_file)) 184 task = pool.submit(_uncompress, args, local_file, code_dir, 185 unzip_dir, unzip_filename, args.mark_file_path) 186 tasks[task] = os.path.basename(huaweicloud_url) 187 continue 188 else: 189 os.remove(local_file) 190 filename = huaweicloud_url.split("/")[-1] 191 if not args.disable_rich: 192 task_id = args.progress.add_task("download", filename=filename, start=False) 193 task = pool.submit(_copy_url, args, task_id, huaweicloud_url, local_file, code_dir, 194 unzip_dir, unzip_filename, args.mark_file_path, args.progress) 195 tasks[task] = os.path.basename(huaweicloud_url) 196 else: 197 task = pool.submit(_copy_url_disable_rich, args, huaweicloud_url, local_file, code_dir, 198 unzip_dir, unzip_filename, args.mark_file_path) 199 200 for task in as_completed(tasks): 201 if not args.disable_rich: 202 args.progress.console.log('{}, download and decompress completed'.format(tasks.get(task)), 203 style='green') 204 else: 205 print('{}, download and decompress completed'.format(tasks.get(task))) 206 207def _npm_install(args): 208 node_path = 'prebuilts/build-tools/common/nodejs/current/bin' 209 os.environ['PATH'] = '{}/{}:{}'.format(args.code_dir, node_path, os.environ.get('PATH')) 210 npm = os.path.join(args.code_dir, node_path, 'npm') 211 if args.skip_ssl: 212 skip_ssl_cmd = '{} config set strict-ssl false;'.format(npm) 213 _run_cmd(skip_ssl_cmd) 214 npm_clean_cmd = '{} cache clean -f'.format(npm) 215 npm_package_lock_cmd = '{} config set package-lock true'.format(npm) 216 _run_cmd(npm_clean_cmd) 217 _run_cmd(npm_package_lock_cmd) 218 print('start npm install, please wait.') 219 for install_info in args.npm_install_config: 220 full_code_path = os.path.join(args.code_dir, install_info) 221 basename = os.path.basename(full_code_path) 222 npm_cache_dir = os.path.join('~/.npm/_cacache', basename) 223 if os.path.exists(full_code_path): 224 cmd = [npm, 'install', '--registry', args.npm_registry, '--cache', npm_cache_dir] 225 if args.unsafe_perm: 226 cmd.append('--unsafe-perm') 227 proc = subprocess.Popen(cmd, cwd=full_code_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 228 # wait proc Popen with 0.1 second 229 time.sleep(0.1) 230 out, err = proc.communicate() 231 if proc.returncode: 232 return False, err.decode() 233 else: 234 raise Exception("{} not exist, it shouldn't happen, pls check...".format(full_code_path)) 235 return True, None 236 237 238def _node_modules_copy(config: dict, code_dir: str, enable_symlink: bool): 239 for config_info in config: 240 src_dir = os.path.join(code_dir, config_info.get('src')) 241 dest_dir = os.path.join(code_dir, config_info.get('dest')) 242 use_symlink = config_info.get('use_symlink') 243 if os.path.exists(os.path.dirname(dest_dir)): 244 shutil.rmtree(os.path.dirname(dest_dir)) 245 if use_symlink == 'True' and enable_symlink == True: 246 os.makedirs(os.path.dirname(dest_dir), exist_ok=True) 247 os.symlink(src_dir, dest_dir) 248 else: 249 shutil.copytree(src_dir, dest_dir, symlinks=True) 250 251def _file_handle(config: dict, code_dir: str, host_platform: str): 252 for config_info in config: 253 src_dir = code_dir + config_info.get('src') 254 dest_dir = code_dir + config_info.get('dest') 255 tmp_dir = config_info.get('tmp') 256 symlink_src = config_info.get('symlink_src') 257 symlink_dest = config_info.get('symlink_dest') 258 rename = config_info.get('rename') 259 if os.path.exists(src_dir): 260 if tmp_dir: 261 tmp_dir = code_dir + tmp_dir 262 shutil.move(src_dir, tmp_dir) 263 cmd = 'mv {}/*.mark {}'.format(dest_dir, tmp_dir) 264 _run_cmd(cmd) 265 if os.path.exists(dest_dir): 266 shutil.rmtree(dest_dir) 267 shutil.move(tmp_dir, dest_dir) 268 elif rename: 269 if os.path.exists(dest_dir) and dest_dir != src_dir: 270 shutil.rmtree(dest_dir) 271 shutil.move(src_dir, dest_dir) 272 if symlink_src and symlink_dest: 273 if os.path.exists(dest_dir + symlink_dest): 274 os.remove(dest_dir + symlink_dest) 275 if host_platform == 'darwin' and os.path.basename(dest_dir) == "nodejs": 276 symlink_src = symlink_src.replace('linux', 'darwin') 277 os.symlink(os.path.basename(symlink_src), dest_dir + symlink_dest) 278 else: 279 _run_cmd('chmod 755 {} -R'.format(dest_dir)) 280 281def _import_rich_module(): 282 module = importlib.import_module('rich.progress') 283 progress = module.Progress( 284 module.TextColumn("[bold blue]{task.fields[filename]}", justify="right"), 285 module.BarColumn(bar_width=None), 286 "[progress.percentage]{task.percentage:>3.1f}%", 287 "•", 288 module.DownloadColumn(), 289 "•", 290 module.TransferSpeedColumn(), 291 "•", 292 module.TimeRemainingColumn(), 293 ) 294 return progress 295 296 297def _install(config: dict, code_dir: str): 298 for config_info in config: 299 install_dir = '{}/{}'.format(code_dir, config_info.get('install_dir')) 300 script = config_info.get('script') 301 cmd = '{}/{}'.format(install_dir, script) 302 args = config_info.get('args') 303 for arg in args: 304 for key in arg.keys(): 305 cmd = '{} --{}={}'.format(cmd, key, arg[key]) 306 dest_dir = '{}/{}'.format(code_dir, config_info.get('destdir')) 307 cmd = '{} --destdir={}'.format(cmd, dest_dir) 308 _run_cmd(cmd) 309 310def main(): 311 parser = argparse.ArgumentParser() 312 parser.add_argument('--skip-ssl', action='store_true', help='skip ssl authentication') 313 parser.add_argument('--unsafe-perm', action='store_true', help='add "--unsafe-perm" for npm install') 314 parser.add_argument('--disable-rich', action='store_true', help='disable the rich module') 315 parser.add_argument('--enable-symlink', action='store_true', help='enable symlink while copying node_modules') 316 parser.add_argument('--build-arkuix', action='store_true', help='build ArkUI-X SDK') 317 parser.add_argument('--tool-repo', default='https://repo.huaweicloud.com', help='prebuilt file download source') 318 parser.add_argument('--npm-registry', default='https://repo.huaweicloud.com/repository/npm/', 319 help='npm download source') 320 parser.add_argument('--host-cpu', help='host cpu', required=True) 321 parser.add_argument('--host-platform', help='host platform', required=True) 322 parser.add_argument('--config-file', help='prebuilts download config file') 323 args = parser.parse_args() 324 args.code_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 325 if args.skip_ssl: 326 ssl._create_default_https_context = ssl._create_unverified_context 327 328 host_platform = args.host_platform 329 host_cpu = args.host_cpu 330 tool_repo = args.tool_repo 331 if args.build_arkuix: 332 config_file = os.path.join(args.code_dir, 'build_plugins/prebuilts_download_config.json') 333 elif args.config_file: 334 config_file = args.config_file 335 else: 336 config_file = os.path.join(args.code_dir, 'build/prebuilts_download_config.json') 337 config_info = read_json_file(config_file) 338 if _is_system_component(): 339 args.npm_install_config = config_info.get('npm_install_path') 340 node_modules_copy_config = config_info.get('node_modules_copy') 341 else: 342 args.npm_install_config = [] 343 node_modules_copy_config = [] 344 file_handle_config = config_info.get('file_handle_config') 345 346 args.bin_dir = os.path.join(args.code_dir, config_info.get('prebuilts_download_dir')) 347 if not os.path.exists(args.bin_dir): 348 os.makedirs(args.bin_dir, exist_ok=True) 349 copy_config = config_info.get(host_platform).get(host_cpu).get('copy_config') 350 node_config = config_info.get(host_platform).get('node_config') 351 copy_config.extend(node_config) 352 install_config = config_info.get(host_platform).get(host_cpu).get('install') 353 if host_platform == 'linux': 354 linux_copy_config = config_info.get(host_platform).get(host_cpu).get('linux_copy_config') 355 copy_config.extend(linux_copy_config) 356 elif host_platform == 'darwin': 357 darwin_copy_config = config_info.get(host_platform).get(host_cpu).get('darwin_copy_config') 358 copy_config.extend(darwin_copy_config) 359 if args.disable_rich: 360 _hwcloud_download(args, copy_config, args.bin_dir, args.code_dir) 361 else: 362 args.progress = _import_rich_module() 363 with args.progress: 364 _hwcloud_download(args, copy_config, args.bin_dir, args.code_dir) 365 366 print('download finished') 367 _file_handle(file_handle_config, args.code_dir, args.host_platform) 368 retry_times = 0 369 max_retry_times = 2 370 while retry_times <= max_retry_times: 371 print('npm install try times:', retry_times + 1) 372 result, error = _npm_install(args) 373 if result: 374 break 375 elif retry_times == max_retry_times: 376 for error_info in error.split('\n'): 377 if error_info.endswith('debug.log'): 378 log_path = error_info.split()[-1] 379 cmd = ['cat', log_path] 380 subprocess.Popen(cmd) 381 raise Exception("npm install error with three times, prebuilts download exit") 382 retry_times += 1 383 _node_modules_copy(node_modules_copy_config, args.code_dir, args.enable_symlink) 384 if install_config: 385 _install(install_config, args.code_dir) 386 387 # delete uninstalled tools 388 uninstalled_tools = config_info.get('uninstalled_tools') 389 for tool_path in uninstalled_tools: 390 subprocess.run(['rm', '-rf', tool_path]) 391 392if __name__ == '__main__': 393 sys.exit(main()) 394