1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2022 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import os 17import sys 18import argparse 19import subprocess 20import ssl 21import shutil 22import importlib 23import time 24import pathlib 25import re 26from multiprocessing import cpu_count 27from concurrent.futures import ThreadPoolExecutor, as_completed 28from functools import partial 29from urllib.request import urlopen 30import urllib.error 31from scripts.util.file_utils import read_json_file 32 33 34def _run_cmd(cmd: str): 35 res = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, 36 stderr=subprocess.PIPE) 37 sout, serr = res.communicate() 38 return sout.rstrip().decode('utf-8'), serr, res.returncode 39 40 41def _check_sha256(check_url: str, local_file: str) -> bool: 42 check_sha256_cmd = 'curl -s -k ' + check_url + '.sha256' 43 local_sha256_cmd = 'sha256sum ' + local_file + "|cut -d ' ' -f1" 44 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 45 local_sha256, err, returncode = _run_cmd(local_sha256_cmd) 46 if check_sha256 != local_sha256: 47 print('remote file {}.sha256 is not found, begin check SHASUMS256.txt'.format(check_url)) 48 check_sha256 = _obtain_sha256_by_sha_sums256(check_url) 49 return check_sha256 == local_sha256 50 51 52def _check_sha256_by_mark(args, check_url: str, code_dir: str, unzip_dir: str, unzip_filename: str) -> bool: 53 check_sha256_cmd = 'curl -s -k ' + check_url + '.sha256' 54 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 55 mark_file_dir = os.path.join(code_dir, unzip_dir) 56 mark_file_name = check_sha256 + '.' + unzip_filename + '.mark' 57 mark_file_path = os.path.join(mark_file_dir, mark_file_name) 58 args.mark_file_path = mark_file_path 59 return os.path.exists(mark_file_path) 60 61 62def _obtain_sha256_by_sha_sums256(check_url: str) -> str: 63 sha_sums256 = 'SHASUMS256.txt' 64 sha_sums256_path = os.path.join(os.path.dirname(check_url), sha_sums256) 65 file_name = os.path.basename(check_url) 66 cmd = 'curl -s -k ' + sha_sums256_path 67 data_sha_sums256, err, returncode = _run_cmd(cmd) 68 check_sha256 = None 69 for line in data_sha_sums256.split('\n'): 70 if file_name in line: 71 check_sha256 = line.split(' ')[0] 72 return check_sha256 73 74 75def _config_parse(config: dict, tool_repo: str, glibc_version: str) -> dict: 76 parse_dict = dict() 77 parse_dict['unzip_dir'] = config.get('unzip_dir') 78 file_path = config.get('file_path') 79 if 'python' in file_path and glibc_version is not None: 80 file_path = re.sub(r'GLIBC[0-9]\.[0-9]{2}', glibc_version, file_path) 81 parse_dict['huaweicloud_url'] = tool_repo + file_path 82 parse_dict['unzip_filename'] = config.get('unzip_filename') 83 md5_huaweicloud_url_cmd = 'echo ' + parse_dict.get('huaweicloud_url') + "|md5sum|cut -d ' ' -f1" 84 parse_dict['md5_huaweicloud_url'], err, returncode = _run_cmd(md5_huaweicloud_url_cmd) 85 parse_dict['bin_file'] = os.path.basename(parse_dict.get('huaweicloud_url')) 86 return parse_dict 87 88 89def _uncompress(args, src_file: str, code_dir: str, unzip_dir: str, unzip_filename: str, mark_file_path: str): 90 dest_dir = os.path.join(code_dir, unzip_dir) 91 if src_file[-3:] == 'zip': 92 cmd = 'unzip -o {} -d {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 93 elif src_file[-6:] == 'tar.gz': 94 cmd = 'tar -xvzf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 95 else: 96 cmd = 'tar -xvf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 97 _run_cmd(cmd) 98 99 100def _copy_url(args, task_id: int, url: str, local_file: str, code_dir: str, unzip_dir: str, 101 unzip_filename: str, mark_file_path: str, progress): 102 retry_times = 0 103 max_retry_times = 3 104 while retry_times < max_retry_times: 105 # download files 106 download_buffer_size = 32768 107 progress.console.log('Requesting {}'.format(url)) 108 try: 109 response = urlopen(url) 110 except urllib.error.HTTPError as e: 111 progress.console.log("Failed to open {}, HTTPError: {}".format(url, e.code), style='red') 112 progress.update(task_id, total=int(response.info()["Content-length"])) 113 with open(local_file, "wb") as dest_file: 114 progress.start_task(task_id) 115 for data in iter(partial(response.read, download_buffer_size), b""): 116 dest_file.write(data) 117 progress.update(task_id, advance=len(data)) 118 progress.console.log("Downloaded {}".format(local_file)) 119 120 if os.path.exists(local_file): 121 if _check_sha256(url, local_file): 122 # decompressing files 123 progress.console.log("Decompressing {}".format(local_file)) 124 _uncompress(args, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path) 125 progress.console.log("Decompressed {}".format(local_file)) 126 break 127 else: 128 os.remove(local_file) 129 retry_times += 1 130 if retry_times == max_retry_times: 131 print('{}, download failed with three times retry, please check network status. Prebuilts download exit.'.format(local_file)) 132 # todo, merge with copy_url_disable_rich 133 sys.exit(1) 134 135 136def _copy_url_disable_rich(args, url: str, local_file: str, code_dir: str, unzip_dir: str, 137 unzip_filename: str, mark_file_path: str): 138 # download files 139 download_buffer_size = 32768 140 print('Requesting {}, please wait'.format(url)) 141 try: 142 response = urlopen(url) 143 except urllib.error.HTTPError as e: 144 print("Failed to open {}, HTTPError: {}".format(url, e.code)) 145 with open(local_file, "wb") as dest_file: 146 for data in iter(partial(response.read, download_buffer_size), b""): 147 dest_file.write(data) 148 print("Downloaded {}".format(local_file)) 149 150 # decompressing files 151 print("Decompressing {}, please wait".format(local_file)) 152 _uncompress(args, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path) 153 print("Decompressed {}".format(local_file)) 154 155 156def _is_system_component() -> bool: 157 root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 158 if pathlib.Path(os.path.join(root_dir, 'interface', 'sdk-js')).exists() or pathlib.Path( 159 os.path.join(root_dir, 'foundation', 'arkui')).exists() or pathlib.Path( 160 os.path.join(root_dir, 'arkcompiler')).exists(): 161 return True 162 else: 163 return False 164 165 166def _hwcloud_download(args, config: dict, bin_dir: str, code_dir: str, glibc_version: str): 167 try: 168 cnt = cpu_count() 169 except Exception as e: 170 cnt = 1 171 with ThreadPoolExecutor(max_workers=cnt) as pool: 172 tasks = dict() 173 for config_info in config: 174 parse_dict = _config_parse(config_info, args.tool_repo, glibc_version) 175 unzip_dir = parse_dict.get('unzip_dir') 176 huaweicloud_url = parse_dict.get('huaweicloud_url') 177 unzip_filename = parse_dict.get('unzip_filename') 178 md5_huaweicloud_url = parse_dict.get('md5_huaweicloud_url') 179 bin_file = parse_dict.get('bin_file') 180 abs_unzip_dir = os.path.join(code_dir, unzip_dir) 181 if not os.path.exists(abs_unzip_dir): 182 os.makedirs(abs_unzip_dir, exist_ok=True) 183 if _check_sha256_by_mark(args, huaweicloud_url, code_dir, unzip_dir, unzip_filename): 184 if not args.disable_rich: 185 args.progress.console.log('{}, Sha256 markword check OK.'.format(huaweicloud_url), style='green') 186 else: 187 print('{}, Sha256 markword check OK.'.format(huaweicloud_url)) 188 else: 189 _run_cmd(('rm -rf {}/{}/*.{}.mark').format(code_dir, unzip_dir, unzip_filename)) 190 _run_cmd(('rm -rf {}/{}/{}').format(code_dir, unzip_dir, unzip_filename)) 191 local_file = os.path.join(bin_dir, '{}.{}'.format(md5_huaweicloud_url, bin_file)) 192 if os.path.exists(local_file): 193 if _check_sha256(huaweicloud_url, local_file): 194 if not args.disable_rich: 195 args.progress.console.log('{}, Sha256 check download OK.'.format(local_file), style='green') 196 else: 197 print('{}, Sha256 check download OK. Start decompression, please wait'.format(local_file)) 198 task = pool.submit(_uncompress, args, local_file, code_dir, 199 unzip_dir, unzip_filename, args.mark_file_path) 200 tasks[task] = os.path.basename(huaweicloud_url) 201 continue 202 else: 203 os.remove(local_file) 204 filename = huaweicloud_url.split("/")[-1] 205 if not args.disable_rich: 206 task_id = args.progress.add_task("download", filename=filename, start=False) 207 task = pool.submit(_copy_url, args, task_id, huaweicloud_url, local_file, code_dir, 208 unzip_dir, unzip_filename, args.mark_file_path, args.progress) 209 tasks[task] = os.path.basename(huaweicloud_url) 210 else: 211 task = pool.submit(_copy_url_disable_rich, args, huaweicloud_url, local_file, code_dir, 212 unzip_dir, unzip_filename, args.mark_file_path) 213 214 for task in as_completed(tasks): 215 if not args.disable_rich: 216 args.progress.console.log('{}, download and decompress completed'.format(tasks.get(task)), 217 style='green') 218 else: 219 print('{}, download and decompress completed'.format(tasks.get(task))) 220 221 222def _npm_install(args): 223 node_path = 'prebuilts/build-tools/common/nodejs/current/bin' 224 os.environ['PATH'] = '{}/{}:{}'.format(args.code_dir, node_path, os.environ.get('PATH')) 225 npm = os.path.join(args.code_dir, node_path, 'npm') 226 if args.skip_ssl: 227 skip_ssl_cmd = '{} config set strict-ssl false;'.format(npm) 228 out, err, retcode = _run_cmd(skip_ssl_cmd) 229 if retcode != 0: 230 return False, err.decode() 231 npm_clean_cmd = '{} cache clean -f'.format(npm) 232 npm_package_lock_cmd = '{} config set package-lock true'.format(npm) 233 out, err, retcode = _run_cmd(npm_clean_cmd) 234 if retcode != 0: 235 return False, err.decode() 236 out, err, retcode = _run_cmd(npm_package_lock_cmd) 237 if retcode != 0: 238 return False, err.decode() 239 install_cmds = [] 240 full_code_paths = [] 241 242 print('start npm install, please wait.') 243 for install_info in args.npm_install_config: 244 full_code_path = os.path.join(args.code_dir, install_info) 245 if full_code_path in args.success_installed: 246 print('{} has been installed, skip'.format(full_code_path)) 247 continue 248 basename = os.path.basename(full_code_path) 249 node_modules_path = os.path.join(full_code_path, "node_modules") 250 npm_cache_dir = os.path.join('~/.npm/_cacache', basename) 251 if os.path.exists(node_modules_path): 252 print('remove node_modules %s' % node_modules_path) 253 _run_cmd(('rm -rf {}'.format(node_modules_path))) 254 if os.path.exists(full_code_path): 255 cmd = ['timeout', '-s', '9', '90s', npm, 'install', '--registry', args.npm_registry, '--cache', npm_cache_dir] 256 if args.host_platform == 'darwin': 257 cmd = [npm, 'install', '--registry', args.npm_registry, '--cache', npm_cache_dir] 258 if args.unsafe_perm: 259 cmd.append('--unsafe-perm') 260 install_cmds.append(cmd) 261 full_code_paths.append(full_code_path) 262 else: 263 raise Exception("{} not exist, it shouldn't happen, pls check...".format(full_code_path)) 264 265 if args.parallel_install: 266 print('run npm install in parallel mode, please wait.') 267 procs = [] 268 for index, install_cmd in enumerate(install_cmds): 269 proc = subprocess.Popen(install_cmd, cwd=full_code_paths[index], stdout=subprocess.PIPE, stderr=subprocess.PIPE) 270 print('run npm install in {}'.format(full_code_paths[index])) 271 time.sleep(0.1) 272 procs.append(proc) 273 for index, proc in enumerate(procs): 274 out, err = proc.communicate() 275 if proc.returncode: 276 print("in dir:{}, executing:{}".format(full_code_paths[index], ' '.join(install_cmds[index]))) 277 return False, err.decode() 278 args.success_installed.append(full_code_paths[index]) 279 280 else: 281 print('run npm install in serial mode, please wait.') 282 for index, install_cmd in enumerate(install_cmds): 283 proc = subprocess.Popen(install_cmd, cwd=full_code_paths[index], stdout=subprocess.PIPE, stderr=subprocess.PIPE) 284 print('run npm install in {}'.format(full_code_paths[index])) 285 time.sleep(0.1) 286 out, err = proc.communicate() 287 if proc.returncode: 288 print("in dir:{}, executing:{}".format(full_code_paths[index], ' '.join(install_cmds[index]))) 289 return False, err.decode() 290 args.success_installed.append(full_code_paths[index]) 291 return True, None 292 293 294def _node_modules_copy(config: dict, code_dir: str, enable_symlink: bool): 295 for config_info in config: 296 src_dir = os.path.join(code_dir, config_info.get('src')) 297 if not os.path.exists(src_dir): 298 print(f"{src_dir} not exist, skip node_modules copy.") 299 continue 300 dest_dir = os.path.join(code_dir, config_info.get('dest')) 301 use_symlink = config_info.get('use_symlink') 302 if os.path.exists(os.path.dirname(dest_dir)): 303 shutil.rmtree(os.path.dirname(dest_dir)) 304 if use_symlink == 'True' and enable_symlink == True: 305 os.makedirs(os.path.dirname(dest_dir), exist_ok=True) 306 os.symlink(src_dir, dest_dir) 307 else: 308 shutil.copytree(src_dir, dest_dir, symlinks=True) 309 310 311def _do_rename(config_info: dict, code_dir: str, host_platform: str, host_cpu: str): 312 src_dir = code_dir + config_info.get('src') 313 dest_dir = code_dir + config_info.get('dest') 314 symlink_src = config_info.get('symlink_src') 315 symlink_dest = config_info.get('symlink_dest') 316 if os.path.exists(dest_dir) and dest_dir != src_dir: 317 shutil.rmtree(dest_dir) 318 shutil.move(src_dir, dest_dir) 319 if symlink_src and symlink_dest and os.path.exists(src_dir + symlink_src): 320 if os.path.exists(dest_dir + symlink_dest) and os.path.islink(dest_dir + symlink_dest): 321 os.unlink(dest_dir + symlink_dest) 322 if os.path.exists(dest_dir + symlink_dest) and os.path.isfile(dest_dir + symlink_dest): 323 os.remove(dest_dir + symlink_dest) 324 if os.path.exists(dest_dir + symlink_dest) and os.path.isdir(dest_dir + symlink_dest): 325 shutil.rmtree(dest_dir + symlink_dest) 326 os.symlink(os.path.basename(symlink_src), dest_dir + symlink_dest) 327 328 329def _handle_tmp(tmp_dir: str, src_dir: str, dest_dir: str): 330 shutil.move(src_dir, tmp_dir) 331 cmd = 'mv {}/*.mark {}'.format(dest_dir, tmp_dir) 332 _run_cmd(cmd) 333 if os.path.exists(dest_dir): 334 shutil.rmtree(dest_dir) 335 shutil.move(tmp_dir, dest_dir) 336 337 338def _file_handle(config: dict, code_dir: str, host_platform: str, host_cpu: str): 339 for config_info in config: 340 src_dir = code_dir + config_info.get('src') 341 dest_dir = code_dir + config_info.get('dest') 342 tmp_dir = config_info.get('tmp') 343 rename = config_info.get('rename') 344 if os.path.exists(src_dir): 345 if tmp_dir: 346 tmp_dir = code_dir + tmp_dir 347 _handle_tmp(tmp_dir, src_dir, dest_dir) 348 elif rename: 349 _do_rename(config_info, code_dir, host_platform, host_cpu) 350 else: 351 _run_cmd('chmod 755 {} -R'.format(dest_dir)) 352 353 354def _import_rich_module(): 355 module = importlib.import_module('rich.progress') 356 progress = module.Progress( 357 module.TextColumn("[bold blue]{task.fields[filename]}", justify="right"), 358 module.BarColumn(bar_width=None), 359 "[progress.percentage]{task.percentage:>3.1f}%", 360 "•", 361 module.DownloadColumn(), 362 "•", 363 module.TransferSpeedColumn(), 364 "•", 365 module.TimeRemainingColumn(), 366 ) 367 return progress 368 369 370def _install(config: dict, code_dir: str): 371 for config_info in config: 372 install_dir = '{}/{}'.format(code_dir, config_info.get('install_dir')) 373 script = config_info.get('script') 374 cmd = '{}/{}'.format(install_dir, script) 375 args = config_info.get('args') 376 for arg in args: 377 for key in arg.keys(): 378 cmd = '{} --{}={}'.format(cmd, key, arg[key]) 379 dest_dir = '{}/{}'.format(code_dir, config_info.get('destdir')) 380 cmd = '{} --destdir={}'.format(cmd, dest_dir) 381 _run_cmd(cmd) 382 383 384def _build_copy_config(config_info: dict, host_platform: str, host_cpu: str): 385 copy_config = config_info.get(host_platform).get(host_cpu).get('copy_config') 386 node_config = config_info.get(host_platform).get('node_config') 387 if node_config is None: 388 node_config = config_info.get(host_platform).get(host_cpu).get('node_config') 389 copy_config.extend(node_config) 390 if host_platform == 'linux': 391 linux_copy_config = config_info.get(host_platform).get(host_cpu).get('linux_copy_config') 392 copy_config.extend(linux_copy_config) 393 elif host_platform == 'darwin': 394 darwin_copy_config = config_info.get(host_platform).get(host_cpu).get('darwin_copy_config') 395 copy_config.extend(darwin_copy_config) 396 return copy_config 397 398 399def main(): 400 parser = argparse.ArgumentParser() 401 parser.add_argument('--skip-ssl', action='store_true', help='skip ssl authentication') 402 parser.add_argument('--unsafe-perm', action='store_true', help='add "--unsafe-perm" for npm install') 403 parser.add_argument('--disable-rich', action='store_true', help='disable the rich module') 404 parser.add_argument('--enable-symlink', action='store_true', help='enable symlink while copying node_modules') 405 parser.add_argument('--build-arkuix', action='store_true', help='build ArkUI-X SDK') 406 parser.add_argument('--tool-repo', default='https://repo.huaweicloud.com', help='prebuilt file download source') 407 parser.add_argument('--npm-registry', default='https://repo.huaweicloud.com/repository/npm/', 408 help='npm download source') 409 parser.add_argument('--host-cpu', help='host cpu', required=True) 410 parser.add_argument('--host-platform', help='host platform', required=True) 411 parser.add_argument('--glibc-version', help='glibc version', required=False) 412 parser.add_argument('--config-file', help='prebuilts download config file') 413 args = parser.parse_args() 414 args.code_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 415 if args.skip_ssl: 416 ssl._create_default_https_context = ssl._create_unverified_context 417 418 host_platform = args.host_platform 419 host_cpu = args.host_cpu 420 glibc_version = args.glibc_version 421 tool_repo = args.tool_repo 422 if args.build_arkuix: 423 config_file = os.path.join(args.code_dir, 'build_plugins/prebuilts_download_config.json') 424 elif args.config_file: 425 config_file = args.config_file 426 else: 427 config_file = os.path.join(args.code_dir, 'build/prebuilts_download_config.json') 428 config_info = read_json_file(config_file) 429 if _is_system_component(): 430 args.npm_install_config = config_info.get('npm_install_path') 431 node_modules_copy_config = config_info.get('node_modules_copy') 432 else: 433 args.npm_install_config = [] 434 node_modules_copy_config = [] 435 file_handle_config = config_info.get('file_handle_config') 436 437 args.bin_dir = os.path.join(args.code_dir, config_info.get('prebuilts_download_dir')) 438 if not os.path.exists(args.bin_dir): 439 os.makedirs(args.bin_dir, exist_ok=True) 440 copy_config = _build_copy_config(config_info, host_platform, host_cpu) 441 if args.disable_rich: 442 _hwcloud_download(args, copy_config, args.bin_dir, args.code_dir, glibc_version) 443 else: 444 args.progress = _import_rich_module() 445 with args.progress: 446 _hwcloud_download(args, copy_config, args.bin_dir, args.code_dir, glibc_version) 447 448 _file_handle(file_handle_config, args.code_dir, args.host_platform, host_cpu) 449 install_config = config_info.get(host_platform).get(host_cpu).get('install') 450 retry_times = 0 451 max_retry_times = 2 452 args.success_installed = [] 453 args.parallel_install = True 454 while retry_times <= max_retry_times: 455 result, error = _npm_install(args) 456 if result: 457 break 458 print("npm install error, error info: %s" % error) 459 args.parallel_install = False 460 if retry_times == max_retry_times: 461 for error_info in error.split('\n'): 462 if error_info.endswith('debug.log'): 463 log_path = error_info.split()[-1] 464 cmd = ['cat', log_path] 465 process_cat = subprocess.Popen(cmd) 466 process_cat.communicate(timeout=60) 467 raise Exception("npm install error with three times, prebuilts download exit") 468 retry_times += 1 469 _node_modules_copy(node_modules_copy_config, args.code_dir, args.enable_symlink) 470 if install_config: 471 _install(install_config, args.code_dir) 472 473 # delete uninstalled tools 474 uninstalled_tools = config_info.get('uninstalled_tools') 475 for tool_path in uninstalled_tools: 476 subprocess.run(['rm', '-rf', tool_path]) 477 478if __name__ == '__main__': 479 sys.exit(main()) 480