1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2022 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import os 17import sys 18import argparse 19import subprocess 20import ssl 21import shutil 22import importlib 23import time 24import pathlib 25from multiprocessing import cpu_count 26from concurrent.futures import ThreadPoolExecutor, as_completed 27from functools import partial 28from urllib.request import urlopen 29import urllib.error 30from scripts.util.file_utils import read_json_file 31 32def _run_cmd(cmd): 33 res = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, 34 stderr=subprocess.PIPE) 35 sout, serr = res.communicate() 36 return sout.rstrip().decode('utf-8'), serr, res.returncode 37 38def _check_sha256(check_url, local_file): 39 check_sha256_cmd = 'curl -s -k ' + check_url + '.sha256' 40 local_sha256_cmd = 'sha256sum ' + local_file + "|cut -d ' ' -f1" 41 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 42 local_sha256, err, returncode = _run_cmd(local_sha256_cmd) 43 return check_sha256 == local_sha256 44 45def _check_sha256_by_mark(args, check_url, code_dir, unzip_dir, unzip_filename): 46 check_sha256_cmd = 'curl -s -k ' + check_url + '.sha256' 47 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 48 mark_file_dir = os.path.join(code_dir, unzip_dir) 49 mark_file_name = check_sha256 + '.' + unzip_filename + '.mark' 50 mark_file_path = os.path.join(mark_file_dir, mark_file_name) 51 args.mark_file_path = mark_file_path 52 return os.path.exists(mark_file_path) 53 54def _config_parse(config, tool_repo): 55 parse_dict = dict() 56 parse_dict['unzip_dir'] = config.get('unzip_dir') 57 parse_dict['huaweicloud_url'] = tool_repo + config.get('file_path') 58 parse_dict['unzip_filename'] = config.get('unzip_filename') 59 md5_huaweicloud_url_cmd = 'echo ' + parse_dict.get('huaweicloud_url') + "|md5sum|cut -d ' ' -f1" 60 parse_dict['md5_huaweicloud_url'], err, returncode = _run_cmd(md5_huaweicloud_url_cmd) 61 parse_dict['bin_file'] = os.path.basename(parse_dict.get('huaweicloud_url')) 62 return parse_dict 63 64def _uncompress(args, src_file, code_dir, unzip_dir, unzip_filename, mark_file_path): 65 dest_dir = os.path.join(code_dir, unzip_dir) 66 if src_file[-3:] == 'zip': 67 cmd = 'unzip -o {} -d {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 68 elif src_file[-6:] == 'tar.gz': 69 cmd = 'tar -xvzf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 70 else: 71 cmd = 'tar -xvf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 72 _run_cmd(cmd) 73 74 75def _copy_url(args, task_id, url, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path, progress): 76 retry_times = 0 77 max_retry_times = 3 78 while retry_times < max_retry_times: 79 # download files 80 download_buffer_size = 32768 81 progress.console.log('Requesting {}'.format(url)) 82 try: 83 response = urlopen(url) 84 except urllib.error.HTTPError as e: 85 progress.console.log("Failed to open {}, HTTPError: {}".format(url, e.code), style='red') 86 progress.update(task_id, total=int(response.info()["Content-length"])) 87 with open(local_file, "wb") as dest_file: 88 progress.start_task(task_id) 89 for data in iter(partial(response.read, download_buffer_size), b""): 90 dest_file.write(data) 91 progress.update(task_id, advance=len(data)) 92 progress.console.log("Downloaded {}".format(local_file)) 93 94 if os.path.exists(local_file): 95 if _check_sha256(url, local_file): 96 # decompressing files 97 progress.console.log("Decompressing {}".format(local_file)) 98 _uncompress(args, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path) 99 progress.console.log("Decompressed {}".format(local_file)) 100 break 101 else: 102 os.remove(local_file) 103 retry_times += 1 104 if retry_times == max_retry_times: 105 print('{}, download failed with three times retry, please check network status. Prebuilts download exit.'.format(local_file)) 106 # todo, merge with copy_url_disable_rich 107 sys.exit(1) 108 109 110def _copy_url_disable_rich(args, url, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path): 111 # download files 112 download_buffer_size = 32768 113 print('Requesting {}, please wait'.format(url)) 114 try: 115 response = urlopen(url) 116 except urllib.error.HTTPError as e: 117 print("Failed to open {}, HTTPError: {}".format(url, e.code)) 118 with open(local_file, "wb") as dest_file: 119 for data in iter(partial(response.read, download_buffer_size), b""): 120 dest_file.write(data) 121 print("Downloaded {}".format(local_file)) 122 123 # decompressing files 124 print("Decompressing {}, please wait".format(local_file)) 125 _uncompress(args, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path) 126 print("Decompressed {}".format(local_file)) 127 128def _is_system_component(): 129 root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 130 if pathlib.Path(root_dir + "/interface/sdk-js").exists() or pathlib.Path(root_dir + "/foundation/arkui").exists() or pathlib.Path(root_dir + "/arkcompiler").exists(): 131 return True 132 else: 133 return False 134 135def _hwcloud_download(args, config, bin_dir, code_dir): 136 try: 137 cnt = cpu_count() 138 except: 139 cnt = 1 140 with ThreadPoolExecutor(max_workers=cnt) as pool: 141 tasks = dict() 142 for config_info in config: 143 parse_dict = _config_parse(config_info, args.tool_repo) 144 unzip_dir = parse_dict.get('unzip_dir') 145 huaweicloud_url = parse_dict.get('huaweicloud_url') 146 unzip_filename = parse_dict.get('unzip_filename') 147 md5_huaweicloud_url = parse_dict.get('md5_huaweicloud_url') 148 bin_file = parse_dict.get('bin_file') 149 abs_unzip_dir = os.path.join(code_dir, unzip_dir) 150 if not os.path.exists(abs_unzip_dir): 151 os.makedirs(abs_unzip_dir, exist_ok=True) 152 if _check_sha256_by_mark(args, huaweicloud_url, code_dir, unzip_dir, unzip_filename): 153 if not args.disable_rich: 154 args.progress.console.log('{}, Sha256 markword check OK.'.format(huaweicloud_url), style='green') 155 else: 156 print('{}, Sha256 markword check OK.'.format(huaweicloud_url)) 157 else: 158 _run_cmd(('rm -rf {}/{}/*.{}.mark').format(code_dir, unzip_dir, unzip_filename)) 159 _run_cmd(('rm -rf {}/{}/{}').format(code_dir, unzip_dir, unzip_filename)) 160 local_file = os.path.join(bin_dir, '{}.{}'.format(md5_huaweicloud_url, bin_file)) 161 if os.path.exists(local_file): 162 if _check_sha256(huaweicloud_url, local_file): 163 if not args.disable_rich: 164 args.progress.console.log('{}, Sha256 check download OK.'.format(local_file), style='green') 165 else: 166 print('{}, Sha256 check download OK. Start decompression, please wait'.format(local_file)) 167 task = pool.submit(_uncompress, args, local_file, code_dir, 168 unzip_dir, unzip_filename, args.mark_file_path) 169 tasks[task] = os.path.basename(huaweicloud_url) 170 continue 171 else: 172 os.remove(local_file) 173 filename = huaweicloud_url.split("/")[-1] 174 if not args.disable_rich: 175 task_id = args.progress.add_task("download", filename=filename, start=False) 176 task = pool.submit(_copy_url, args, task_id, huaweicloud_url, local_file, code_dir, 177 unzip_dir, unzip_filename, args.mark_file_path, args.progress) 178 tasks[task] = os.path.basename(huaweicloud_url) 179 else: 180 task = pool.submit(_copy_url_disable_rich, args, huaweicloud_url, local_file, code_dir, 181 unzip_dir, unzip_filename, args.mark_file_path) 182 183 for task in as_completed(tasks): 184 if not args.disable_rich: 185 args.progress.console.log('{}, download and decompress completed'.format(tasks.get(task)), 186 style='green') 187 else: 188 print('{}, download and decompress completed'.format(tasks.get(task))) 189 190def _npm_install(args): 191 procs = [] 192 node_path = 'prebuilts/build-tools/common/nodejs/current/bin' 193 os.environ['PATH'] = '{}/{}:{}'.format(args.code_dir, node_path, os.environ.get('PATH')) 194 npm = os.path.join(args.code_dir, node_path, 'npm') 195 if args.skip_ssl: 196 skip_ssl_cmd = '{} config set strict-ssl false;'.format(npm) 197 _run_cmd(skip_ssl_cmd) 198 npm_clean_cmd = '{} cache clean -f'.format(npm) 199 _run_cmd(npm_clean_cmd) 200 print('start npm install, please wait.') 201 for install_info in args.npm_install_config: 202 full_code_path = os.path.join(args.code_dir, install_info) 203 basename = os.path.basename(full_code_path) 204 npm_cache_dir = os.path.join('~/.npm/_cacache', basename) 205 if os.path.exists(full_code_path): 206 cmd = [npm, 'install', '--registry', args.npm_registry, '--cache', npm_cache_dir] 207 if args.unsafe_perm: 208 cmd.append('--unsafe-perm') 209 proc = subprocess.Popen(cmd, cwd=full_code_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 210 # wait proc Popen with 0.1 second 211 time.sleep(0.1) 212 procs.append(proc) 213 else: 214 raise Exception("{} not exist, it shouldn't happen, pls check...".format(full_code_path)) 215 for proc in procs: 216 out, err = proc.communicate() 217 if proc.returncode: 218 raise Exception(err.decode()) 219 220 221def _node_modules_copy(config, code_dir, enable_symlink): 222 for config_info in config: 223 src_dir = os.path.join(code_dir, config_info.get('src')) 224 dest_dir = os.path.join(code_dir, config_info.get('dest')) 225 use_symlink = config_info.get('use_symlink') 226 if os.path.exists(os.path.dirname(dest_dir)): 227 shutil.rmtree(os.path.dirname(dest_dir)) 228 if use_symlink == 'True' and enable_symlink == True: 229 os.makedirs(os.path.dirname(dest_dir), exist_ok=True) 230 os.symlink(src_dir, dest_dir) 231 else: 232 shutil.copytree(src_dir, dest_dir, symlinks=True) 233 234def _file_handle(config, code_dir, host_platform): 235 for config_info in config: 236 src_dir = code_dir + config_info.get('src') 237 dest_dir = code_dir + config_info.get('dest') 238 tmp_dir = config_info.get('tmp') 239 symlink_src = config_info.get('symlink_src') 240 symlink_dest = config_info.get('symlink_dest') 241 rename = config_info.get('rename') 242 if os.path.exists(src_dir): 243 if tmp_dir: 244 tmp_dir = code_dir + tmp_dir 245 shutil.move(src_dir, tmp_dir) 246 cmd = 'mv {}/*.mark {}'.format(dest_dir, tmp_dir) 247 _run_cmd(cmd) 248 if os.path.exists(dest_dir): 249 shutil.rmtree(dest_dir) 250 shutil.move(tmp_dir, dest_dir) 251 elif rename: 252 if os.path.exists(dest_dir) and dest_dir != src_dir: 253 shutil.rmtree(dest_dir) 254 shutil.move(src_dir, dest_dir) 255 if symlink_src and symlink_dest: 256 if os.path.exists(dest_dir + symlink_dest): 257 os.remove(dest_dir + symlink_dest) 258 if host_platform == 'darwin' and os.path.basename(dest_dir) == "nodejs": 259 symlink_src = symlink_src.replace('linux', 'darwin') 260 os.symlink(os.path.basename(symlink_src), dest_dir + symlink_dest) 261 else: 262 _run_cmd('chmod 755 {} -R'.format(dest_dir)) 263 264def _import_rich_module(): 265 module = importlib.import_module('rich.progress') 266 progress = module.Progress( 267 module.TextColumn("[bold blue]{task.fields[filename]}", justify="right"), 268 module.BarColumn(bar_width=None), 269 "[progress.percentage]{task.percentage:>3.1f}%", 270 "•", 271 module.DownloadColumn(), 272 "•", 273 module.TransferSpeedColumn(), 274 "•", 275 module.TimeRemainingColumn(), 276 ) 277 return progress 278 279 280def _install(config, code_dir): 281 for config_info in config: 282 install_dir = '{}/{}'.format(code_dir, config_info.get('install_dir')) 283 script = config_info.get('script') 284 cmd = '{}/{}'.format(install_dir, script) 285 args = config_info.get('args') 286 for arg in args: 287 for key in arg.keys(): 288 cmd = '{} --{}={}'.format(cmd, key, arg[key]) 289 dest_dir = '{}/{}'.format(code_dir, config_info.get('destdir')) 290 cmd = '{} --destdir={}'.format(cmd, dest_dir) 291 _run_cmd(cmd) 292 293def main(): 294 parser = argparse.ArgumentParser() 295 parser.add_argument('--skip-ssl', action='store_true', help='skip ssl authentication') 296 parser.add_argument('--unsafe-perm', action='store_true', help='add "--unsafe-perm" for npm install') 297 parser.add_argument('--disable-rich', action='store_true', help='disable the rich module') 298 parser.add_argument('--enable-symlink', action='store_true', help='enable symlink while copying node_modules') 299 parser.add_argument('--build-arkuix', action='store_true', help='build ArkUI-X SDK') 300 parser.add_argument('--tool-repo', default='https://repo.huaweicloud.com', help='prebuilt file download source') 301 parser.add_argument('--npm-registry', default='https://repo.huaweicloud.com/repository/npm/', 302 help='npm download source') 303 parser.add_argument('--host-cpu', help='host cpu', required=True) 304 parser.add_argument('--host-platform', help='host platform', required=True) 305 parser.add_argument('--config-file', help='prebuilts download config file') 306 args = parser.parse_args() 307 args.code_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 308 if args.skip_ssl: 309 ssl._create_default_https_context = ssl._create_unverified_context 310 311 host_platform = args.host_platform 312 host_cpu = args.host_cpu 313 tool_repo = args.tool_repo 314 if args.build_arkuix: 315 config_file = os.path.join(args.code_dir, 'build_plugins/prebuilts_download_config.json') 316 elif args.config_file: 317 config_file = args.config_file 318 else: 319 config_file = os.path.join(args.code_dir, 'build/prebuilts_download_config.json') 320 config_info = read_json_file(config_file) 321 if _is_system_component(): 322 args.npm_install_config = config_info.get('npm_install_path') 323 node_modules_copy_config = config_info.get('node_modules_copy') 324 else: 325 args.npm_install_config = [] 326 node_modules_copy_config = [] 327 file_handle_config = config_info.get('file_handle_config') 328 329 args.bin_dir = os.path.join(args.code_dir, config_info.get('prebuilts_download_dir')) 330 if not os.path.exists(args.bin_dir): 331 os.makedirs(args.bin_dir, exist_ok=True) 332 copy_config = config_info.get(host_platform).get(host_cpu).get('copy_config') 333 node_config = config_info.get(host_platform).get('node_config') 334 copy_config.extend(node_config) 335 install_config = config_info.get(host_platform).get(host_cpu).get('install') 336 if host_platform == 'linux': 337 linux_copy_config = config_info.get(host_platform).get(host_cpu).get('linux_copy_config') 338 copy_config.extend(linux_copy_config) 339 elif host_platform == 'darwin': 340 darwin_copy_config = config_info.get(host_platform).get(host_cpu).get('darwin_copy_config') 341 copy_config.extend(darwin_copy_config) 342 if args.disable_rich: 343 _hwcloud_download(args, copy_config, args.bin_dir, args.code_dir) 344 else: 345 args.progress = _import_rich_module() 346 with args.progress: 347 _hwcloud_download(args, copy_config, args.bin_dir, args.code_dir) 348 349 _file_handle(file_handle_config, args.code_dir, args.host_platform) 350 _npm_install(args) 351 _node_modules_copy(node_modules_copy_config, args.code_dir, args.enable_symlink) 352 if install_config: 353 _install(install_config, args.code_dir) 354 355 # delete uninstalled tools 356 uninstalled_tools = config_info.get('uninstalled_tools') 357 for tool_path in uninstalled_tools: 358 subprocess.run(['rm', '-rf', tool_path]) 359 360if __name__ == '__main__': 361 sys.exit(main()) 362