1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2022 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import os 17import sys 18import argparse 19import subprocess 20import ssl 21import shutil 22import importlib 23import time 24from multiprocessing import cpu_count 25from concurrent.futures import ThreadPoolExecutor, as_completed 26from functools import partial 27from urllib.request import urlopen 28import urllib.error 29from scripts.util.file_utils import read_json_file 30 31def _run_cmd(cmd): 32 res = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, 33 stderr=subprocess.PIPE) 34 sout, serr = res.communicate() 35 return sout.rstrip().decode('utf-8'), serr, res.returncode 36 37def _check_sha256(check_url, local_file): 38 check_sha256_cmd = 'curl -s -k ' + check_url + '.sha256' 39 local_sha256_cmd = 'sha256sum ' + local_file + "|cut -d ' ' -f1" 40 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 41 local_sha256, err, returncode = _run_cmd(local_sha256_cmd) 42 return check_sha256 == local_sha256 43 44def _check_sha256_by_mark(args, check_url, code_dir, unzip_dir, unzip_filename): 45 check_sha256_cmd = 'curl -s -k ' + check_url + '.sha256' 46 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 47 mark_file_dir = os.path.join(code_dir, unzip_dir) 48 mark_file_name = check_sha256 + '.' + unzip_filename + '.mark' 49 mark_file_path = os.path.join(mark_file_dir, mark_file_name) 50 args.mark_file_path = mark_file_path 51 return os.path.exists(mark_file_path) 52 53def _config_parse(config, tool_repo): 54 parse_dict = dict() 55 parse_dict['unzip_dir'] = config.get('unzip_dir') 56 parse_dict['huaweicloud_url'] = tool_repo + config.get('file_path') 57 parse_dict['unzip_filename'] = config.get('unzip_filename') 58 md5_huaweicloud_url_cmd = 'echo ' + parse_dict.get('huaweicloud_url') + "|md5sum|cut -d ' ' -f1" 59 parse_dict['md5_huaweicloud_url'], err, returncode = _run_cmd(md5_huaweicloud_url_cmd) 60 parse_dict['bin_file'] = os.path.basename(parse_dict.get('huaweicloud_url')) 61 return parse_dict 62 63def _uncompress(args, src_file, code_dir, unzip_dir, unzip_filename, mark_file_path): 64 dest_dir = os.path.join(code_dir, unzip_dir) 65 if src_file[-3:] == 'zip': 66 cmd = 'unzip -o {} -d {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 67 elif src_file[-6:] == 'tar.gz': 68 cmd = 'tar -xvzf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 69 else: 70 cmd = 'tar -xvf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 71 _run_cmd(cmd) 72 73 # npm install 74 if os.path.basename(unzip_dir) == 'nodejs': 75 _npm_install(args, code_dir, unzip_dir, unzip_filename) 76 77def _copy_url(args, task_id, url, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path, progress): 78 # download files 79 download_buffer_size = 32768 80 progress.console.log('Requesting {}'.format(url)) 81 try: 82 response = urlopen(url) 83 except urllib.error.HTTPError as e: 84 progress.console.log("Failed to open {}, HTTPError: {}".format(url, e.code), style='red') 85 progress.update(task_id, total=int(response.info()["Content-length"])) 86 with open(local_file, "wb") as dest_file: 87 progress.start_task(task_id) 88 for data in iter(partial(response.read, download_buffer_size), b""): 89 dest_file.write(data) 90 progress.update(task_id, advance=len(data)) 91 progress.console.log("Downloaded {}".format(local_file)) 92 93 # decompressing files 94 progress.console.log("Decompressing {}".format(local_file)) 95 _uncompress(args, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path) 96 progress.console.log("Decompressed {}".format(local_file)) 97 98def _copy_url_disable_rich(args, url, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path): 99 # download files 100 download_buffer_size = 32768 101 print('Requesting {}, please wait'.format(url)) 102 try: 103 response = urlopen(url) 104 except urllib.error.HTTPError as e: 105 print("Failed to open {}, HTTPError: {}".format(url, e.code)) 106 with open(local_file, "wb") as dest_file: 107 for data in iter(partial(response.read, download_buffer_size), b""): 108 dest_file.write(data) 109 print("Downloaded {}".format(local_file)) 110 111 # decompressing files 112 print("Decompressing {}, please wait".format(local_file)) 113 _uncompress(args, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path) 114 print("Decompressed {}".format(local_file)) 115 116def _hwcloud_download(args, config, bin_dir, code_dir): 117 try: 118 cnt = cpu_count() 119 except: 120 cnt = 1 121 with ThreadPoolExecutor(max_workers=cnt) as pool: 122 tasks = dict() 123 for config_info in config: 124 parse_dict = _config_parse(config_info, args.tool_repo) 125 unzip_dir = parse_dict.get('unzip_dir') 126 huaweicloud_url = parse_dict.get('huaweicloud_url') 127 unzip_filename = parse_dict.get('unzip_filename') 128 md5_huaweicloud_url = parse_dict.get('md5_huaweicloud_url') 129 bin_file = parse_dict.get('bin_file') 130 abs_unzip_dir = os.path.join(code_dir, unzip_dir) 131 if not os.path.exists(abs_unzip_dir): 132 os.makedirs(abs_unzip_dir) 133 if _check_sha256_by_mark(args, huaweicloud_url, code_dir, unzip_dir, unzip_filename): 134 if not args.disable_rich: 135 args.progress.console.log('{}, Sha256 markword check OK.'.format(huaweicloud_url), style='green') 136 else: 137 print('{}, Sha256 markword check OK.'.format(huaweicloud_url)) 138 if os.path.basename(abs_unzip_dir) == 'nodejs': 139 _npm_install(args, code_dir, unzip_dir, unzip_filename) 140 else: 141 _run_cmd('rm -rf ' + code_dir + '/' + unzip_dir + '/*.' + unzip_filename + '.mark') 142 _run_cmd('rm -rf ' + code_dir + '/' + unzip_dir + '/' + unzip_filename) 143 local_file = os.path.join(bin_dir, md5_huaweicloud_url + '.' + bin_file) 144 if os.path.exists(local_file): 145 if _check_sha256(huaweicloud_url, local_file): 146 if not args.disable_rich: 147 args.progress.console.log('{}, Sha256 check download OK.'.format(local_file), style='green') 148 else: 149 print('{}, Sha256 check download OK. Start decompression, please wait'.format(local_file)) 150 task = pool.submit(_uncompress, args, local_file, code_dir, 151 unzip_dir, unzip_filename, args.mark_file_path) 152 tasks[task] = os.path.basename(huaweicloud_url) 153 else: 154 os.remove(local_file) 155 else: 156 filename = huaweicloud_url.split("/")[-1] 157 if not args.disable_rich: 158 task_id = args.progress.add_task("download", filename=filename, start=False) 159 task = pool.submit(_copy_url, args, task_id, huaweicloud_url, local_file, code_dir, 160 unzip_dir, unzip_filename, args.mark_file_path, args.progress) 161 tasks[task] = os.path.basename(huaweicloud_url) 162 else: 163 task = pool.submit(_copy_url_disable_rich, args, huaweicloud_url, local_file, code_dir, 164 unzip_dir, unzip_filename, args.mark_file_path) 165 for task in as_completed(tasks): 166 if not args.disable_rich: 167 args.progress.console.log('{}, download and decompress completed'.format(tasks.get(task)), style='green') 168 else: 169 print('{}, download and decompress completed'.format(tasks.get(task))) 170 171def _npm_install(args, code_dir, unzip_dir, unzip_filename): 172 procs = [] 173 skip_ssl_cmd = '' 174 unsafe_perm_cmd = '' 175 os.environ['PATH'] = '{}/{}/{}/bin:{}'.format(code_dir, unzip_dir, unzip_filename, os.environ.get('PATH')) 176 for install_info in args.npm_install_config: 177 full_code_path = os.path.join(code_dir, install_info) 178 if os.path.exists(full_code_path): 179 npm = '{}/{}/{}/bin/npm'.format(code_dir, unzip_dir, unzip_filename) 180 if args.skip_ssl: 181 skip_ssl_cmd = '{} config set strict-ssl false;'.format(npm) 182 if args.unsafe_perm: 183 unsafe_perm_cmd = '--unsafe-perm;' 184 cmd = 'cd {};{}{} cache clean -f;{} install --registry {} {}'.format( 185 full_code_path, skip_ssl_cmd, npm, npm, args.npm_registry, unsafe_perm_cmd) 186 proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 187 # wait proc Popen with 0.1 second 188 time.sleep(0.1) 189 procs.append(proc) 190 else: 191 raise Exception("{} not exist, it shouldn't happen, pls check...".format(full_code_path)) 192 for proc in procs: 193 out, err = proc.communicate() 194 if proc.returncode: 195 raise Exception(err.decode()) 196 197def _node_modules_copy(config, code_dir, enable_symlink): 198 for config_info in config: 199 src_dir = os.path.join(code_dir, config_info.get('src')) 200 dest_dir = os.path.join(code_dir, config_info.get('dest')) 201 use_symlink = config_info.get('use_symlink') 202 if os.path.exists(os.path.dirname(dest_dir)): 203 shutil.rmtree(os.path.dirname(dest_dir)) 204 if use_symlink == 'True' and enable_symlink == True: 205 os.makedirs(os.path.dirname(dest_dir)) 206 os.symlink(src_dir, dest_dir) 207 else: 208 shutil.copytree(src_dir, dest_dir, symlinks=True) 209 210def _file_handle(config, code_dir): 211 for config_info in config: 212 src_dir = code_dir + config_info.get('src') 213 dest_dir = code_dir + config_info.get('dest') 214 tmp_dir = config_info.get('tmp') 215 symlink_src = config_info.get('symlink_src') 216 symlink_dest = config_info.get('symlink_dest') 217 rename = config_info.get('rename') 218 if os.path.exists(src_dir): 219 if tmp_dir: 220 tmp_dir = code_dir + tmp_dir 221 shutil.move(src_dir, tmp_dir) 222 cmd = 'mv {}/*.mark {}'.format(dest_dir, tmp_dir) 223 _run_cmd(cmd) 224 if os.path.exists(dest_dir): 225 shutil.rmtree(dest_dir) 226 shutil.move(tmp_dir, dest_dir) 227 elif rename: 228 if os.path.exists(dest_dir): 229 shutil.rmtree(dest_dir) 230 shutil.move(src_dir, dest_dir) 231 if symlink_src and symlink_dest: 232 os.symlink(dest_dir + symlink_src, dest_dir + symlink_dest) 233 else: 234 _run_cmd('chmod 755 {} -R'.format(dest_dir)) 235 236def _import_rich_module(): 237 module = importlib.import_module('rich.progress') 238 progress = module.Progress( 239 module.TextColumn("[bold blue]{task.fields[filename]}", justify="right"), 240 module.BarColumn(bar_width=None), 241 "[progress.percentage]{task.percentage:>3.1f}%", 242 "•", 243 module.DownloadColumn(), 244 "•", 245 module.TransferSpeedColumn(), 246 "•", 247 module.TimeRemainingColumn(), 248 ) 249 return progress 250 251def main(): 252 parser = argparse.ArgumentParser() 253 parser.add_argument('--skip-ssl', action='store_true', help='skip ssl authentication') 254 parser.add_argument('--unsafe-perm', action='store_true', help='add "--unsafe-perm" for npm install') 255 parser.add_argument('--disable-rich', action='store_true', help='disable the rich module') 256 parser.add_argument('--enable-symlink', action='store_true', help='enable symlink while copying node_modules') 257 parser.add_argument('--tool-repo', default='https://repo.huaweicloud.com', help='prebuilt file download source') 258 parser.add_argument('--npm-registry', default='https://repo.huaweicloud.com/repository/npm/', 259 help='npm download source') 260 parser.add_argument('--host-cpu', help='host cpu', required=True) 261 parser.add_argument('--host-platform', help='host platform', required=True) 262 args = parser.parse_args() 263 args.code_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 264 if args.skip_ssl: 265 ssl._create_default_https_context = ssl._create_unverified_context 266 267 host_platform = args.host_platform 268 host_cpu = args.host_cpu 269 tool_repo = args.tool_repo 270 config_file = os.path.join(args.code_dir, 'build/prebuilts_download_config.json') 271 config_info = read_json_file(config_file) 272 args.npm_install_config = config_info.get('npm_install_path') 273 node_modules_copy_config = config_info.get('node_modules_copy') 274 file_handle_config = config_info.get('file_handle_config') 275 276 args.bin_dir = os.path.join(args.code_dir, config_info.get('prebuilts_download_dir')) 277 if not os.path.exists(args.bin_dir): 278 os.makedirs(args.bin_dir) 279 copy_config = config_info.get(host_platform).get(host_cpu).get('copy_config') 280 node_config = config_info.get(host_platform).get('node_config') 281 copy_config.extend(node_config) 282 if host_platform == 'linux': 283 linux_copy_config = config_info.get(host_platform).get(host_cpu).get('linux_copy_config') 284 copy_config.extend(linux_copy_config) 285 elif host_platform == 'darwin': 286 darwin_copy_config = config_info.get(host_platform).get(host_cpu).get('darwin_copy_config') 287 copy_config.extend(darwin_copy_config) 288 if args.disable_rich: 289 _hwcloud_download(args, copy_config, args.bin_dir, args.code_dir) 290 else: 291 args.progress = _import_rich_module() 292 with args.progress: 293 _hwcloud_download(args, copy_config, args.bin_dir, args.code_dir) 294 295 _file_handle(file_handle_config, args.code_dir) 296 _node_modules_copy(node_modules_copy_config, args.code_dir, args.enable_symlink) 297 298if __name__ == '__main__': 299 sys.exit(main()) 300