1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2022 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import os 17import sys 18import argparse 19import subprocess 20import tarfile 21import zipfile 22import ssl 23import shutil 24from multiprocessing import cpu_count 25from concurrent.futures import ThreadPoolExecutor, as_completed 26from functools import partial 27from urllib.request import urlopen 28import urllib.error 29from rich.progress import ( 30 BarColumn, 31 DownloadColumn, 32 Progress, 33 TaskID, 34 TextColumn, 35 TimeRemainingColumn, 36 TransferSpeedColumn, 37) 38from util import read_json_file 39 40progress = Progress( 41 TextColumn("[bold blue]{task.fields[filename]}", justify="right"), 42 BarColumn(bar_width=None), 43 "[progress.percentage]{task.percentage:>3.1f}%", 44 "•", 45 DownloadColumn(), 46 "•", 47 TransferSpeedColumn(), 48 "•", 49 TimeRemainingColumn(), 50) 51 52def _run_cmd(cmd): 53 res = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, 54 stderr=subprocess.PIPE) 55 sout, serr = res.communicate() 56 return sout.rstrip().decode('utf-8'), serr, res.returncode 57 58def _check_sha256(check_url, local_file): 59 check_sha256_cmd = ''.join(['curl -s -k ', check_url, '.sha256']) 60 local_sha256_cmd = ''.join(['sha256sum ', local_file, "|cut -d ' ' -f1"]) 61 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 62 local_sha256, err, returncode = _run_cmd(local_sha256_cmd) 63 return check_sha256 == local_sha256 64 65def _check_sha256_by_mark(args, check_url, code_dir, unzip_dir, unzip_filename): 66 check_sha256_cmd = ''.join(['curl -s -k ', check_url, '.sha256']) 67 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 68 mark_file_dir = os.path.join(code_dir, unzip_dir) 69 mark_file_name = ''.join([check_sha256, '.', unzip_filename, '.mark']) 70 mark_file_path = os.path.join(mark_file_dir, mark_file_name) 71 args.mark_file_path = mark_file_path 72 return os.path.exists(mark_file_path) 73 74def _config_parse(config, tool_repo): 75 unzip_dir = config.get('unzip_dir') 76 huaweicloud_url = ''.join([tool_repo, config.get('file_path')]) 77 unzip_filename = config.get('unzip_filename') 78 md5_huaweicloud_url_cmd = ''.join(['echo ', huaweicloud_url, "|md5sum|cut -d ' ' -f1"]) 79 md5_huaweicloud_url, err, returncode = _run_cmd(md5_huaweicloud_url_cmd) 80 bin_file = os.path.basename(huaweicloud_url) 81 return unzip_dir, huaweicloud_url, unzip_filename, md5_huaweicloud_url, bin_file 82 83def _uncompress(args, src_file, code_dir, unzip_dir, unzip_filename, mark_file_path): 84 dest_dir = os.path.join(code_dir, unzip_dir) 85 if src_file[-3:] == 'zip': 86 cmd = 'unzip -o {} -d {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 87 elif src_file[-6:] == 'tar.gz': 88 cmd = 'tar -xvzf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 89 else: 90 cmd = 'tar -xvf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 91 _run_cmd(cmd) 92 93 # npm install 94 if os.path.basename(unzip_dir) == 'nodejs': 95 _npm_install(args, code_dir, unzip_dir, unzip_filename) 96 97def _copy_url(args, task_id, url, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path): 98 # download files 99 download_buffer_size = 32768 100 progress.console.log('Requesting {}'.format(url)) 101 try: 102 response = urlopen(url) 103 except urllib.error.HTTPError as e: 104 progress.console.log("Failed to open {}, HTTPError: {}".format(url, e.code), style='red') 105 progress.update(task_id, total=int(response.info()["Content-length"])) 106 with open(local_file, "wb") as dest_file: 107 progress.start_task(task_id) 108 for data in iter(partial(response.read, download_buffer_size), b""): 109 dest_file.write(data) 110 progress.update(task_id, advance=len(data)) 111 progress.console.log("Downloaded {}".format(local_file)) 112 113 # decompressing files 114 progress.console.log("Decompressing {}".format(local_file)) 115 _uncompress(args, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path) 116 progress.console.log("Decompressed {}".format(local_file)) 117 118def _hwcloud_download(args, config, bin_dir, code_dir): 119 try: 120 cnt = cpu_count() 121 except: 122 cnt = 1 123 with progress: 124 with ThreadPoolExecutor(max_workers=cnt) as pool: 125 tasks = dict() 126 for config_info in config: 127 unzip_dir, huaweicloud_url, unzip_filename, md5_huaweicloud_url, bin_file = _config_parse(config_info, 128 args.tool_repo) 129 abs_unzip_dir = os.path.join(code_dir, unzip_dir) 130 if not os.path.exists(abs_unzip_dir): 131 os.makedirs(abs_unzip_dir) 132 if _check_sha256_by_mark(args, huaweicloud_url, code_dir, unzip_dir, unzip_filename): 133 progress.console.log('{}, Sha256 markword check OK.'.format(huaweicloud_url), style='green') 134 if os.path.basename(abs_unzip_dir) == 'nodejs': 135 _npm_install(args, code_dir, unzip_dir, unzip_filename) 136 else: 137 _run_cmd(''.join(['rm -rf ', code_dir, '/', unzip_dir, '/*.', unzip_filename, '.mark'])) 138 _run_cmd(''.join(['rm -rf ', code_dir, '/', unzip_dir, '/', unzip_filename])) 139 local_file = os.path.join(bin_dir, ''.join([md5_huaweicloud_url, '.', bin_file])) 140 if os.path.exists(local_file): 141 if _check_sha256(huaweicloud_url, local_file): 142 progress.console.log('{}, Sha256 check download OK.'.format(local_file), style='green') 143 task = pool.submit(_uncompress, args, local_file, code_dir, unzip_dir, unzip_filename, 144 args.mark_file_path) 145 tasks[task] = os.path.basename(huaweicloud_url) 146 else: 147 os.remove(local_file) 148 else: 149 filename = huaweicloud_url.split("/")[-1] 150 task_id = progress.add_task("download", filename=filename, start=False) 151 task = pool.submit(_copy_url, args, task_id, huaweicloud_url, local_file, code_dir, unzip_dir, 152 unzip_filename, args.mark_file_path) 153 tasks[task] = os.path.basename(huaweicloud_url) 154 for task in as_completed(tasks): 155 progress.console.log('{}, download and decompress completed'.format(tasks.get(task)), style='green') 156 157def _npm_install(args, code_dir, unzip_dir, unzip_filename): 158 procs = [] 159 skip_ssl_cmd = '' 160 unsafe_perm_cmd = '' 161 os.environ['PATH'] = '{}/{}/{}/bin:{}'.format(code_dir, unzip_dir, unzip_filename, os.environ.get('PATH')) 162 for install_info in args.npm_install_config: 163 full_code_path = os.path.join(code_dir, install_info) 164 full_code_path = os.path.join(install_info) 165 if os.path.exists(full_code_path): 166 npm = '{}/{}/{}/bin/npm'.format(code_dir, unzip_dir, unzip_filename) 167 if args.skip_ssl: 168 skip_ssl_cmd = '{} config set strict-ssl false;'.format(npm) 169 if args.unsafe_perm: 170 unsafe_perm_cmd = '--unsafe-perm;' 171 cmd = 'cd {};{} config set registry {};{}{} cache clean -f;{} install {}'.format( 172 full_code_path, npm, args.npm_registry, skip_ssl_cmd, npm, npm, unsafe_perm_cmd) 173 proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 174 procs.append(proc) 175 else: 176 raise Exception("{} not exist, it shouldn't happen, pls check...".format(full_code_path)) 177 for proc in procs: 178 out, err = proc.communicate() 179 if proc.returncode: 180 raise Exception(err.decode()) 181 182def _node_modules_copy(config, code_dir): 183 for config_info in config: 184 src_dir = os.path.join(code_dir, config_info.get('src')) 185 dest_dir = os.path.join(code_dir, config_info.get('dest')) 186 use_symlink = config_info.get('use_symlink') 187 if os.path.exists(os.path.dirname(dest_dir)): 188 shutil.rmtree(os.path.dirname(dest_dir)) 189 if use_symlink == 'True': 190 os.makedirs(os.path.dirname(dest_dir)) 191 os.symlink(src_dir, dest_dir) 192 else: 193 shutil.copytree(src_dir, dest_dir, symlinks=True) 194 195def _file_handle(config, code_dir): 196 for config_info in config: 197 src_dir = ''.join([code_dir, config_info.get('src')]) 198 dest_dir = ''.join([code_dir, config_info.get('dest')]) 199 tmp_dir = config_info.get('tmp') 200 symlink_src = config_info.get('symlink_src') 201 symlink_dest = config_info.get('symlink_dest') 202 if os.path.exists(src_dir): 203 if tmp_dir: 204 tmp_dir = ''.join([code_dir, tmp_dir]) 205 shutil.move(src_dir, tmp_dir) 206 cmd = 'mv {}/*.mark {}'.format(dest_dir, tmp_dir) 207 _run_cmd(cmd) 208 if os.path.exists(dest_dir): 209 shutil.rmtree(dest_dir) 210 shutil.move(tmp_dir, dest_dir) 211 elif symlink_src and symlink_dest: 212 if os.path.exists(dest_dir): 213 shutil.rmtree(dest_dir) 214 shutil.move(src_dir, dest_dir) 215 os.symlink(''.join([dest_dir, symlink_src]), ''.join([dest_dir, symlink_dest])) 216 else: 217 _run_cmd('chmod 755 {} -R'.format(dest_dir)) 218 219def main(): 220 parser = argparse.ArgumentParser() 221 parser.add_argument('--skip-ssl', action='store_true', help='skip ssl authentication') 222 parser.add_argument('--unsafe-perm', action='store_true', help='add "--unsafe-perm" for npm install') 223 parser.add_argument('--tool-repo', default='https://repo.huaweicloud.com', help='prebuilt file download source') 224 parser.add_argument('--npm-registry', default='https://repo.huaweicloud.com/repository/npm/', 225 help='npm download source') 226 parser.add_argument('--host-cpu', help='host cpu', required=True) 227 parser.add_argument('--host-platform', help='host platform', required=True) 228 args = parser.parse_args() 229 args.code_dir = os.path.abspath(os.path.join(os.getcwd())) 230 if args.skip_ssl: 231 ssl._create_default_https_context = ssl._create_unverified_context 232 233 host_platform = args.host_platform 234 host_cpu = args.host_cpu 235 tool_repo = args.tool_repo 236 config_file = os.path.join(args.code_dir, 'toolchain/build/prebuilts_download/prebuilts_download_config.json') 237 config_info = read_json_file(config_file) 238 args.npm_install_config = config_info.get('npm_install_path') 239 node_modules_copy_config = config_info.get('node_modules_copy') 240 file_handle_config = config_info.get('file_handle_config') 241 242 args.bin_dir = os.path.join(args.code_dir, config_info.get('prebuilts_download_dir')) 243 if not os.path.exists(args.bin_dir): 244 os.makedirs(args.bin_dir) 245 copy_config = config_info.get(host_platform).get(host_cpu).get('copy_config') 246 if host_platform == 'linux': 247 linux_copy_config = config_info.get(host_platform).get(host_cpu).get('linux_copy_config') 248 copy_config.extend(linux_copy_config) 249 elif host_platform == 'darwin': 250 darwin_copy_config = config_info.get(host_platform).get(host_cpu).get('darwin_copy_config') 251 copy_config.extend(darwin_copy_config) 252 _hwcloud_download(args, copy_config, args.bin_dir, args.code_dir) 253 _file_handle(file_handle_config, args.code_dir) 254 _node_modules_copy(node_modules_copy_config, args.code_dir) 255 256if __name__ == '__main__': 257 sys.exit(main()) 258