1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2022-2023 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import os 17import sys 18import argparse 19import subprocess 20import tarfile 21import zipfile 22import ssl 23import shutil 24from multiprocessing import cpu_count 25from concurrent.futures import ThreadPoolExecutor, as_completed 26from functools import partial 27from urllib.request import urlopen 28import urllib.error 29from rich.progress import ( 30 BarColumn, 31 DownloadColumn, 32 Progress, 33 TaskID, 34 TextColumn, 35 TimeRemainingColumn, 36 TransferSpeedColumn, 37) 38from util import read_json_file 39 40progress = Progress( 41 TextColumn("[bold blue]{task.fields[filename]}", justify="right"), 42 BarColumn(bar_width=None), 43 "[progress.percentage]{task.percentage:>3.1f}%", 44 "•", 45 DownloadColumn(), 46 "•", 47 TransferSpeedColumn(), 48 "•", 49 TimeRemainingColumn(), 50) 51 52def _run_cmd(cmd): 53 res = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, 54 stderr=subprocess.PIPE) 55 sout, serr = res.communicate() 56 return sout.rstrip().decode('utf-8'), serr, res.returncode 57 58def _check_sha256(check_url, local_file): 59 check_sha256_cmd = ''.join(['curl -s -k ', check_url, '.sha256']) 60 local_sha256_cmd = ''.join(['sha256sum ', local_file, "|cut -d ' ' -f1"]) 61 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 62 local_sha256, err, returncode = _run_cmd(local_sha256_cmd) 63 return check_sha256 == local_sha256 64 65def _check_sha256_by_mark(args, check_url, code_dir, unzip_dir, unzip_filename): 66 check_sha256_cmd = ''.join(['curl -s -k ', check_url, '.sha256']) 67 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 68 mark_file_dir = os.path.join(code_dir, unzip_dir) 69 mark_file_name = ''.join([check_sha256, '.', unzip_filename, '.mark']) 70 mark_file_path = os.path.join(mark_file_dir, mark_file_name) 71 args.mark_file_path = mark_file_path 72 return os.path.exists(mark_file_path) 73 74def _config_parse(config, tool_repo): 75 unzip_dir = config.get('unzip_dir') 76 huaweicloud_url = ''.join([tool_repo, config.get('file_path')]) 77 unzip_filename = config.get('unzip_filename') 78 md5_huaweicloud_url_cmd = ''.join(['echo ', huaweicloud_url, "|md5sum|cut -d ' ' -f1"]) 79 md5_huaweicloud_url, err, returncode = _run_cmd(md5_huaweicloud_url_cmd) 80 bin_file = os.path.basename(huaweicloud_url) 81 return unzip_dir, huaweicloud_url, unzip_filename, md5_huaweicloud_url, bin_file 82 83def _uncompress(args, src_file, code_dir, unzip_dir, unzip_filename, mark_file_path): 84 dest_dir = os.path.join(code_dir, unzip_dir) 85 if src_file[-3:] == 'zip': 86 cmd = 'unzip -o {} -d {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 87 elif src_file[-6:] == 'tar.gz': 88 cmd = 'tar -xvzf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 89 else: 90 cmd = 'tar -xvf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 91 _run_cmd(cmd) 92 93def _copy_url(args, task_id, url, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path): 94 # download files 95 download_buffer_size = 32768 96 progress.console.log('Requesting {}'.format(url)) 97 try: 98 response = urlopen(url) 99 except urllib.error.HTTPError as e: 100 progress.console.log("Failed to open {}, HTTPError: {}".format(url, e.code), style='red') 101 progress.update(task_id, total=int(response.info()["Content-length"])) 102 with open(local_file, "wb") as dest_file: 103 progress.start_task(task_id) 104 for data in iter(partial(response.read, download_buffer_size), b""): 105 dest_file.write(data) 106 progress.update(task_id, advance=len(data)) 107 progress.console.log("Downloaded {}".format(local_file)) 108 109 # decompressing files 110 progress.console.log("Decompressing {}".format(local_file)) 111 _uncompress(args, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path) 112 progress.console.log("Decompressed {}".format(local_file)) 113 114def _hwcloud_download(args, config, bin_dir, code_dir): 115 try: 116 cnt = cpu_count() 117 except: 118 cnt = 1 119 with progress: 120 with ThreadPoolExecutor(max_workers=cnt) as pool: 121 tasks = dict() 122 for config_info in config: 123 unzip_dir, huaweicloud_url, unzip_filename, md5_huaweicloud_url, bin_file = _config_parse(config_info, 124 args.tool_repo) 125 abs_unzip_dir = os.path.join(code_dir, unzip_dir) 126 if not os.path.exists(abs_unzip_dir): 127 os.makedirs(abs_unzip_dir) 128 if _check_sha256_by_mark(args, huaweicloud_url, code_dir, unzip_dir, unzip_filename): 129 progress.console.log('{}, Sha256 markword check OK.'.format(huaweicloud_url), style='green') 130 else: 131 _run_cmd(''.join(['rm -rf ', code_dir, '/', unzip_dir, '/*.', unzip_filename, '.mark'])) 132 _run_cmd(''.join(['rm -rf ', code_dir, '/', unzip_dir, '/', unzip_filename])) 133 local_file = os.path.join(bin_dir, ''.join([md5_huaweicloud_url, '.', bin_file])) 134 if os.path.exists(local_file): 135 if _check_sha256(huaweicloud_url, local_file): 136 progress.console.log('{}, Sha256 check download OK.'.format(local_file), style='green') 137 task = pool.submit(_uncompress, args, local_file, code_dir, unzip_dir, unzip_filename, 138 args.mark_file_path) 139 tasks[task] = os.path.basename(huaweicloud_url) 140 else: 141 os.remove(local_file) 142 else: 143 filename = huaweicloud_url.split("/")[-1] 144 task_id = progress.add_task("download", filename=filename, start=False) 145 task = pool.submit(_copy_url, args, task_id, huaweicloud_url, local_file, code_dir, unzip_dir, 146 unzip_filename, args.mark_file_path) 147 tasks[task] = os.path.basename(huaweicloud_url) 148 for task in as_completed(tasks): 149 progress.console.log('{}, download and decompress completed'.format(tasks.get(task)), style='green') 150 151def _file_handle(config, code_dir): 152 for config_info in config: 153 src_dir = ''.join([code_dir, config_info.get('src')]) 154 dest_dir = ''.join([code_dir, config_info.get('dest')]) 155 tmp_dir = config_info.get('tmp') 156 symlink_src = config_info.get('symlink_src') 157 symlink_dest = config_info.get('symlink_dest') 158 if os.path.exists(src_dir): 159 if tmp_dir: 160 tmp_dir = ''.join([code_dir, tmp_dir]) 161 shutil.move(src_dir, tmp_dir) 162 cmd = 'mv {}/*.mark {}'.format(dest_dir, tmp_dir) 163 _run_cmd(cmd) 164 if os.path.exists(dest_dir): 165 shutil.rmtree(dest_dir) 166 shutil.move(tmp_dir, dest_dir) 167 elif symlink_src and symlink_dest: 168 if os.path.exists(dest_dir) and dest_dir != src_dir: 169 shutil.rmtree(dest_dir) 170 shutil.move(src_dir, dest_dir) 171 os.symlink(''.join([dest_dir, symlink_src]), ''.join([dest_dir, symlink_dest])) 172 else: 173 _run_cmd('chmod 755 {} -R'.format(dest_dir)) 174 175def main(): 176 parser = argparse.ArgumentParser() 177 parser.add_argument('--skip-ssl', action='store_true', help='skip ssl authentication') 178 parser.add_argument('--tool-repo', default='https://repo.huaweicloud.com', help='prebuilt file download source') 179 parser.add_argument('--host-cpu', help='host cpu', required=True) 180 parser.add_argument('--host-platform', help='host platform', required=True) 181 args = parser.parse_args() 182 args.code_dir = os.path.abspath(os.path.join(os.getcwd())) 183 if args.skip_ssl: 184 ssl._create_default_https_context = ssl._create_unverified_context 185 186 host_platform = args.host_platform 187 host_cpu = args.host_cpu 188 tool_repo = args.tool_repo 189 config_file = os.path.join(args.code_dir, 190 'arkcompiler/toolchain/build/prebuilts_download/prebuilts_download_config.json') 191 config_info = read_json_file(config_file) 192 file_handle_config = config_info.get('file_handle_config') 193 194 args.bin_dir = os.path.join(args.code_dir, config_info.get('prebuilts_download_dir')) 195 if not os.path.exists(args.bin_dir): 196 os.makedirs(args.bin_dir) 197 copy_config = config_info.get(host_platform).get(host_cpu).get('copy_config') 198 if host_platform == 'linux': 199 linux_copy_config = config_info.get(host_platform).get(host_cpu).get('linux_copy_config') 200 copy_config.extend(linux_copy_config) 201 elif host_platform == 'darwin': 202 darwin_copy_config = config_info.get(host_platform).get(host_cpu).get('darwin_copy_config') 203 copy_config.extend(darwin_copy_config) 204 _hwcloud_download(args, copy_config, args.bin_dir, args.code_dir) 205 _file_handle(file_handle_config, args.code_dir) 206 207if __name__ == '__main__': 208 sys.exit(main()) 209