1#!/usr/bin/env python3 2# coding: utf-8 3 4""" 5Copyright (c) 2024 Huawei Device Co., Ltd. 6Licensed under the Apache License, Version 2.0 (the "License"); 7you may not use this file except in compliance with the License. 8You may obtain a copy of the License at 9 10 http://www.apache.org/licenses/LICENSE-2.0 11 12Unless required by applicable law or agreed to in awriting, software 13distributed under the License is distributed on an "AS IS" BASIS, 14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15See the License for the specific language governing permissions and 16limitations under the License. 17""" 18 19import argparse 20import re 21import subprocess 22import tarfile 23import datetime 24import gzip 25from urllib.parse import urlparse 26 27import httpx 28import json 29import os 30import requests 31import shutil 32import stat 33import sys 34import time 35import tqdm 36import yaml 37import zipfile 38 39 40def is_windows(): 41 return sys.platform == 'win32' or sys.platform == 'cygwin' 42 43 44def is_mac(): 45 return sys.platform == 'darwin' 46 47 48def parse_configs(): 49 config_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../config.yaml') 50 with open(config_file_path, 'r', encoding='utf-8') as config_file: 51 configs = yaml.safe_load(config_file) 52 return configs 53 54 55def parse_file_name(download_url): 56 parsed_url = urlparse(download_url) 57 path = parsed_url.path 58 file_full_name = path.split("/")[-1] 59 file_name = file_full_name.split(".tar.gz")[0] 60 return file_name 61 62 63def convert_to_datetime(date_str): 64 date_format = '%Y%m%d%H%M%S' 65 date_time = datetime.datetime.strptime(date_str, date_format) 66 return date_time 67 68 69def get_download_url(task_name, image_date): 70 image_date = datetime.datetime.now().strftime('%Y%m%d%H%M%S') if image_date is None else \ 71 datetime.datetime.strptime(image_date, '%Y-%m-%d').strftime('%Y%m%d') + '235959' 72 last_hour = (convert_to_datetime(image_date) + 73 datetime.timedelta(hours=-24)).strftime('%Y%m%d%H%M%S') 74 url = 'http://ci.openharmony.cn/api/daily_build/build/tasks' 75 downnload_job = { 76 'pageNum': 1, 77 'pageSize': 1000, 78 'startTime': '', 79 'endTime': '', 80 'projectName': 'openharmony', 81 'branch': 'master', 82 'component': '', 83 'deviceLevel': '', 84 'hardwareBoard': '', 85 'buildStatus': '', 86 'buildFailReason': '', 87 'testResult': '', 88 } 89 downnload_job['startTime'] = str(last_hour) 90 downnload_job['endTime'] = str(image_date) 91 post_result = requests.post(url, json=downnload_job) 92 post_data = json.loads(post_result.text) 93 download_url_suffix = '' 94 for ohos_sdk_list in post_data['data']['dailyBuildVos']: 95 try: 96 if get_remote_download_name(task_name) in ohos_sdk_list['obsPath']: 97 download_url_suffix = ohos_sdk_list['obsPath'] 98 break 99 except BaseException as err: 100 print(err) 101 download_url = 'http://download.ci.openharmony.cn/' + download_url_suffix 102 return download_url 103 104 105def retry_after_download_failed(download_url, temp_file, temp_file_name, max_retries=3): 106 for i in range(max_retries): 107 try: 108 download(download_url, temp_file, temp_file_name) 109 return True 110 except Exception as e: 111 print(f"download failed! retrying... ({i + 1}/{max_retries})") 112 time.sleep(2) 113 return False 114 115 116def download_progress_bar(response, temp, temp_file_name): 117 total_length = int(response.headers.get("content-length")) 118 with tqdm.tqdm(total=total_length, unit="B", unit_scale=True) as pbar: 119 pbar.set_description(temp_file_name) 120 chunk_sum = 0 121 count = 0 122 for chunk in response.iter_bytes(): 123 temp.write(chunk) 124 chunk_sum += len(chunk) 125 percentage = chunk_sum / total_length * 100 126 while str(percentage).startswith(str(count)): 127 count += 1 128 pbar.update(len(chunk)) 129 130 131def download(download_url, temp_file, temp_file_name): 132 with httpx.stream('GET', download_url) as response: 133 flags = os.O_WRONLY | os.O_CREAT 134 mode = stat.S_IWUSR | stat.S_IRUSR 135 with os.fdopen(os.open(temp_file, flags, mode), 'wb') as temp: 136 download_progress_bar(response, temp, temp_file_name) 137 return True 138 139 140def check_gzip_file(file_path): 141 try: 142 with gzip.open(file_path, 'rb') as gzfile: 143 gzfile.read(1) 144 except Exception as e: 145 print(e) 146 return False 147 return True 148 149 150def check_zip_file(file_path): 151 try: 152 if zipfile.is_zipfile(file_path): 153 with zipfile.ZipFile(file_path, 'r') as zip_file: 154 return True 155 else: 156 return False 157 except Exception as e: 158 print(e) 159 return False 160 return True 161 162 163def get_remote_download_name(task_name): 164 if is_windows(): 165 if 'sdk' in task_name: 166 return 'ohos-sdk-full.tar.gz' 167 return 'dayu200.tar.gz' 168 elif is_mac(): 169 if 'sdk' in task_name: 170 return 'L2-MAC-SDK-FULL.tar.gz' 171 return 'dayu200.tar.gz' 172 else: 173 print('Unsuport platform to get sdk from daily build') 174 return '' 175 176 177def get_api_version(json_path): 178 with open(json_path, 'r') as uni: 179 uni_cont = uni.read() 180 uni_data = json.loads(uni_cont) 181 api_version = uni_data['apiVersion'] 182 return api_version 183 184 185def copy_to_output_path(file_name, file_path, output_path_list): 186 update_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'update.py') 187 cmd = ['python', update_file_path] 188 if 'sdk' in file_name.lower(): 189 sdk_temp_file = os.path.join(file_path, 'sdk_temp') 190 cmd.extend(['--sdkFilePath', sdk_temp_file]) 191 cmd.extend(['--sdkOutputPath']) 192 for output_path in output_path_list: 193 cmd.extend([output_path]) 194 195 if 'dayu' in file_name: 196 dayu_temp_file = os.path.join(file_path, 'dayu200_xts') 197 cmd.extend(['--dayuFilePath', dayu_temp_file]) 198 cmd.extend(['--dayuOutputPath']) 199 for output_path in output_path_list: 200 cmd.extend([output_path]) 201 print(cmd) 202 subprocess.run(cmd, shell=False) 203 204 205def get_the_unzip_file(download_url, save_path): 206 download_name = get_remote_download_name(download_url) 207 download_temp_file = os.path.join(save_path, download_name) 208 if not os.path.exists(save_path): 209 os.mkdir(save_path) 210 print(f'download {download_name} from {download_url}, please wait!!!') 211 success = retry_after_download_failed(download_url, download_temp_file, download_name) 212 if not success: 213 return False 214 if check_gzip_file(download_temp_file): 215 with tarfile.open(download_temp_file, 'r:gz') as tar: 216 print(f'Unpacking {download_temp_file}') 217 if 'dayu' in download_name: 218 save_path = os.path.join(save_path, 'dayu200_xts') 219 tar.extractall(save_path) 220 print(f'Decompression {download_temp_file} completed') 221 elif check_zip_file(download_temp_file): 222 with zipfile.ZipFile(download_temp_file, 'r') as zip_file: 223 print(f'Unpacking {download_temp_file}') 224 zip_file.extractall(save_path) 225 print(f'Decompression {download_temp_file} completed') 226 else: 227 print('The downloaded file is not a valid gzip or zip file.') 228 229 if 'sdk' in download_name: 230 unpack_sdk_file(save_path) 231 232 return True 233 234 235def unpack_sdk_file(path): 236 sdk_zip_path_list = [path, 'ohos-sdk', 'windows'] 237 if is_mac(): 238 sdk_zip_path_list = [path, 'sdk', 239 'packages', 'ohos-sdk', 'darwin'] 240 sdk_floder = os.path.join(path, 'sdk_temp') 241 sdk_zip_path = os.path.join(*sdk_zip_path_list) 242 for item in os.listdir(sdk_zip_path): 243 if item != '.DS_Store': 244 print(f'Unpacking {item}') 245 with zipfile.ZipFile(os.path.join(sdk_zip_path, item)) as zip_file: 246 zip_file.extractall(os.path.join(sdk_floder)) 247 print(f'Decompression {item} completed') 248 249 250def get_task_config(file_name): 251 configs = parse_configs() 252 download_list = configs['download_list'] 253 for item in download_list: 254 if item['name'] in file_name: 255 url = item['url'] 256 date = item['date'] 257 output_path_list = item['output_path_list'] 258 return url, date, output_path_list 259 260 261def delete_redundant_files(task_name, file_name, download_save_path, is_save): 262 if is_save: 263 date_time = re.search(r"\d{8}", file_name).group() 264 new_file_path = os.path.join(download_save_path, f'{date_time}-{task_name}') 265 if not os.path.exists(new_file_path): 266 temp_file_name = 'sdk_temp' if 'sdk' in task_name else 'dayu200_xts' 267 temp_file_path = os.path.join(download_save_path, temp_file_name) 268 os.rename(temp_file_path, new_file_path) 269 subdirs_and_files = [subdir_or_file for subdir_or_file in os.listdir(download_save_path)] 270 for subdir_or_file in subdirs_and_files: 271 if not subdir_or_file[0].isdigit(): 272 path = os.path.join(download_save_path, subdir_or_file) 273 if os.path.isdir(path): 274 shutil.rmtree(path) 275 elif os.path.isfile(path): 276 os.remove(path) 277 278 279def write_download_url_to_txt(task_name, download_url): 280 download_url_txt = os.path.join(os.path.dirname(os.path.abspath(__file__)), 281 'download_url.txt') 282 flags = os.O_WRONLY | os.O_CREAT 283 mode = stat.S_IWUSR | stat.S_IRUSR 284 with os.fdopen(os.open(download_url_txt, flags, mode), 'a+', encoding='utf-8') as file: 285 file.write(f'{task_name}, {download_url}\n') 286 287 288def get_the_image(task_name, download_url, image_date, output_path_list): 289 configs = parse_configs() 290 is_save = configs.get('is_save') 291 download_save_path = configs.get('download_save_path') 292 if download_url == '': 293 download_url = get_download_url(task_name, image_date) 294 if download_url == 'http://download.ci.openharmony.cn/': 295 print('get download url failed') 296 return False 297 file_name = parse_file_name(download_url) 298 if output_path_list is None: 299 output_path_list = get_task_config(file_name)[2] 300 print(f'output_path_list: {output_path_list}') 301 success = get_the_unzip_file(download_url, download_save_path) 302 if not success: 303 print(f'get {task_name} unzip file failed') 304 return False 305 306 copy_to_output_path(file_name, download_save_path, output_path_list) 307 delete_redundant_files(task_name, file_name, download_save_path, is_save) 308 write_download_url_to_txt(task_name, download_url) 309 310 return True 311 312 313def clean_download_log(): 314 download_url_txt = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'download_url.txt') 315 if os.path.exists(download_url_txt): 316 os.remove(download_url_txt) 317 318 319def parse_args(): 320 parser = argparse.ArgumentParser() 321 parser.add_argument('--sdkUrl', type=str, dest='sdk_url', nargs='?', const='', default=None, 322 help='specify which sdk you want to download') 323 parser.add_argument('--dayuUrl', type=str, dest='dayu_url', nargs='?', const='', default=None, 324 help='specify which dayu200 you want to download') 325 parser.add_argument('--sdkDate', type=str, dest='sdk_date', default=None, 326 help='specify which day you want to download the sdk') 327 parser.add_argument('--dayuDate', type=str, dest='dayu_date', default=None, 328 help='specify which day you want to download the dayu') 329 parser.add_argument('--sdkPath', type=str, dest='sdk_path', default=None, 330 nargs='+', 331 help='specify where you want to store the sdk') 332 parser.add_argument('--dayuPath', type=str, dest='dayu_path', default=None, 333 nargs='+', 334 help='specify where you want to store the dayu200') 335 336 return parser.parse_args() 337 338 339def main(): 340 clean_download_log() 341 arguments = parse_args() 342 if arguments.sdk_url is not None: 343 get_the_image('sdk', arguments.sdk_url, arguments.sdk_date, arguments.sdk_path) 344 else: 345 sdk_config = get_task_config('sdk') 346 get_the_image('sdk', sdk_config[0], sdk_config[1], sdk_config[2]) 347 if arguments.dayu_url is not None: 348 get_the_image('dayu', arguments.dayu_url, arguments.dayu_date, arguments.dayu_path) 349 else: 350 dayu_config = get_task_config('sdk') 351 get_the_image('dayu', dayu_config[0], dayu_config[1], dayu_config[2]) 352 353 354if __name__ == '__main__': 355 main()