1# encoding=utf-8 2 3""" 4====================================================================================== 5copyright (C) 2018-2019, Huawei Technologies Co. 6======================================================================================== 7@文件名称: Download.py 8@描述: 文件下载相关动作 9@作者: lwx587017 10@生成日期: 20181019 11====================================================================================== 12""" 13import codecs 14import os 15import shutil 16import urllib 17import time 18import sys 19import re 20import datetime 21import subprocess 22import platform 23import traceback 24import requests 25import json 26import threading 27import zipfile 28 29# from util.config import * 30from util.log_info import logger 31from util.time_info import get_now_time_str_info, timeout 32 33current_path = os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))) 34BitCometjar = os.path.join(current_path, "Resource", "thirdtools", "TTorrentFull.jar") 35from aw.Common.Constant import CONSTANT 36 37from aw.Common.Common import subProcessPopen, copyFile 38 39 40Aria2cbin = CONSTANT.SysTool.ARIA2CBIN 41 42def downloadByBitComet(source_path, download_dir, os_method): 43 ''' 44 #=================================================================================== 45 # @Method: downloadByBitComet(source_path, dest_path) 46 # @Precondition: none 47 # @Func: 通过bt下载，将资源文件下载到本地 48 # @PostStatus: none 49 # @Param: source_path：资源文件路径 50 # download_dir:文件下载到本地的文件夹路径 51 # os_method:绑定操作系统用到公共方法的对象 52 # @eg: downloadByBitComet（"xxxx", "D:\\local\\image", os_method） 53 # @return: True or Flase 54 #=================================================================================== 55 ''' 56 server_ip = "100.104.224.115" 57 server_port = "2976" 58 source_path = source_path.replace(' ', '%20') 59 torrent_exist_url = "http://" + server_ip + ":" + server_port + "/centerDistributeService/BtCenter/addTaskDistribute?path=" \ 60 + source_path 61 logger.info(torrent_exist_url) 62 version_path, destdir = os.path.split(download_dir) 63 64 exist_log = os.path.join(version_path, destdir + "_exist.log") 65 ret = False 66 for _ in range(600): 67 urllib.request.urlretrieve(torrent_exist_url, exist_log) 68 if os.path.isfile(exist_log): 69 open_file = codecs.open(exist_log, "r", "utf-8") 70 content = open_file.readline() 71 if str(content).find("\"status\":\"done\"") != -1: 72 ret = True 73 logger.info("The torrent of %s created seccessfully" % source_path) 74 break 75 else: 76 time.sleep(6) 77 logger.info("The torrent of %s is creating , please wait for a moment" % source_path) 78 continue 79 logger.info(ret) 80 if not ret: 81 logger.info("The torrent of %s created failed" % source_path) 82 return False 83 torrent_down_url = "http://" + server_ip + ":" + server_port + "/centerDistributeService/BtCenter/seed?path=" \ 84 + source_path 85 download_torrent = os.path.join(version_path, destdir + ".torrent") 86 for _ in range(3): 87 urllib.request.urlretrieve(torrent_down_url, download_torrent) 88 if os.path.isfile(download_torrent): 89 ret = False 90 logger.info("The torrent of %s downloaded successed" % source_path) 91 break 92 else: 93 time.sleep(3) 94 continue 95 if ret: 96 logger.info("The torrent of %s downloaded failed" % source_path) 97 return False 98 99 if os.path.isfile(Aria2cbin): 100 BT_Command = "aria2c -l - -d %s --enable-dht=false --seed-time=0 --log-level=warn \ 101 --max-download-limit=50M --bt-stop-timeout=120 %s" % (version_path, download_torrent) 102 else: 103 BT_Command = "java -cp %s com.turn.ttorrent.demo.Downloader %s %s" % ( 104 BitCometjar, download_torrent, version_path) 105 result = os.system(BT_Command) 106 if result: 107 logger.error("BT-Downloader returns: %s, download failed." % result) 108 return False 109 for ld in os.listdir(version_path): 110 if len(ld) == 32 and os.path.isdir(os.path.join(version_path, ld)): 111 copyDirectory(os.path.join(version_path, ld), download_dir, os_method) 112 shutil.rmtree(os.path.join(version_path, ld), True) 113 return True 114 115def waitotherdown(suc_mark): 116 ''' 117 #=================================================================================== 118 # @Method: waitotherdown(suc_mark) 119 # @Precondition: none 120 # @Func: 如果版本相同且此版本没下载成功过，会等待正在下载的任务去下载， 121 # 等待时长2小时，若超过2小时正在下载的任务还未下载成功，则自行下载 122 # @PostStatus: none 123 # @Param: suc_mark：成功标志文件的路径 124 # @eg: waitotherdown("xxx\\suc.txt") 125 # @return: True or Flase 126 #=================================================================================== 127 ''' 128 logger.info("wait other process download") 129 for i in range(240): 130 checktime = 5 131 time.sleep(checktime) 132 if os.path.isfile(suc_mark): 133 logger.info("%s exists. Package has been download successfully" \ 134 " by other process." % suc_mark) 135 break 136 if i == 239: 137 logger.error("wait other process download timeout") 138 return False 139 return True 140 141def isDownLoadSuccess(download_dir, suc_mark, failed_mark): 142 ''' 143 #=================================================================================== 144 # @Method: isDownLoadSuccess(download_dir, suc_mark, failed_mark) 145 # @Precondition: none 146 # @Func: 下载文件前，判断此文件是否已下载成功或失败，有下载失败标志，则重新下载 147 # ,若此文件未下载过，则创建文件夹重新下载 148 # @PostStatus: none 149 # @Param: download_dir:文件下载到本地的文件夹路径路径 150 # suc_mark：成功标志文件的路径 151 # @eg: isDownLoadSuccess("xxx", "xxx\\suc.txt", "xxx\\fail.txt") 152 # @return: True or Flase 153 #=================================================================================== 154 ''' 155 if os.path.isfile(suc_mark): 156 logger.info("%s has been download by other process." % download_dir) 157 return True 158 elif os.path.isfile(failed_mark): 159 logger.info("%s exists. Other process has download package and failed. It will download by self" % failed_mark) 160 os.remove(failed_mark) 161 else: 162 try: 163 #如果当前版本的目录存在，则认为版本已经下载 164 if not os.path.isdir(download_dir): 165 logger.debug("mkdir %s" % download_dir) 166 os.makedirs(download_dir) 167 else: 168 logger.debug("%s is already downloading by other process,please wait for a moment!" % download_dir) 169 ret = waitotherdown(suc_mark) 170 return ret 171 except Exception as e: 172 logger.error("make dir fail with %s" % e) 173 shutil.rmtree(download_dir) 174 175 return False 176 177def saveVersion(save_path_file, versionsavepath): 178 ''' 179 #=================================================================================== 180 # @Method: saveVersion(save_path_file, versionsavepath) 181 # @Precondition: none 182 # @Func: 此方法用于文件下载，如果文件下载成功，就缓存版本记录到record/xxx.txt中， 183 # 格式：/data/MobileUpgrade/local_img/1663de89-4a0b-5b26-bfad-5d2fd7bf0740 184 # @PostStatus: none 185 # @Param: save_path_file：文件名称，例5c0d84a3a51fff875f8ed4de16791b1c7d3d2cfe.txt 186 # versionsavepath:本地存储路径 187 # @eg: saveVersion(save_path_file， versionsavepath) 188 # @return: None 189 #=================================================================================== 190 ''' 191 savepath = os.path.dirname(save_path_file) 192 if not os.path.isdir(savepath): 193 logger.debug("mkdir %s" % savepath) 194 os.makedirs(savepath) 195 if os.path.isfile(save_path_file): 196 logger.info("%s exists." % save_path_file) 197 else: 198 logger.info("save_path_file: %s" % save_path_file) 199 with open(save_path_file, "a") as f: 200 f.write(versionsavepath) 201 202def downlaodByDownloadTool(dir_path, version_type, upgrade_type, pbiid): 203 ''' 204 #=================================================================================== 205 # @Method: downlaodByDownloadTool(dir_path, version_type, upgrade_type, pbiid) 206 # @Precondition: none 207 # @Func: 用下载工具提供的jar包进行下载，需要传入版本的pbiid,升级类型,版本类型， 208 # 具体使用方法可咨询wwx515469 209 # @PostStatus: none 210 # @Param: dir_path：本地储存版本的路径 211 # version_type:版本的类型，如odex,dex,no_log 212 # upgrade_type:升级的类型，如FASTBOOT,SD 213 # pbiid:版本的pbiid号 214 # os_method:绑定操作系统用到公共方法的对象 215 # @eg: downlaodByDownloadTool(dir_path, version_type, upgrade_type, pbiid, os_method) 216 # @return: None 217 #=================================================================================== 218 ''' 219 upgrade_type = upgrade_type.upper() 220 jar_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))), "resource", "thirdtools", "downloadTool.jar") 221 os_name = platform.system() 222 logger.debug("copy res to dir_path") 223 res_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))), "resource", "thirdtools", "res") 224 local_res_path = os.path.join(dir_path, "res") 225 if not os.path.isdir(local_res_path): 226 os.makedirs(local_res_path) 227 if os_method.copyDirectory(res_path, local_res_path): 228 logger.info("copy res to the local image path sucess") 229 else: 230 logger.info("copy res to the local image path fail") 231 return False 232 if (os_name == "Linux"): 233 jar_cmd = "cd %s & java -jar %s %s %s:%s" % (dir_path, jar_path, pbiid, upgrade_type, version_type) 234 else: 235 jar_cmd = "D: & cd %s & java -jar %s %s %s:%s" % (dir_path, jar_path, pbiid, upgrade_type, version_type) 236 logger.info("download by download.jar,cmd is %s" % jar_cmd) 237 ret = subprocess.getoutput(jar_cmd) 238 logger.info("the result is %s after excute downloadTool.jar" % ret) 239 if "download success" in ret: 240 list_jardir = os.listdir(dir_path) 241 for dir in list_jardir: 242 jardir = os.path.join(dir_path, dir) 243 if os.path.isdir(jardir) and jardir != "res": 244 os.rename(jardir, os.path.join(dir_path, "jarimg")) 245 break 246 logger.info("download success by downloadjar") 247 return True 248 else: 249 if "failreason" in ret: 250 com = re.search("failreason\((.*)\)", ret) 251 failreason = com.group(1) 252 logger.info("download fail by downloadjar,failreson is %s" % failreason) 253 else: 254 logger.info("download fail by downloadjar,and jartool not support failreason") 255 return False 256 257 258def downloadLinkFileFromHttp(link, path): 259 ''' 260 #=================================================================================== 261 # @Method: downloadLinkFileFromHttp(link, path) 262 # @Precondition: none 263 # @Func: 通过http下载资源文件到本地 264 # @PostStatus: none 265 # @Param: link：资源文件路径 266 # path:本地文件路径 267 # @eg: downloadLinkFileFromHttp("http:xxx", "D:\local_img") 268 # @return: True or False 269 #=================================================================================== 270 ''' 271 file_name = link.split('/')[-1] 272 destination_file = os.path.join(path, file_name) 273 logger.info('file_name is %s' % file_name) 274 logger.info('destination_file is %s' % destination_file) 275 if not os.path.isdir(path): 276 os.makedirs(path) 277 if not os.path.isfile(destination_file): 278 downloadFileFromHttp(link, destination_file) 279 280 if not os.path.isfile(destination_file): 281 logger.error('download link file error') 282 return False 283 else: 284 return True 285 286 287def downloadFileFromHttp(file_link, destination_file): 288 ''' 289 #=================================================================================== 290 # @Method: downloadFileFromHttp(file_link, destination_file) 291 # @Precondition: none 292 # @Func: 通过http执行下载动作 293 # @PostStatus: none 294 # @Param: file_link：资源文件路径 295 # destination_file:本地文件路径 296 # @eg: destination_file("http:xxx", "D:\local_img") 297 # @return: None 298 #=================================================================================== 299 ''' 300 import urllib 301 logger.info("download file from %s to %s" % (file_link, destination_file)) 302 urllib.request.urlretrieve(file_link, destination_file, scheduleDownloadPercent) 303 304 305def scheduleDownloadPercent(downloadsize, blocksize, filesize): 306 ''' 307 #=================================================================================== 308 # @Method: scheduleDownloadPercent(downloadsize, blocksize, filesize) 309 # @Precondition: none 310 # @Func: 打印当前下载的进度，用百分比形式显示 311 # @PostStatus: none 312 # @Param: downloadsize：已下载文件的大小 313 # blocksize:下载块数 314 # filesize:总文件大小 315 # @eg: scheduleDownloadPercent(xxx,2,xxx ) 316 # @return: None 317 #=================================================================================== 318 ''' 319 percent = 100.0 * downloadsize * blocksize / filesize 320 if percent >= 100 : 321 percent = 100 322 downloadpercent = 100 323 percent = float("%.4f" % percent) 324 if percent % 1 == 0: 325 logger.info("file download percent:%.0f%%" % percent) 326 327 328def downloadByCopyWindows(source_path, download_dir, is_file): 329 ''' 330 #=================================================================================== 331 # @Method: excutedown(source_path, download_dir, is_file, params) 332 # @Precondition: none 333 # @Func: 执行下载动作 334 # @PostStatus: none 335 # @Param: source_path：资源文件路径 336 # download_dir:文件下载到本地的文件夹路径 337 # is_file:是否是文件 338 # @eg: excutedown("xxxx", "D:\\local\\image", Flase) 339 # @return: True or Flase 340 #=================================================================================== 341 ''' 342 try: 343 if not is_file: 344 for f in os.listdir(source_path): 345 file_name = os.path.join(source_path, f) 346 child_path = os.path.join(download_dir, f) 347 if os.path.isfile(file_name): 348 copyFile(file_name, child_path, 32 * 1024) 349 if not os.path.isfile(child_path): 350 logger.error("Copy %s to %s failed" % (file_name, child_path)) 351 return False 352 logger.info("Copy %s to %s successful" % (file_name, child_path)) 353 else: 354 if not os.path.isdir(child_path): 355 logger.info("Mkdir %s successful" % child_path) 356 os.makedirs(child_path) 357 ret = downloadByCopyWindows(file_name, child_path, False) 358 if not ret: 359 return False 360 else: 361 copyFile(source_path, download_dir) 362 return True 363 except Exception as e: 364 logger.error(e) 365 return False 366 return True 367 368def downloadByCopyLinux(source_path, download_dir, is_file): 369 ''' 370 #=================================================================================== 371 # @Method: excutedown(source_path, download_dir, is_file, params) 372 # @Precondition: none 373 # @Func: 执行下载动作 374 # @PostStatus: none 375 # @Param: source_path：资源文件路径 376 # download_dir:文件下载到本地的文件夹路径 377 # is_file:是否是文件 378 # @eg: excutedown("xxxx", "D:\\local\\image", Flase) 379 # @return: True or Flase 380 #=================================================================================== 381 ''' 382 try: 383 region = "" 384 source_path = source_path.replace("\\", "/") 385 temp = source_path.split("/") 386 host, share, path, file = '', '', '', '' 387 if is_file: 388 file = temp.pop() 389 else: 390 file = '*' 391 for i in temp: 392 if i: 393 if not host: 394 host = i 395 elif not share: 396 share = i 397 else: 398 path = os.path.join(path, i) 399 else: 400 continue 401 cmd = ("smbclient //%s/%s" % (host, share) + " -U CHINA/" 402 + CONSTANT.OSType.ACCOUNT 403 + "%'" 404 + CONSTANT.OSType.PASSWORD 405 + "' -c 'cd \"%s\"'" % path) 406 result = subProcessPopen(cmd) 407 if not result["ret"]: 408 logger.error("invalid samba path.") 409 return False 410 cmd = ("smbclient //%s/%s" % (host, share) + " -U CHINA/" 411 + CONSTANT.OSType.ACCOUNT 412 + "%'" 413 + CONSTANT.OSType.PASSWORD) 414 if region == "green": 415 cmd = cmd + "' -m SMB2 -c 'cd \"%s\";prompt OFF;recurse ON;lcd %s;mget %s'" % (path, download_dir, file) 416 else: 417 cmd = cmd + "' -c 'cd \"%s\";prompt OFF;recurse ON;lcd %s;mget %s'" % (path, download_dir, file) 418 logger.debug(cmd) 419 result = subProcessPopen(cmd) 420 if ("connect failed" in str(result["out"]) or "NT_STATUS_IO_TIMEOUT" in str(result["out"])) or ( 421 not result["ret"]): 422 return False 423 else: 424 return True 425 except Exception: 426 traceback.print_exc() 427 return False 428 429def downloadByCopy(source_path, download_dir, is_file): 430 os_name = platform.system() 431 if os_name == "Windows": 432 return downloadByCopyWindows(source_path, download_dir, is_file) 433 if os_name == "Linux": 434 return downloadByCopyLinux(source_path, download_dir, is_file) 435 436def getToken(domin_name, name, password): 437 ''' 438 #=================================================================================== 439 # @Method: getToken(domin_name, name, passwd, proxy) 440 # @Precondition: none 441 # @Func: 通过http下载蓝区文件 442 # @PostStatus: none 443 # @Param: domin_name: 帐号名 444 # username：用户名 445 # password：密码 446 # 447 # @eg: getToken("xxx", "xxx", "xxx") 448 # @return: None 449 #=================================================================================== 450 ''' 451 body_1 = { 452 "auth": { 453 "identity": { 454 "methods": [ 455 "password" 456 ], 457 "password": { 458 "user": { 459 "domain": { 460 "name": domin_name 461 }, 462 "name": name, 463 "password": password 464 } 465 } 466 }, 467 "scope": { 468 "project": { 469 "name": "cn-north-4" 470 } 471 } 472 } 473 } 474 header_1 = {"content-type":"application/json;charset=UTF-8"} 475 resp = requests.request("POST", "https://iam.myhuaweicloud.com/v3/auth/tokens", data=json.dumps(body_1), headers=header_1) 476 #print("auth result:", resp.status_code, resp.headers) 477 return resp.headers.get("X-Subject-Token") 478 479def downloadFileFromDevCloud(link, destination_file, start, end): 480 ''' 481 #=================================================================================== 482 # @Method: downloadFileFromDevCloud(link, username, password, path) 483 # @Precondition: none 484 # @Func: 通过http下载蓝区文件 485 # @PostStatus: none 486 # @Param: link：资源文件路径 487 # username：用户名 488 # password：密码 489 # path:本地文件路径 490 # @eg: downloadFileFromDevCloud("http:xxx", "xxx", "xxx","D:\local_img") 491 # @return: None 492 #=================================================================================== 493 ''' 494 fd = open(destination_file, 'wb+') 495 dname = decrypt(CONSTANT.Key.DEV_DNAME, CONSTANT.Key.DEV_KEY) 496 pw = decrypt(CONSTANT.Key.DEV_PASSWD, CONSTANT.Key.DEV_KEY) 497 token = getToken("hwstaff_harmonyos_ci", dname, pw) 498 logger.info("get token end") 499 heard = {'Range': 'Bytes=%s-%s' % (start, end), "content-type": "application/json;charset=UTF-8", "X-Language": "zh-cn", "X-Auth-Token": token} 500 res = requests.get(link, headers=heard, stream=True, timeout=600) 501 logger.info("connect status: %s" % res.status_code) 502 fd.seek(start) 503 fd.write(res.content) 504 try: 505 if res.status_code == 206 or res.status_code == 200: 506 if not os.path.isfile(destination_file): 507 logger.error('download link file and write file error') 508 return False 509 else: 510 logger.error("downloadFromDevCloud fail, error: %s" % res) 511 return False 512 except Exception as e: 513 traceback.print_exc() 514 logger.error(e) 515 516 517 518def decrypt(text, key): 519 from Crypto.Cipher import AES 520 from binascii import a2b_hex 521 522 cryptor = AES.new(key.encode('utf-8'), AES.MODE_CBC, b'0000000000000000') 523 plain_text = cryptor.decrypt(a2b_hex(text)) 524 return bytes.decode(plain_text).rstrip('\0') 525 526 527def run_download(link, path): 528 num = 6 529 r = requests.head(link) 530 total = int(r.headers['Content-Length']) 531 logger.info('total is %s' % total) 532 ranges = [] 533 offset = int(total / num) 534 for i in range(num): 535 if i == num - 1: 536 ranges.append((i * offset, '')) 537 else: 538 ranges.append((i * offset, (i + 1) * offset)) 539 # 保存文件打开对象 540 if "file_id=" in link: 541 if link.endswith(".tar.gz"): 542 file_name = link.split('=')[-1] 543 else: 544 file_name = "out.tar.gz" 545 else: 546 file_name = link.split('/')[-1] 547 548 destination_file = os.path.join(path, file_name) 549 logger.info('file_name is %s' % file_name) 550 logger.info('destination_file is %s' % destination_file) 551 if not os.path.isdir(path): 552 os.makedirs(path) 553 if os.path.isfile(destination_file): 554 logger.info('%s is exit' % destination_file) 555 return True 556 # 保存文件打开对象 557 logger.info("downloadFileFromDevCloud start") 558 logger.info(link) 559 thread_list = [] 560 # 一个数字,用来标记打印每个线程 561 n = 0 562 for ran in ranges: 563 start, end = ran 564 # 打印信息 565 logger.info('thread %d start:%s,end:%s' % (n, start, end)) 566 n += 1 567 # 创建线程 传参,处理函数为download 568 thread = threading.Thread(target=downloadFileFromDevCloud, args=(link, destination_file, start, end)) 569 # 启动 570 thread.start() 571 thread_list.append(thread) 572 for i in thread_list: 573 # 设置等待 574 i.join() 575 if os.path.isfile(destination_file): 576 logger.info('%s is exit' % destination_file) 577 return True 578 return False 579 580