1# encoding=utf-8 2 3""" 4====================================================================================== 5copyright (C) 2018-2019, Huawei Technologies Co. 6======================================================================================== 7@文件名称: Download.py 8@描述: 文件下载相关动作 9@作者: lwx587017 10@生成日期: 20181019 11====================================================================================== 12""" 13import codecs 14import os 15import shutil 16import urllib 17import time 18import sys 19import re 20import datetime 21import subprocess 22import platform 23import traceback 24import requests 25import json 26import threading 27import zipfile 28 29# from util.config import * 30from util.log_info import logger 31from util.time_info import get_now_time_str_info, timeout 32 33current_path = os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))) 34BitCometjar = os.path.join(current_path, "Resource", "thirdtools", "TTorrentFull.jar") 35from aw.Common.Constant import CONSTANT 36 37from aw.Common.Common import subProcessPopen, copyFile 38 39Aria2cbin = CONSTANT.SysTool.ARIA2CBIN 40 41 42def downloadByBitComet(source_path, download_dir, os_method): 43 ''' 44 #=================================================================================== 45 # @Method: downloadByBitComet(source_path, dest_path) 46 # @Precondition: none 47 # @Func: 通过bt下载，将资源文件下载到本地 48 # @PostStatus: none 49 # @Param: source_path：资源文件路径 50 # download_dir:文件下载到本地的文件夹路径 51 # os_method:绑定操作系统用到公共方法的对象 52 # @eg: downloadByBitComet（"xxxx", "D:\\local\\image", os_method） 53 # @return: True or Flase 54 #=================================================================================== 55 ''' 56 server_ip = "100.104.224.115" 57 server_port = "2976" 58 source_path = source_path.replace(' ', '%20') 59 torrent_exist_url = "http://" + server_ip + ":" + server_port + "/centerDistributeService/BtCenter/addTaskDistribute?path=" \ 60 + source_path 61 logger.info(torrent_exist_url) 62 version_path, destdir = os.path.split(download_dir) 63 64 exist_log = os.path.join(version_path, destdir + "_exist.log") 65 ret = False 66 for _ in range(600): 67 urllib.request.urlretrieve(torrent_exist_url, exist_log) 68 if os.path.isfile(exist_log): 69 open_file = codecs.open(exist_log, "r", "utf-8") 70 content = open_file.readline() 71 if str(content).find("\"status\":\"done\"") != -1: 72 ret = True 73 logger.info("The torrent of %s created seccessfully" % source_path) 74 break 75 else: 76 time.sleep(6) 77 logger.info("The torrent of %s is creating , please wait for a moment" % source_path) 78 continue 79 logger.info(ret) 80 if not ret: 81 logger.info("The torrent of %s created failed" % source_path) 82 return False 83 torrent_down_url = "http://" + server_ip + ":" + server_port + "/centerDistributeService/BtCenter/seed?path=" \ 84 + source_path 85 download_torrent = os.path.join(version_path, destdir + ".torrent") 86 for _ in range(3): 87 urllib.request.urlretrieve(torrent_down_url, download_torrent) 88 if os.path.isfile(download_torrent): 89 ret = False 90 logger.info("The torrent of %s downloaded successed" % source_path) 91 break 92 else: 93 time.sleep(3) 94 continue 95 if ret: 96 logger.info("The torrent of %s downloaded failed" % source_path) 97 return False 98 99 if os.path.isfile(Aria2cbin): 100 BT_Command = "aria2c -l - -d %s --enable-dht=false --seed-time=0 --log-level=warn \ 101 --max-download-limit=50M --bt-stop-timeout=120 %s" % (version_path, download_torrent) 102 else: 103 BT_Command = "java -cp %s com.turn.ttorrent.demo.Downloader %s %s" % ( 104 BitCometjar, download_torrent, version_path) 105 result = os.system(BT_Command) 106 if result: 107 logger.error("BT-Downloader returns: %s, download failed." % result) 108 return False 109 for ld in os.listdir(version_path): 110 if len(ld) == 32 and os.path.isdir(os.path.join(version_path, ld)): 111 copyDirectory(os.path.join(version_path, ld), download_dir, os_method) 112 shutil.rmtree(os.path.join(version_path, ld), True) 113 return True 114 115 116def waitotherdown(suc_mark): 117 ''' 118 #=================================================================================== 119 # @Method: waitotherdown(suc_mark) 120 # @Precondition: none 121 # @Func: 如果版本相同且此版本没下载成功过，会等待正在下载的任务去下载， 122 # 等待时长2小时，若超过2小时正在下载的任务还未下载成功，则自行下载 123 # @PostStatus: none 124 # @Param: suc_mark：成功标志文件的路径 125 # @eg: waitotherdown("xxx\\suc.txt") 126 # @return: True or Flase 127 #=================================================================================== 128 ''' 129 logger.info("wait other process download") 130 for i in range(240): 131 checktime = 5 132 time.sleep(checktime) 133 if os.path.isfile(suc_mark): 134 logger.info("%s exists. Package has been download successfully" \ 135 " by other process." % suc_mark) 136 break 137 if i == 80: 138 logger.error("wait other process download timeout") 139 return False 140 return True 141 142 143def isDownLoadSuccess(download_dir, suc_mark, failed_mark): 144 ''' 145 #=================================================================================== 146 # @Method: isDownLoadSuccess(download_dir, suc_mark, failed_mark) 147 # @Precondition: none 148 # @Func: 下载文件前，判断此文件是否已下载成功或失败，有下载失败标志，则重新下载 149 # ,若此文件未下载过，则创建文件夹重新下载 150 # @PostStatus: none 151 # @Param: download_dir:文件下载到本地的文件夹路径路径 152 # suc_mark：成功标志文件的路径 153 # @eg: isDownLoadSuccess("xxx", "xxx\\suc.txt", "xxx\\fail.txt") 154 # @return: True or Flase 155 #=================================================================================== 156 ''' 157 if os.path.isfile(suc_mark): 158 logger.info("%s has been download by other process." % download_dir) 159 return True 160 elif os.path.isfile(failed_mark): 161 logger.info("%s exists. Other process has download package and failed. It will download by self" % failed_mark) 162 os.remove(failed_mark) 163 else: 164 try: 165 # 如果当前版本的目录存在，则认为版本已经下载 166 if not os.path.isdir(download_dir): 167 logger.debug("mkdir %s" % download_dir) 168 os.makedirs(download_dir) 169 else: 170 logger.debug("%s is already downloading by other process,please wait for a moment!" % download_dir) 171 ret = waitotherdown(suc_mark) 172 return ret 173 except Exception as e: 174 logger.error("make dir fail with %s" % e) 175 shutil.rmtree(download_dir) 176 177 return False 178 179 180def saveVersion(save_path_file, versionsavepath): 181 ''' 182 #=================================================================================== 183 # @Method: saveVersion(save_path_file, versionsavepath) 184 # @Precondition: none 185 # @Func: 此方法用于文件下载，如果文件下载成功，就缓存版本记录到record/xxx.txt中， 186 # 格式：/data/MobileUpgrade/local_img/1663de89-4a0b-5b26-bfad-5d2fd7bf0740 187 # @PostStatus: none 188 # @Param: save_path_file：文件名称，例5c0d84a3a51fff875f8ed4de16791b1c7d3d2cfe.txt 189 # versionsavepath:本地存储路径 190 # @eg: saveVersion(save_path_file， versionsavepath) 191 # @return: None 192 #=================================================================================== 193 ''' 194 savepath = os.path.dirname(save_path_file) 195 if not os.path.isdir(savepath): 196 logger.debug("mkdir %s" % savepath) 197 os.makedirs(savepath) 198 if os.path.isfile(save_path_file): 199 logger.info("%s exists." % save_path_file) 200 else: 201 logger.info("save_path_file: %s" % save_path_file) 202 with open(save_path_file, "a") as f: 203 f.write(versionsavepath) 204 205 206def downlaodByDownloadTool(dir_path, version_type, upgrade_type, pbiid): 207 ''' 208 #=================================================================================== 209 # @Method: downlaodByDownloadTool(dir_path, version_type, upgrade_type, pbiid) 210 # @Precondition: none 211 # @Func: 用下载工具提供的jar包进行下载，需要传入版本的pbiid,升级类型,版本类型， 212 # 具体使用方法可咨询wwx515469 213 # @PostStatus: none 214 # @Param: dir_path：本地储存版本的路径 215 # version_type:版本的类型，如odex,dex,no_log 216 # upgrade_type:升级的类型，如FASTBOOT,SD 217 # pbiid:版本的pbiid号 218 # os_method:绑定操作系统用到公共方法的对象 219 # @eg: downlaodByDownloadTool(dir_path, version_type, upgrade_type, pbiid, os_method) 220 # @return: None 221 #=================================================================================== 222 ''' 223 upgrade_type = upgrade_type.upper() 224 jar_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))), "resource", 225 "thirdtools", "downloadTool.jar") 226 os_name = platform.system() 227 logger.debug("copy res to dir_path") 228 res_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))), "resource", 229 "thirdtools", "res") 230 local_res_path = os.path.join(dir_path, "res") 231 if not os.path.isdir(local_res_path): 232 os.makedirs(local_res_path) 233 if os_method.copyDirectory(res_path, local_res_path): 234 logger.info("copy res to the local image path sucess") 235 else: 236 logger.info("copy res to the local image path fail") 237 return False 238 if (os_name == "Linux"): 239 jar_cmd = "cd %s & java -jar %s %s %s:%s" % (dir_path, jar_path, pbiid, upgrade_type, version_type) 240 else: 241 jar_cmd = "D: & cd %s & java -jar %s %s %s:%s" % (dir_path, jar_path, pbiid, upgrade_type, version_type) 242 logger.info("download by download.jar,cmd is %s" % jar_cmd) 243 ret = subprocess.getoutput(jar_cmd) 244 logger.info("the result is %s after excute downloadTool.jar" % ret) 245 if "download success" in ret: 246 list_jardir = os.listdir(dir_path) 247 for dir in list_jardir: 248 jardir = os.path.join(dir_path, dir) 249 if os.path.isdir(jardir) and jardir != "res": 250 os.rename(jardir, os.path.join(dir_path, "jarimg")) 251 break 252 logger.info("download success by downloadjar") 253 return True 254 else: 255 if "failreason" in ret: 256 com = re.search("failreason\((.*)\)", ret) 257 failreason = com.group(1) 258 logger.info("download fail by downloadjar,failreson is %s" % failreason) 259 else: 260 logger.info("download fail by downloadjar,and jartool not support failreason") 261 return False 262 263 264def downloadLinkFileFromHttp(link, path): 265 ''' 266 #=================================================================================== 267 # @Method: downloadLinkFileFromHttp(link, path) 268 # @Precondition: none 269 # @Func: 通过http下载资源文件到本地 270 # @PostStatus: none 271 # @Param: link：资源文件路径 272 # path:本地文件路径 273 # @eg: downloadLinkFileFromHttp("http:xxx", "D:\local_img") 274 # @return: True or False 275 #=================================================================================== 276 ''' 277 file_name = link.split('/')[-1] 278 destination_file = os.path.join(path, file_name) 279 logger.info('file_name is %s' % file_name) 280 logger.info('destination_file is %s' % destination_file) 281 if not os.path.isdir(path): 282 os.makedirs(path) 283 if not os.path.isfile(destination_file): 284 downloadFileFromHttp(link, destination_file) 285 286 if not os.path.isfile(destination_file): 287 logger.error('download link file error') 288 return False 289 else: 290 return True 291 292 293def downloadFileFromHttp(file_link, destination_file): 294 ''' 295 #=================================================================================== 296 # @Method: downloadFileFromHttp(file_link, destination_file) 297 # @Precondition: none 298 # @Func: 通过http执行下载动作 299 # @PostStatus: none 300 # @Param: file_link：资源文件路径 301 # destination_file:本地文件路径 302 # @eg: destination_file("http:xxx", "D:\local_img") 303 # @return: None 304 #=================================================================================== 305 ''' 306 import urllib 307 logger.info("download file from %s to %s" % (file_link, destination_file)) 308 urllib.request.urlretrieve(file_link, destination_file, scheduleDownloadPercent) 309 310 311def scheduleDownloadPercent(downloadsize, blocksize, filesize): 312 ''' 313 #=================================================================================== 314 # @Method: scheduleDownloadPercent(downloadsize, blocksize, filesize) 315 # @Precondition: none 316 # @Func: 打印当前下载的进度，用百分比形式显示 317 # @PostStatus: none 318 # @Param: downloadsize：已下载文件的大小 319 # blocksize:下载块数 320 # filesize:总文件大小 321 # @eg: scheduleDownloadPercent(xxx,2,xxx ) 322 # @return: None 323 #=================================================================================== 324 ''' 325 percent = 100.0 * downloadsize * blocksize / filesize 326 if percent >= 100: 327 percent = 100 328 downloadpercent = 100 329 percent = float("%.4f" % percent) 330 if percent % 1 == 0: 331 logger.info("file download percent:%.0f%%" % percent) 332 333 334def downloadByCopyWindows(source_path, download_dir, is_file): 335 ''' 336 #=================================================================================== 337 # @Method: excutedown(source_path, download_dir, is_file, params) 338 # @Precondition: none 339 # @Func: 执行下载动作 340 # @PostStatus: none 341 # @Param: source_path：资源文件路径 342 # download_dir:文件下载到本地的文件夹路径 343 # is_file:是否是文件 344 # @eg: excutedown("xxxx", "D:\\local\\image", Flase) 345 # @return: True or Flase 346 #=================================================================================== 347 ''' 348 try: 349 if not is_file: 350 for f in os.listdir(source_path): 351 file_name = os.path.join(source_path, f) 352 child_path = os.path.join(download_dir, f) 353 if os.path.isfile(file_name): 354 copyFile(file_name, child_path, 32 * 1024) 355 if not os.path.isfile(child_path): 356 logger.error("Copy %s to %s failed" % (file_name, child_path)) 357 return False 358 logger.info("Copy %s to %s successful" % (file_name, child_path)) 359 else: 360 if not os.path.isdir(child_path): 361 logger.info("Mkdir %s successful" % child_path) 362 os.makedirs(child_path) 363 ret = downloadByCopyWindows(file_name, child_path, False) 364 if not ret: 365 return False 366 else: 367 copyFile(source_path, download_dir) 368 return True 369 except Exception as e: 370 logger.error(e) 371 return False 372 return True 373 374 375def downloadByCopyLinux(source_path, download_dir, is_file): 376 ''' 377 #=================================================================================== 378 # @Method: excutedown(source_path, download_dir, is_file, params) 379 # @Precondition: none 380 # @Func: 执行下载动作 381 # @PostStatus: none 382 # @Param: source_path：资源文件路径 383 # download_dir:文件下载到本地的文件夹路径 384 # is_file:是否是文件 385 # @eg: excutedown("xxxx", "D:\\local\\image", Flase) 386 # @return: True or Flase 387 #=================================================================================== 388 ''' 389 try: 390 region = "" 391 source_path = source_path.replace("\\", "/") 392 temp = source_path.split("/") 393 host, share, path, file = '', '', '', '' 394 if is_file: 395 file = temp.pop() 396 else: 397 file = '*' 398 for i in temp: 399 if i: 400 if not host: 401 host = i 402 elif not share: 403 share = i 404 else: 405 path = os.path.join(path, i) 406 else: 407 continue 408 cmd = ("smbclient //%s/%s" % (host, share) + " -U CHINA/" 409 + CONSTANT.OSType.ACCOUNT 410 + "%'" 411 + CONSTANT.OSType.PASSWORD 412 + "' -c 'cd \"%s\"'" % path) 413 result = subProcessPopen(cmd) 414 if not result["ret"]: 415 logger.error("invalid samba path.") 416 return False 417 cmd = ("smbclient //%s/%s" % (host, share) + " -U CHINA/" 418 + CONSTANT.OSType.ACCOUNT 419 + "%'" 420 + CONSTANT.OSType.PASSWORD) 421 if region == "green": 422 cmd = cmd + "' -m SMB2 -c 'cd \"%s\";prompt OFF;recurse ON;lcd %s;mget %s'" % (path, download_dir, file) 423 else: 424 cmd = cmd + "' -c 'cd \"%s\";prompt OFF;recurse ON;lcd %s;mget %s'" % (path, download_dir, file) 425 logger.debug(cmd) 426 result = subProcessPopen(cmd) 427 if ("connect failed" in str(result["out"]) or "NT_STATUS_IO_TIMEOUT" in str(result["out"])) or ( 428 not result["ret"]): 429 return False 430 else: 431 return True 432 except Exception: 433 traceback.print_exc() 434 return False 435 436 437def downloadByCopy(source_path, download_dir, is_file): 438 os_name = platform.system() 439 if os_name == "Windows": 440 return downloadByCopyWindows(source_path, download_dir, is_file) 441 if os_name == "Linux": 442 return downloadByCopyLinux(source_path, download_dir, is_file) 443 444 445@timeout(240) 446def getToken(domin_name, name, password): 447 ''' 448 #=================================================================================== 449 # @Method: getToken(domin_name, name, passwd, proxy) 450 # @Precondition: none 451 # @Func: 通过http下载蓝区文件 452 # @PostStatus: none 453 # @Param: domin_name: 帐号名 454 # username：用户名 455 # password：密码 456 # 457 # @eg: getToken("xxx", "xxx", "xxx") 458 # @return: None 459 #=================================================================================== 460 ''' 461 body_1 = { 462 "auth": { 463 "identity": { 464 "methods": [ 465 "password" 466 ], 467 "password": { 468 "user": { 469 "domain": { 470 "name": domin_name 471 }, 472 "name": name, 473 "password": password 474 } 475 } 476 }, 477 "scope": { 478 "project": { 479 "name": "cn-north-4" 480 } 481 } 482 } 483 } 484 header_1 = {"content-type": "application/json;charset=UTF-8"} 485 resp = requests.request("POST", "https://iam.myhuaweicloud.com/v3/auth/tokens", data=json.dumps(body_1), 486 headers=header_1) 487 # print("auth result:", resp.status_code, resp.headers) 488 return resp.headers.get("X-Subject-Token") 489 490 491def downloadFileFromDevCloud(link, destination_file, start, end): 492 ''' 493 #=================================================================================== 494 # @Method: downloadFileFromDevCloud(link, username, password, path) 495 # @Precondition: none 496 # @Func: 通过http下载蓝区文件 497 # @PostStatus: none 498 # @Param: link：资源文件路径 499 # username：用户名 500 # password：密码 501 # path:本地文件路径 502 # @eg: downloadFileFromDevCloud("http:xxx", "xxx", "xxx","D:\local_img") 503 # @return: None 504 #=================================================================================== 505 ''' 506 fd = open(destination_file, 'wb+') 507 dname = decrypt(CONSTANT.Key.DEV_DNAME, CONSTANT.Key.DEV_KEY) 508 pw = decrypt(CONSTANT.Key.DEV_PASSWD, CONSTANT.Key.DEV_KEY) 509 try: 510 try: 511 token = getToken("hwstaff_harmonyos_ci", dname, pw) 512 logger.info("get token end") 513 except Exception as f: 514 logger.error(f) 515 time.sleep(3) 516 logger.info("get token again") 517 token = getToken("hwstaff_harmonyos_ci", dname, pw) 518 logger.info("get token again end") 519 heard = {'Range': 'Bytes=%s-%s' % (start, end), "content-type": "application/json;charset=UTF-8", 520 "X-Language": "zh-cn", "X-Auth-Token": token} 521 try: 522 res = requests.get(link, headers=heard, stream=True, timeout=240) 523 logger.info("connect status: %s" % res.status_code) 524 except Exception as p: 525 logger.error(p) 526 time.sleep(3) 527 logger.info("connect again!") 528 res = requests.get(link, headers=heard, stream=True, timeout=240) 529 logger.info("connect again status: %s" % res.status_code) 530 fd.seek(start) 531 fd.write(res.content) 532 if res.status_code == 206 or res.status_code == 200: 533 if not os.path.isfile(destination_file): 534 logger.error('download link file and write file error') 535 return False 536 else: 537 logger.error("downloadFromDevCloud fail, error: %s" % res) 538 return False 539 except Exception as e: 540 traceback.print_exc() 541 logger.error(e) 542 543 544def decrypt(text, key): 545 from Crypto.Cipher import AES 546 from binascii import a2b_hex 547 cryptor = AES.new(key.encode('utf-8'), AES.MODE_CBC, b'0000000000000000') 548 plain_text = cryptor.decrypt(a2b_hex(text)) 549 return bytes.decode(plain_text).rstrip('\0') 550 551 552def run_download(link, path): 553 try: 554 num = 3 555 r = requests.head(link) 556 total = int(r.headers['Content-Length']) 557 logger.info('total is %s' % total) 558 ranges = [] 559 offset = int(total / num) 560 for i in range(num): 561 if i == num - 1: 562 ranges.append((i * offset, '')) 563 else: 564 ranges.append((i * offset, (i + 1) * offset)) 565 # 保存文件打开对象 566 if "file_id=" in link: 567 if link.endswith(".tar.gz"): 568 file_name = link.split('=')[-1] 569 else: 570 file_name = "out.tar.gz" 571 else: 572 file_name = link.split('/')[-1] 573 destination_file = os.path.join(path, file_name) 574 logger.info('file_name is %s' % file_name) 575 logger.info('destination_file is %s' % destination_file) 576 if not os.path.isdir(path): 577 os.makedirs(path) 578 if os.path.isfile(destination_file): 579 logger.info('%s is exit' % destination_file) 580 return True 581 # 保存文件打开对象 582 logger.info("downloadFileFromDevCloud start") 583 logger.info(link) 584 thread_list = [] 585 # 一个数字,用来标记打印每个线程 586 n = 0 587 for ran in ranges: 588 start, end = ran 589 # 打印信息 590 logger.info('thread %d start:%s,end:%s' % (n, start, end)) 591 n += 1 592 # 创建线程 传参,处理函数为download 593 thread = threading.Thread(target=downloadFileFromDevCloud, args=(link, destination_file, start, end)) 594 # 启动 595 thread.start() 596 thread_list.append(thread) 597 for i in thread_list: 598 # 设置等待 599 i.join() 600 if os.path.isfile(destination_file): 601 logger.info('%s is exit' % destination_file) 602 return True 603 return False 604 except Exception as p: 605 logger.error(p) 606 return False