1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2023 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16 17import requests 18import json 19import datetime 20import os 21import sys 22import tarfile 23import subprocess 24import argparse 25 26from urllib.request import urlretrieve 27 28 29def find_top(): 30 cur_dir = os.getcwd() 31 while cur_dir != "/": 32 build_scripts = os.path.join( 33 cur_dir, 'build/config/BUILDCONFIG.gn') 34 if os.path.exists(build_scripts): 35 return cur_dir 36 cur_dir = os.path.dirname(cur_dir) 37 38 39def reporthook(data_download, data_size, total_size): 40 ''' 41 display the progress of download 42 :param data_download: data downloaded 43 :param data_size: data size 44 :param total_size: remote file size 45 :return:None 46 ''' 47 progress = int(0) 48 if progress != int(data_download * data_size * 1000 / total_size): 49 progress = int(data_download * data_size * 1000 / total_size) 50 print("\rDownloading: %5.1f%%" % 51 (data_download * data_size * 100.0 / total_size), end="") 52 sys.stdout.flush() 53 54 55def download(download_url, savepath): 56 filename = os.path.basename(download_url) 57 58 if not os.path.isfile(os.path.join(savepath, filename)): 59 print('Downloading data form %s' % download_url) 60 urlretrieve(download_url, os.path.join( 61 savepath, filename), reporthook=reporthook) 62 print('\nDownload finished!') 63 else: 64 print("\nFile exsits!") 65 66 filesize = os.path.getsize(os.path.join(savepath, filename)) 67 print('File size = %.2f Mb' % (filesize/1024/1024)) 68 69 70def extract_file(filename): 71 72 target_dir = os.path.dirname(filename) 73 74 if not os.path.exists(target_dir): 75 os.makedirs(target_dir, exist_ok=True) 76 with tarfile.open(filename, "r:gz") as tar: 77 tar.extractall(target_dir) 78 79 if os.path.exists(os.path.join(target_dir, "daily_build.log")): 80 os.remove(os.path.join(target_dir, "daily_build.log")) 81 if os.path.exists(os.path.join(target_dir, "manifest_tag.xml")): 82 os.remove(os.path.join(target_dir, "manifest_tag.xml")) 83 84 85def npm_install(target_dir): 86 87 sdk_dir = os.path.join(target_dir, "ohos-sdk/linux") 88 os.chdir(sdk_dir) 89 subprocess.run(['ls', '-d', '*/', '|', 'xargs', 'rm', '-rf']) 90 91 for filename in os.listdir(sdk_dir): 92 if filename.endswith('.zip'): 93 subprocess.run(['unzip', filename]) 94 95 p1 = subprocess.Popen( 96 ["grep", "apiVersion", "toolchains/oh-uni-package.json"], stdout=subprocess.PIPE) 97 p2 = subprocess.Popen(["awk", "{print $2}"], 98 stdin=p1.stdout, stdout=subprocess.PIPE) 99 p3 = subprocess.Popen(["sed", "-r", "s/\",?//g"], 100 stdin=p2.stdout, stdout=subprocess.PIPE) 101 output = p3.communicate(timeout=5)[0] 102 api_version = output.decode("utf-8").strip() 103 104 p4 = subprocess.Popen( 105 ["grep", "version", "toolchains/oh-uni-package.json"], stdout=subprocess.PIPE) 106 p5 = subprocess.Popen(["awk", "{print $2}"], 107 stdin=p4.stdout, stdout=subprocess.PIPE) 108 p6 = subprocess.Popen(["sed", "-r", "s/\",?//g"], 109 stdin=p5.stdout, stdout=subprocess.PIPE) 110 output = p6.communicate(timeout=5)[0] 111 sdk_version = output.decode("utf-8").strip() 112 113 for dirname in os.listdir("."): 114 if os.path.isdir(dirname): 115 subprocess.run(['mkdir', '-p', api_version]) 116 subprocess.run(['mv', dirname, api_version]) 117 118 119def main(): 120 parser = argparse.ArgumentParser() 121 parser.add_argument('--branch', default='master', help='OHOS branch name') 122 parser.add_argument('--product-name', default='ohos-sdk-full', help='OHOS product name') 123 args = parser.parse_args() 124 default_save_path = os.path.join(find_top(), 'prebuilts') 125 if not os.path.exists(default_save_path): 126 os.makedirs(default_save_path, exist_ok=True) 127 print(default_save_path) 128 try: 129 now_time = datetime.datetime.now().strftime('%Y%m%d%H%M%S') 130 last_hour = (datetime.datetime.now() + 131 datetime.timedelta(hours=-72)).strftime('%Y%m%d%H%M%S') 132 133 url = "http://ci.openharmony.cn/api/daily_build/build/tasks" 134 myobj = {"pageNum": 1, 135 "pageSize": 1000, 136 "startTime": "", 137 "endTime": "", 138 "projectName": "openharmony", 139 "branch": args.branch, 140 "component": "", 141 "deviceLevel": "", 142 "hardwareBoard": "", 143 "buildStatus": "success", 144 "buildFailReason": "", 145 "testResult": ""} 146 myobj["startTime"] = str(last_hour) 147 myobj["endTime"] = str(now_time) 148 x = requests.post(url, json=myobj) 149 data = json.loads(x.text) 150 except BaseException: 151 Exception("Unable to establish connection with ci.openharmony.cn") 152 153 products_list = data['data']['dailyBuildVos'] 154 for product in products_list: 155 product_name = product['component'] 156 if product_name == args.product_name: 157 if os.path.exists(os.path.join(default_save_path, product_name)): 158 print('{} already exists. Please backup or delete it first!'.format( 159 os.path.join(default_save_path, product_name))) 160 print("Download canceled!") 161 break 162 163 if product['obsPath'] and os.path.exists(default_save_path): 164 download_url = 'http://download.ci.openharmony.cn/{}'.format(product['obsPath']) 165 save_path2 = default_save_path 166 167 try: 168 download(download_url, savepath=save_path2) 169 print(download_url, "done") 170 except BaseException: 171 172 # remove the incomplete downloaded files 173 if os.path.exists(os.path.join(save_path2, os.path.basename(download_url))): 174 os.remove(os.path.join( 175 save_path2, os.path.basename(download_url))) 176 Exception("Unable to download {}".format(download_url)) 177 178 extract_file(os.path.join( 179 save_path2, os.path.basename(download_url))) 180 npm_install(save_path2) 181 break 182 183 184if __name__ == '__main__': 185 sys.exit(main()) 186