• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3#
4# Copyright (c) 2024 Huawei Device Co., Ltd.
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import json
19import os
20import subprocess
21import shutil
22import argparse
23import sys
24import urllib.request
25import socket
26
27
28def _run_cmd(cmd):
29    res = subprocess.Popen(cmd, stdout=subprocess.PIPE,
30                           stderr=subprocess.PIPE)
31    sout, serr = res.communicate(timeout=10000)
32    return sout.rstrip().decode('utf-8'), serr, res.returncode
33
34
35def _check_sha256(check_url: str, local_file: str) -> bool:
36    check_sha256_cmd = ('curl -s -k ' + check_url + '.sha256').split(' ')
37    local_sha256_cmd = ['sha256sum', local_file]
38    check_sha256, err, returncode = _run_cmd(check_sha256_cmd)
39    local_sha256, err, returncode = _run_cmd(local_sha256_cmd)
40    local_sha256 = local_sha256.split(' ')[0]
41    if check_sha256 != local_sha256:
42        print('remote file {}.sha256 is not found, begin check SHASUMS256.txt'.format(check_url))
43        check_sha256 = _obtain_sha256_by_sha_sums256(check_url)
44    return check_sha256 == local_sha256
45
46
47def _obtain_sha256_by_sha_sums256(check_url: str) -> str:
48    sha_sums256 = 'SHASUMS256.txt'
49    sha_sums256_path = os.path.join(os.path.dirname(check_url), sha_sums256)
50    file_name = os.path.basename(check_url)
51    cmd = ('curl -s -k ' + sha_sums256_path).split(' ')
52    data_sha_sums256, err, returncode = _run_cmd(cmd)
53    check_sha256 = None
54    for line in data_sha_sums256.split('\n'):
55        if file_name in line:
56            check_sha256 = line.split(' ')[0]
57    return check_sha256
58
59
60def npm_install(dir):
61    result = subprocess.run(["npm", "install", "--prefix", dir], capture_output=True, text=True)
62    if result.returncode == 0:
63        print("{}目录下npm install完毕".format(dir))
64    else:
65        print("依赖项安装失败:", result.stderr)
66
67
68def copy_file(src, dest):
69    if not os.path.exists(dest):
70        os.makedirs(dest)
71    shutil.copy(src, dest)
72
73
74def copy_folder(src, dest):
75    if os.path.exists(dest):
76        if os.path.islink(dest):
77            os.unlink(dest)
78        else:
79            shutil.rmtree(dest)
80    shutil.copytree(src, dest)
81
82
83def symlink_src2dest(src_dir, dest_dir):
84    if os.path.exists(dest_dir):
85        if os.path.islink(dest_dir):
86            os.unlink(dest_dir)
87        elif os.path.isdir(dest_dir):
88            shutil.rmtree(dest_dir)
89        else:
90            os.remove(dest_dir)
91    else:
92        os.makedirs(os.path.dirname(dest_dir), exist_ok=True)
93
94    print("symlink {} ---> {}".format(src_dir, dest_dir))
95    os.symlink(src_dir, dest_dir)
96
97
98def install_hpm(code_path, download_dir, symlink_dir, home_path):
99    content = """\
100package-lock=true
101registry=http://repo.huaweicloud.com/repository/npm
102strict-ssl=false
103lockfile=false
104"""
105    with os.fdopen(os.open(os.path.join(home_path, '.npmrc'), os.O_WRONLY | os.O_CREAT, mode=0o640), 'w') as f:
106        os.truncate(f.fileno(), 0)
107        f.write(content)
108    if not os.path.exists(download_dir):
109        os.makedirs(download_dir)
110    with os.fdopen(os.open(os.path.join(download_dir, 'package.json'), os.O_WRONLY | os.O_CREAT, mode=0o640),
111                    'w') as f:
112        os.truncate(f.fileno(), 0)
113        f.write('{}\n')
114    npm_path = os.path.join(code_path, "prebuilts/build-tools/common/nodejs/current/bin/npm")
115    node_bin_path = os.path.join(code_path, "prebuilts/build-tools/common/nodejs/current/bin")
116    os.environ['PATH'] = f"{node_bin_path}:{os.environ['PATH']}"
117    subprocess.run(
118        [npm_path, 'install', '@ohos/hpm-cli', '--registry', 'https://repo.huaweicloud.com/repository/npm/', '--prefix',
119         download_dir])
120    symlink_src2dest(os.path.join(download_dir, 'node_modules'), symlink_dir)
121
122
123def process_npm(npm_dict, args):
124    code_path = args.code_path
125    home_path = args.home_path
126    name = npm_dict.get('name')
127    npm_download = npm_dict.get('download')
128    package_path = os.path.join(code_path, npm_download.get('package_path'))
129    package_lock_path = os.path.join(code_path, npm_download.get('package_lock_path'))
130    hash_value = \
131        subprocess.run(['sha256sum', package_lock_path], capture_output=True, text=True).stdout.strip().split(' ')[0]
132    download_dir = os.path.join(home_path, npm_download.get('download_dir'), hash_value)
133
134    if '@ohos/hpm-cli' == name:
135        symlink = os.path.join(code_path, npm_download.get('symlink'))
136        install_hpm(code_path, os.path.join(home_path, download_dir), os.path.join(code_path, symlink), home_path)
137        return
138
139    copy_file(package_path, download_dir)
140    copy_file(package_lock_path, download_dir)
141
142    if not os.path.exists(os.path.join(download_dir, "npm-install.js")):
143        npm_install_script = os.path.join(os.path.dirname(package_path), "npm-install.js")
144        if os.path.exists(npm_install_script):
145            shutil.copyfile(npm_install_script, os.path.join(download_dir, "npm-install.js"))
146
147    npm_install(download_dir)
148
149    if name == 'legacy_bin':
150        for link in npm_download.get('symlink', []):
151            symlink_src2dest(os.path.join(download_dir, "node_modules"), os.path.join(code_path, link))
152        return
153
154    symlink = os.path.join(code_path, npm_download.get('symlink'))
155
156    if name in ['parse5']:
157        copy_folder(os.path.join(download_dir, "node_modules"), symlink)
158        return
159
160    copy_folder(os.path.join(download_dir, "node_modules"), symlink)
161
162    for copy_entry in npm_download.get('copy', []):
163        copy_folder(os.path.join(code_path, copy_entry['src']), os.path.join(code_path, copy_entry['dest']))
164
165    for copy_ext_entry in npm_download.get('copy_ext', []):
166        copy_folder(os.path.join(code_path, copy_ext_entry['src']), os.path.join(code_path, copy_ext_entry['dest']))
167
168
169def download_url(url, folder_path):
170    filename = url.split('/')[-1]
171    file_path = os.path.join(folder_path, filename)
172    if os.path.exists(file_path):
173        if _check_sha256(url, file_path):
174            return
175        else:
176            os.remove(file_path)
177    if not os.path.exists(folder_path):
178        os.makedirs(folder_path)
179    try:
180        print("Downloading {}".format(url))
181        with urllib.request.urlopen(url) as response, os.fdopen(
182                os.open(file_path, os.O_WRONLY | os.O_CREAT, mode=0o640), 'wb') as out_file:
183            total_size = int(response.headers['Content-Length'])
184            chunk_size = 16 * 1024
185            downloaded_size = 0
186            print_process(chunk_size, downloaded_size, out_file, response, total_size)
187        print("\n{} downloaded successfully".format(url))
188    except urllib.error.URLError as e:
189        print("Error:", e.reason)
190    except socket.timeout:
191        print("Timeout error: Connection timed out")
192
193
194def print_process(chunk_size, downloaded_size, out_file, response, total_size):
195    while True:
196        chunk = response.read(chunk_size)
197        if not chunk:
198            break
199        out_file.write(chunk)
200        downloaded_size += len(chunk)
201        if total_size != 0:
202            progress = downloaded_size / total_size * 50
203            print(f'\r[{"=" * int(progress)}{" " * (50 - int(progress))}] {progress * 2:.2f}%', end='', flush=True)
204        else:
205            print("\r[Error] Total size is zero, unable to calculate progress.", end='', flush=True)
206
207
208def extract_compress_files(source_file, destination_folder):
209    if not os.path.exists(destination_folder):
210        os.makedirs(destination_folder, exist_ok=True)
211    if source_file.endswith('.zip'):
212        command = ['unzip', '-qq', '-o', '-d', destination_folder, source_file]
213    elif source_file.endswith('.tar.gz'):
214        command = ['tar', '-xzf', source_file, '-C', destination_folder]
215    elif source_file.endswith('.tar.xz'):
216        command = ['tar', '-xJf', source_file, '-C', destination_folder]
217    else:
218        print("暂不支持解压此类型压缩文件！")
219        command = []
220    try:
221        subprocess.run(command, check=True)
222        print(f"{source_file} extracted to {destination_folder}")
223    except subprocess.CalledProcessError as e:
224        print(f"Error: {e}")
225
226
227def install_python(version, download_dir, one_type, code_path):
228    after_extract_folder = os.listdir(os.path.join(download_dir, version))[0]
229    copy_folder(os.path.join(download_dir, version, after_extract_folder, one_type.get('copy_src')),
230                os.path.join(download_dir, version, after_extract_folder, one_type.get('copy_dest')))
231    copy_folder(os.path.join(download_dir, version, after_extract_folder),
232                os.path.join(code_path, one_type.get('symlink')))
233    python3_path = os.path.join(download_dir, version, after_extract_folder, one_type.get('copy_src'),
234                                'bin', 'python3.10')
235    pip3_path = os.path.join(download_dir, version, after_extract_folder, one_type.get('copy_src'), 'bin',
236                             'pip3.10')
237    subprocess.run(
238        [python3_path, pip3_path, "install", "--trusted-host", "repo.huaweicloud.com", "-i",
239         "http://repo.huaweicloud.com/repository/pypi/simple"] + one_type.get(
240            'pip_install'))
241
242
243def install_rustc(url, download_dir, version, one_type, code_path):
244    part_file_name = url.split('/')[-1].split('linux')[0]
245    after_extract_file_list = os.listdir(os.path.join(download_dir, version))
246    for file_name in after_extract_file_list:
247        if part_file_name in file_name:
248            script_path = os.path.join(download_dir, version, file_name, "install.sh")
249            subprocess.run([script_path, "--prefix=''",
250                            "--destdir='{}'".format(os.path.join(code_path, one_type.get('destdir')))])
251            break
252
253
254def process_tar(tar_dict, args):
255    code_path = args.code_path
256    home_path = args.home_path
257    os_type = args.target_os
258    cpu_type = args.target_cpu
259    tar_name = tar_dict.get('name')
260    print(tar_name)
261    tar_download = tar_dict.get('download')
262    for one_type in tar_download:
263        download_flag = download_or_not(cpu_type, one_type, os_type)
264        if download_flag == True:
265            version = one_type.get('version')
266            url = os.path.join(args.repo_https, one_type.get('url'))
267            download_dir = os.path.join(home_path, one_type.get('download_dir'))
268            download_url(url, download_dir)
269            extract_compress_files(os.path.join(download_dir, url.split('/')[-1]),
270                                   os.path.join(download_dir, version))
271            if tar_name == 'python':
272                install_python(version, download_dir, one_type, code_path)
273                continue
274            if 'rustc' in tar_name:
275                install_rustc(url, download_dir, version, one_type, code_path)
276                continue
277            if tar_name == 'llvm':
278                after_extract_folder = os.listdir(os.path.join(download_dir, version))[0]
279                copy_folder(os.path.join(download_dir, version, after_extract_folder, one_type.get('copy_src')),
280                            os.path.join(download_dir, version, after_extract_folder, one_type.get('copy_dest')))
281                copy_folder(os.path.join(download_dir, version, after_extract_folder),
282                            os.path.join(code_path, one_type.get('symlink')))
283                continue
284            if tar_name in ['ark_tools']:
285                after_extract_folder = os.listdir(os.path.join(download_dir, version))[0]
286                copy_folder(os.path.join(download_dir, version, after_extract_folder),
287                            os.path.join(code_path, one_type.get('symlink')))
288                continue
289            after_extract_file_list = os.listdir(os.path.join(download_dir, version))
290            symlink = os.path.join(code_path, one_type.get('symlink'))
291            if one_type.get('type') == 'dir':
292                deal_tar_dir(after_extract_file_list, download_dir, symlink, tar_name, version)
293            else:
294                file_name = after_extract_file_list[0]
295                symlink_src2dest(os.path.join(download_dir, version, file_name), symlink)
296            if tar_name == 'nodejs' and url == os.path.join(args.repo_https,
297                                                            "nodejs/v14.21.1/node-v14.21.1-linux-x64.tar.gz"):
298                symlink_src2dest(symlink, os.path.join(os.path.dirname(symlink), 'current'))
299
300
301def download_or_not(cpu_type, one_type, os_type):
302    download_flag = False
303    if cpu_type == 'any':
304        if one_type.get('target_os') == os_type:
305            download_flag = True
306    else:
307        if one_type.get('target_os') == os_type and (
308                one_type.get('target_cpu') == cpu_type or one_type.get('target_cpu') == ""):
309            download_flag = True
310    return download_flag
311
312
313def deal_tar_dir(after_extract_file_list, download_dir, symlink, tar_name, version):
314    for one_file in after_extract_file_list:
315        if tar_name == 'packing_tool':
316            symlink_src2dest(os.path.join(download_dir, version, one_file),
317                             os.path.join(symlink, one_file))
318        else:
319            symlink_src2dest(os.path.join(download_dir, version, one_file),
320                             symlink)
321
322
323def main():
324    parser = argparse.ArgumentParser()
325    parser.add_argument('--code_path', required=True, type=str, help='path of openharmony code')
326    parser.add_argument('--home_path', required=True, type=str, help='path of home')
327    parser.add_argument('--config_file', required=True, type=str, help='path of prebuilts_config.json')
328    parser.add_argument('--repo_https', required=True, type=str, default='https://repo.huaweicloud.com',
329                        help='path of prebuilts_config.json')
330    parser.add_argument('--target_os', required=True, type=str, help='type of os')
331    parser.add_argument('--target_cpu', type=str, default='any', help='type of cpu')
332    args = parser.parse_args()
333    config_file = args.config_file
334    with open(config_file, 'r', encoding='utf-8') as r:
335        config = json.load(r)
336    tar = config['tar']
337    for one_tar in tar:
338        process_tar(one_tar, args)
339    npm = config['npm']
340    for one_npm in npm:
341        process_npm(one_npm, args)
342
343
344if __name__ == '__main__':
345    sys.exit(main())
346