1#!/usr/bin/env python3 2# encoding: utf-8 3# Copyright 2024 Huawei Technologies Co., Ltd 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# ============================================================================ 17 18import os 19import shutil 20import zipfile 21import argparse 22import hashlib 23import subprocess 24 25def extract_source(in_zip_path, out_src_path): 26 "depress source code form release package" 27 print('Extracting zipped release package...') 28 f = zipfile.ZipFile(in_zip_path, "r") 29 f.extractall(path=out_src_path) 30 old_src_dir = out_src_path + "/mindspore-v2.1.0/" 31 new_src_dir = out_src_path + "/source/" 32 os.rename(old_src_dir, new_src_dir) 33 print("Done extraction.") 34 35def do_patch(patch_dir, target_dir): 36 patches = [ 37 '0001-build-gn-c-api-for-OHOS.patch', 38 '0002-train-and-build.patch', 39 '0003-add-js-api.patch', 40 '0004-cross-compile-ndkso-fp16-nnrt-train_capi.patch', 41 '0005-micro-for-ohos.patch', 42 '0006-remove-lite-expression-fix-double-loadso.patch', 43 '0007-deobfuscator.patch', 44 '0008-upgrade-flatbuffers-fix_crash.patch', 45 '0009-npu-zero-copy.patch', 46 '0010-micro-dynamic-shape-support-discrete-value.patch', 47 '0011-fix-npu-infer-memory-leak-delete-liteGraph.patch', 48 '0012-add-mindir-ops.patch', 49 '0013-hiappevent.patch', 50 '0014-DynamicQuant-strategy-optimization.patch', 51 '0015-bugfix-for-cpu-kernel.patch', 52 '0016-bugfix-for-argminmax-swish-int8-and-vad-asan.patch', 53 '0017-bugfix-for-onnx-parser.patch', 54 '0018-nnrt-litegraph-dequant.patch', 55 '0019-adaper-NNCore-Api.patch', 56 '0020-fix-ocr-gcn-model-crash.patch', 57 '0021-add-mindir-ops.patch', 58 '0022-adapter-HiAI-Foundation-NPU.patch', 59 '0023-support-x86-emulator-build.patch', 60 '0024-fix-gcn-model-squeeze-transpose-infershape-not-do.patch', 61 '0025-support-kirin-npu-dynamic-dims.patch', 62 '0026-fix-depthwise-conv-kernel.patch', 63 '0027-reduce-memory-when-npu-compilation-with-cache.patch', 64 '0028-fix-onnx-parser-and-cpu-kernel.patch', 65 '0029-revert-cache-executor.patch', 66 '0030-generate-flatbuffer-notice.patch', 67 '0031-fix-matmul-assemble-can-not-protect-stack-in-mutil-thread.patch', 68 ] 69 70 cwd = os.getcwd() 71 os.chdir(target_dir) 72 print('Change dir to', os.getcwd()) 73 subprocess.run(['git', 'init', '.']) 74 subprocess.run(['git', 'add', '.']) 75 subprocess.run(['git', 'commit', '-m', '"init"']) 76 77 for patch in patches: 78 print('Applying ', patch, '...') 79 ret = subprocess.run(['git', 'apply', '{0}/{1}'.format(patch_dir, patch)]) 80 if ret.returncode != 0: 81 raise Exception("Apply patch {0} failed, ret: {1}".format(patch, ret)) 82 subprocess.run(['git', 'add', '.']) 83 subprocess.run(['git', 'commit', '-m', "auto-apply {0}".format(patch)]) 84 print('Done') 85 os.chdir(cwd) 86 87def create_status_file(out_src_path): 88 with open("{0}/.status".format(out_src_path), 'w+') as f: 89 f.write('ok') 90 91 92def compute_md5(file): 93 m = hashlib.md5() 94 with open(file, 'rb') as f: 95 m.update(f.read()) 96 return m.hexdigest() 97 98 99def save_md5s(folder_path, out_path): 100 files_list = [] 101 for file_name in os.listdir(folder_path): 102 if (file_name.endswith(".patch")): 103 files_list.append(file_name) 104 105 os.makedirs(out_path, exist_ok=True) 106 for pf in files_list: 107 md5_path = os.path.join(out_path, pf.replace(".patch", ".md5")) 108 with open(md5_path, 'w') as f: 109 f.write(compute_md5(os.path.join(folder_path, pf))) 110 111 112def md5_changed(patch_path, md5_path): 113 if not os.path.exists(md5_path): 114 return True 115 patch_list = [] 116 md5_list = [] 117 for file_name in os.listdir(patch_path): 118 if (file_name.endswith(".patch")): 119 patch_list.append(file_name) 120 for file_name in os.listdir(md5_path): 121 if (file_name.endswith(".md5")): 122 md5_list.append(file_name) 123 if (len(patch_list) != len(md5_list)): 124 return True 125 126 for md5_file in md5_list: 127 if not os.path.exists(os.path.join(patch_path, md5_file.replace(".md5", ".patch"))): 128 return True 129 with open(os.path.join(md5_path, md5_file), 'r') as f: 130 origin_v = f.read().strip() 131 if (origin_v != compute_md5(os.path.join(patch_path, md5_file.replace(".md5", ".patch")))): 132 return True 133 return False 134 135 136def source_has_changed(out_src_path, patch_path, md5_path): 137 if not os.path.exists(os.path.join(out_src_path, ".status")): 138 print(".status not exist.") 139 return True 140 return md5_changed(patch_path, md5_path) 141 142 143def main_work(): 144 parser = argparse.ArgumentParser(description="mindspore build helper") 145 parser.add_argument('--in_zip_path') 146 parser.add_argument('--out_src_path') 147 parser.add_argument('--patch_dir') 148 args = vars(parser.parse_args()) 149 150 in_zip_path = os.path.realpath(args['in_zip_path']) 151 out_src_path = args['out_src_path'] 152 patch_dir = os.path.realpath(args['patch_dir']) 153 154 md5_dir = os.path.join(out_src_path, "patches_md5") 155 if source_has_changed(out_src_path, patch_dir, md5_dir): 156 print("remove ", out_src_path) 157 if os.path.exists(out_src_path): 158 shutil.rmtree(out_src_path) 159 save_md5s(patch_dir, md5_dir) 160 161 if os.path.exists(os.path.join(out_src_path, ".status")): 162 print("patch files not changed and " + os.path.join(out_src_path, ".status") + " exists.") 163 return 164 165 os.makedirs(out_src_path, exist_ok=True) 166 out_src_path = os.path.realpath(out_src_path) 167 168 extract_source(in_zip_path, out_src_path) 169 170 do_patch(patch_dir, out_src_path + '/source/') 171 172 create_status_file(out_src_path) 173 174 175if __name__ == "__main__": 176 main_work() 177 178