1#!/usr/bin/env python3 2# coding=utf-8 3 4# Copyright (c) 2020 HiSilicon (Shanghai) Technologies CO., LIMITED. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import json 18import os 19import sys 20import time 21import re 22 23from utils.build_utils import root_path, script_path, target_config_path, output_root 24from utils.build_utils import pkg_tools_path, jlink_tools_path, lzma_tools_path, sign_tools_path, derived_tools_path 25from utils.build_utils import CopyModule, exec_shell, cmp_file, rm_pyc, rm_all, fn_get_subdirs 26from enviroment import TargetEnvironment 27from sdk_generator.target_config_genarator import genarate_reserve_config 28 29sdk_copy_common_files = [ 30 os.path.join(script_path), 31 os.path.join(root_path, 'build.py'), 32 os.path.join(target_config_path), 33 os.path.join(pkg_tools_path), 34 os.path.join(jlink_tools_path), 35 os.path.join(lzma_tools_path), 36 os.path.join(sign_tools_path), 37 os.path.join(derived_tools_path), 38] 39 40sdk_close_components = [ 41] 42 43def compare_path_bin(path1, path2): 44 for file in os.listdir(path1): 45 if not file.endswith('.bin'): 46 continue 47 if file.endswith('sign.bin'): 48 continue 49 f1 = os.path.join(path1, file) 50 f2 = os.path.join(path2, file) 51 print("Comparing:") 52 print(f1) 53 print(f2) 54 if not cmp_file(f1, f2): 55 print("DIFF") 56 return False 57 print("SAME") 58 return True 59 60 61class SdkGenerator: 62 def __init__(self, env: TargetEnvironment, sdk_root_path: str): 63 self.env = env 64 self.sdk_root_path = sdk_root_path 65 replace_suffix = ['.srcrelease'] 66 if self.env.get('replace_suffix', False): 67 replace_suffix = self.env.get('replace_suffix', False) 68 print(replace_suffix) 69 self.copy_module = CopyModule(replace_root=sdk_root_path, replace_suffix=replace_suffix, copy_header = False) 70 self.org_target_output_path = [] 71 self.rm_line_map = {} 72 self.sub_cmake = {} 73 self.all_cmake = {} 74 chip = self.env.get("chip", False) 75 if not isinstance(chip, list): 76 self.chip = [chip] 77 else: 78 self.chip = chip 79 self.sdk_copy_module_mask_add() 80 self.copy_module.append_mask('.srcrelease') 81 82 def parse_depend_cmake_files(self, cmake_trace_file): 83 with open(cmake_trace_file, 'r') as fp: 84 lines = fp.readlines() 85 86 sdk_cmake_dict = {} 87 depend_cmake_files = [] 88 89 for line in lines: 90 dict_ = json.loads(line) 91 if 'file' not in dict_: 92 continue 93 f = dict_['file'] 94 if not f.startswith(root_path) or f.startswith(output_root): 95 continue 96 if 'cmd' in dict_ and dict_['cmd'] == 'add_subdirectory': 97 if f not in self.sub_cmake: 98 self.sub_cmake[f] = set() 99 file_path = os.path.dirname(f) 100 if "$" in dict_['args'][0]: 101 continue 102 depend_cmake = os.path.join(file_path, dict_['args'][0], "CMakeLists.txt") 103 self.sub_cmake[f].add((int(dict_['line']), depend_cmake)) 104 if f in sdk_cmake_dict: 105 continue 106 sdk_cmake_dict[f] = None 107 self.all_cmake[f] = None 108 tmp = f.replace(root_path, self.sdk_root_path) 109 if self.is_close_component(tmp) == False: 110 if tmp.endswith("CMakeLists.txt"): 111 if os.path.exists(os.path.dirname(tmp)): 112 depend_cmake_files.append(f) 113 else: 114 depend_cmake_files.append(f) 115 return depend_cmake_files 116 117 def copy_menuconfig(self): 118 for chip in self.chip: 119 if os.path.exists(os.path.join(root_path, 'build', 'config', 'target_config', chip, 'menuconfig')): 120 self.copy_srcs([os.path.join(root_path, 'build', 'config', 'target_config', chip, 'menuconfig')]) 121 self.copy_srcs([os.path.join(root_path, 'config.in')]) 122 123 def copy_kconfig(self, cmake_trace_file): 124 depend_cmake_files = self.parse_depend_cmake_files(cmake_trace_file) 125 kconfig_fils = [] 126 for f in depend_cmake_files: 127 tmp = f.replace("CMakeLists.txt", 'Kconfig') 128 if os.path.exists(tmp): 129 kconfig_fils.append(tmp) 130 self.copy_srcs(kconfig_fils) 131 132 def copy_depends(self, cmake_trace_file): 133 depend_cmake_files = self.parse_depend_cmake_files(cmake_trace_file) 134 cmake_dest = self.copy_srcs(depend_cmake_files) 135 for path in sdk_copy_common_files: 136 if not os.path.exists(path): 137 sdk_copy_common_files.remove(path) 138 self.copy_srcs(sdk_copy_common_files) 139 if self.env.get('use_memuconfig') != False: 140 self.copy_menuconfig() 141 self.copy_kconfig(cmake_trace_file) 142 if self.env.get('auto_gen_config'): 143 self.genarate_sdk_target_config(self.env.get('pkg_target_name', False)) 144 145 def genarate_efuse_cfg_bin(self, chip_name): 146 if self.env.get('generate_efuse_bin') == True and chip_name in ["ws63", "ws53"]: 147 print("source build efuse_cfg.bin...") 148 exec_shell([sys.executable, os.path.join(target_config_path, chip_name, "script", "efuse_cfg_gen.py")]) 149 150 def genarate_sdk_target_config(self, targets): 151 reserve = {} 152 for target in targets: 153 env = TargetEnvironment(target) 154 chip = env.get('chip') 155 if chip not in reserve: 156 reserve[chip] = {'target': [], 'template': []} 157 reserve[chip]['target'].append(target) 158 reserve[chip]['template'].append(env.get_target_template()) 159 160 print(reserve) 161 for chip in reserve: 162 path = os.path.join(self.sdk_root_path, 'build', 'config', 'target_config', chip) 163 config_path = os.path.join(path, 'config.py') 164 template_path = os.path.join(path, 'target_config.py') 165 genarate_reserve_config(reserve[chip]['target'], config_path) 166 genarate_reserve_config(reserve[chip]['template'], template_path) 167 168 169 def rm_lines_in_file(self, _file, lines): 170 with open(_file, 'r') as fp_read: 171 text = fp_read.readlines() 172 lines = sorted(list(set(lines)), reverse=True) 173 for idx in lines: 174 text.pop(idx - 1) 175 176 with open(_file, 'w') as fp_write: 177 fp_write.write("".join(text)) 178 179 def copy_srcs(self, file_list): 180 dest_src = [] 181 for file in file_list: 182 if not os.path.exists(file): 183 print("SDK GENERATE ERROR!!") 184 print("FILE: %s is not exists!!" % file) 185 raise 186 dest = self.copy_module.copy(file) 187 if dest is not None: 188 dest_src.append(dest) 189 return dest_src 190 191 def register_org_target_path(self, path): 192 self.org_target_output_path.append(path) 193 194 def rm_cmake_lines(self): 195 rm_lines = {} 196 for f, line_dep in self.sub_cmake.items(): 197 for line, dep_cmake in line_dep: 198 if dep_cmake in self.all_cmake: 199 continue 200 if f not in rm_lines: 201 rm_lines[f] = [] 202 rm_lines[f].append(line) 203 for f, lines in rm_lines.items(): 204 sdk_file = f.replace(root_path, self.sdk_root_path) 205 self.rm_lines_in_file(sdk_file, lines) 206 207 def sdk_copy_module_mask_add(self): 208 chip_mask = [] 209 path_list = [target_config_path] 210 for path_mask in path_list: 211 for name in fn_get_subdirs(path_mask): 212 if name not in self.chip and name not in chip_mask: 213 chip_mask.append(name) 214 self.copy_module.append_mask(chip_mask) 215 216 def is_closed_component(self, component_name): 217 closed_components = self.env.get('closed_components', cmake_type=False) 218 open_components = self.env.get('open_components', cmake_type=False) 219 if isinstance(closed_components, list) and component_name in closed_components: 220 return True 221 elif isinstance(open_components, list) and component_name not in open_components: 222 return True 223 return False 224 225 def sdk_delete_tmp_files(self): 226 delete_files = [] 227 delete_files.append(os.path.join(output_root, 'sdk', 'output')) 228 delete_files.append(os.path.join(output_root, 'sdk', 'make.cmd')) 229 for dir_path, dir_names, file_names in os.walk(os.path.join(output_root, 'sdk', 'interim_binary', self.env.get('chip'), 'libs'), topdown=False): 230 for name in file_names: 231 if name.endswith('.a'): 232 if not self.is_closed_component(name[3:-2]): 233 delete_files.append(os.path.join(dir_path, name)) 234 for dir_path, dir_names, file_names in os.walk(os.path.join(output_root, 'sdk', 'build', 'config', 'target_config', self.env.get('chip')), topdown=False): 235 for name in file_names: 236 if name.endswith('.srcrelease'): 237 delete_files.append(os.path.join(dir_path, name)) 238 rm_all(delete_files) 239 rm_pyc(os.path.join(output_root, 'sdk', 'build')) 240 rm_pyc(os.path.join(output_root, 'sdk', 'tools', 'pkg')) 241 242 def sdk_build(self, build_time, nhso, build_level): 243 org_pwd = os.getcwd() 244 os.chdir(self.sdk_root_path) 245 246 self.close_sdk_mem_limit() 247 self.change_sdk_version() 248 249 build_targets = self.env.get('pkg_target_name', cmake_type=False) 250 print(build_targets) 251 sdk_type_list = self.env.get('sdk_type').split(';') 252 for idx, target in enumerate(build_targets): 253 sdk_build_cmd = ['./build.py', target] 254 if build_time != '': 255 sdk_build_cmd.append("-build_time=%s" %build_time) 256 if nhso == True: 257 sdk_build_cmd.append("-nhso") 258 if build_level == 'release': 259 sdk_build_cmd.append("-release") 260 261 ret_code = exec_shell(sdk_build_cmd) 262 if ret_code: 263 sys.exit(1) 264 org_output_path = self.org_target_output_path[idx] 265 sdk_output_path = org_output_path.replace(root_path, self.sdk_root_path) 266 if not compare_path_bin(sdk_output_path, org_output_path): 267 print("sdk build failed") 268 sys.exit(1) 269 os.chdir(org_pwd) 270 if "makefile" in sdk_type_list: 271 self.sdk_makefile_build(nhso, build_level) 272 self.sdk_delete_tmp_files() 273 274 def close_sdk_mem_limit(self): 275 if "SDK_NOT_MEM_LIMIT" not in self.env.config.get("defines", []): 276 return 277 chip_config_path = os.path.join('build', 'config', 'target_config', self.env.get('chip'), 'target_config.py') 278 with open(chip_config_path, "r") as f: 279 config_content = f.read() 280 print("new defines: SDK_NOT_MEM_LIMI, do not limit the memory size of the target in sdk") 281 with open(chip_config_path, "w") as f: 282 config_content = re.sub(r"'defines'\s?:\s?\[", r"'defines': [" + "'SDK_NOT_MEM_LIMIT', ", config_content) 283 f.write(config_content) 284 return 285 286 def change_sdk_version(self): 287 sdk_version_cmd_define = "" 288 for build_def in self.env.config.get("defines", []): 289 if "SDK_VERSION=" not in build_def: 290 continue 291 sdk_version_cmd_define = build_def 292 break 293 if not sdk_version_cmd_define: 294 return 295 chip_config_path = os.path.join('build', 'config', 'target_config', self.env.get('chip'), 'target_config.py') 296 with open(chip_config_path, "r") as f: 297 config_content = f.read() 298 print(config_content) 299 print(f"change defines: {sdk_version_cmd_define}, change the version of sdk") 300 with open(chip_config_path, "w") as f: 301 sdk_version_cmd_define = sdk_version_cmd_define.replace("\"", "\\\"") 302 config_content = re.sub("SDK_VERSION" + r'''=.*?",''', sdk_version_cmd_define + '''",''', config_content) 303 print(config_content) 304 f.write(config_content) 305 return 306 307 def is_close_component(self, path): 308 for f in sdk_close_components: 309 tmp = f.replace(root_path, self.sdk_root_path) 310 if path.startswith(tmp): 311 return True 312 return False 313 314 def sdk_makefile_build(self, nhso, build_level): 315 with open(os.path.join(self.sdk_root_path, 'make.cmd'), 'r') as fp: 316 lines = fp.readlines() 317 idx = 0 318 for line in lines: 319 org_pwd = os.getcwd() 320 start_time = time.time() 321 line = line.replace('\n', '') 322 os.chdir(line) 323 sdk_build_cmd = ['make', '-j48'] 324 if nhso == True: 325 sdk_build_cmd.append("nhso=true") 326 if build_level == 'release': 327 sdk_build_cmd.append("build_level=release") 328 ret_code = exec_shell(sdk_build_cmd) 329 if ret_code: 330 sys.exit(1) 331 end_time = time.time() 332 print("build %s takes %f s" % (line, end_time - start_time)) 333 org_output_path = self.org_target_output_path[idx] 334 sdk_output_path_makefile = "%s-makefile" % org_output_path.replace(root_path, self.sdk_root_path) 335 if not compare_path_bin(sdk_output_path_makefile, org_output_path): 336 print("sdk build failed") 337 sys.exit(1) 338 os.chdir(org_pwd) 339 idx = idx + 1 340