1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# 4# Copyright (c) 2022 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18from __future__ import print_function 19from datetime import datetime 20from fnmatch import fnmatch 21import errno 22import json 23import os 24import platform 25import subprocess 26import sys 27 28CURRENT_FILENAME = os.path.basename(__file__) 29 30 31def str_of_time_now() -> str: 32 return datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f")[:-3] 33 34 35def _call(cmd: str): 36 print("# %s" % cmd) 37 return subprocess.call(cmd, shell=True) 38 39 40def _write(filename: str, content: str, mode: str): 41 with open(filename, mode) as f: 42 f.write(content) 43 44 45def call_with_output(cmd: str, file: str): 46 print("# %s" % cmd) 47 host = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) 48 while True: 49 try: 50 build_data = host.stdout.readline().decode('utf-8') 51 sys.stdout.flush() 52 print(build_data) 53 _write(file, build_data, "a") 54 except OSError as error: 55 if error == errno.ENOENT: 56 print("no such file") 57 elif error == errno.EPERM: 58 print("permission denied") 59 break 60 if not build_data: 61 break 62 host.wait() 63 return host.returncode 64 65 66class ArkPy: 67 # constants determined by designer of this class 68 NAME_OF_OUT_DIR_OF_FIRST_LEVEL = "out" 69 DELIMITER_BETWEEN_OS_CPU_MODE_FOR_COMMAND = "." 70 DELIMITER_FOR_SECOND_OUT_DIR_NAME = "." 71 GN_TARGET_LOG_FILE_NAME = "build.log" 72 UNITTEST_LOG_FILE_NAME = "unittest.log" 73 TEST262_LOG_FILE_NAME = "test262.log" 74 PREBUILTS_DOWNLOAD_CONFIG_FILE_PATH = \ 75 "./arkcompiler/toolchain/build/prebuilts_download/prebuilts_download_config.json" 76 INDENTATION_STRING_PER_LEVEL = " " # for help message 77 # In ARG_DICT, "flags" and "description" are must-keys for the leaf-dicts in it. 78 # (Future designer need know.) 79 ARG_DICT = { 80 "os_cpu": { 81 "linux_x64": { 82 "flags": ["linux_x64", "x64"], 83 "description": 84 "Build for arkcompiler target of target-operating-system linux and " 85 "target-central-processing-unit x64.", 86 "gn_args": ["target_os=\"linux\"", "target_cpu=\"x64\""], 87 "prefix_of_name_of_out_dir_of_second_level": "x64", 88 }, 89 "linux_x86": { 90 "flags": ["linux_x86", "x86"], 91 "description": 92 "Build for arkcompiler target of target-operating-system linux and " 93 "target-central-processing-unit x86.", 94 "gn_args": ["target_os=\"linux\"", "target_cpu=\"x86\""], 95 "prefix_of_name_of_out_dir_of_second_level": "x86", 96 }, 97 "ohos_arm": { 98 "flags": ["ohos_arm", "arm"], 99 "description": 100 "Build for arkcompiler target of target-operating-system ohos and " 101 "target-central-processing-unit arm.", 102 "gn_args": ["target_os=\"ohos\"", "target_cpu=\"arm\""], 103 "prefix_of_name_of_out_dir_of_second_level": "arm", 104 }, 105 "ohos_arm64": { 106 "flags": ["ohos_arm64", "arm64"], 107 "description": 108 "Build for arkcompiler target of target-operating-system ohos and " 109 "target-central-processing-unit arm64.", 110 "gn_args": ["target_os=\"ohos\"", "target_cpu=\"arm64\""], 111 "prefix_of_name_of_out_dir_of_second_level": "arm64", 112 }, 113 "android_arm64": { 114 "flags": ["android_arm64"], 115 "description": 116 "Build for arkcompiler target of target-operating-system android and " 117 "target-central-processing-unit arm64.", 118 "gn_args": ["target_os=\"android\"", "target_cpu=\"arm64\""], 119 "prefix_of_name_of_out_dir_of_second_level": "android_arm64", 120 }, 121 "mingw_x86_64": { 122 "flags": ["mingw_x86_64"], 123 "description": 124 "Build for arkcompiler target of target-operating-system MinGW(Minimalist GNU on Windows) and " 125 "target-central-processing-unit x86_64.", 126 "gn_args": ["target_os=\"mingw\"", "target_cpu=\"x86_64\""], 127 "prefix_of_name_of_out_dir_of_second_level": "mingw_x86_64", 128 }, 129 "ohos_mipsel": { 130 "flags": ["ohos_mipsel", "mipsel"], 131 "description": 132 "Build for arkcompiler target of target-operating-system ohos and " 133 "target-central-processing-unit mipsel(32-bit little-endian mips).", 134 "gn_args": ["target_os=\"ohos\"", "target_cpu=\"mipsel\""], 135 "prefix_of_name_of_out_dir_of_second_level": "mipsel", 136 }, 137 }, 138 "mode": { 139 "release": { 140 "flags": ["release", "r"], 141 "description": "Build for arkcompiler target(executables and libraries) for distribution.", 142 "gn_args": ["is_debug=false"], 143 "suffix_of_name_of_out_dir_of_second_level": "release", 144 }, 145 "debug": { 146 "flags": ["debug", "d"], 147 "description": "Build for arkcompiler target(executables and libraries) for debugging.", 148 "gn_args": ["is_debug=true"], 149 "suffix_of_name_of_out_dir_of_second_level": "debug", 150 }, 151 }, 152 "target": { 153 "test262": { 154 "flags": ["test262", "test-262", "test_262", "262test", "262-test", "262_test", "262"], 155 "description": "Compile arkcompiler target and run test262 with arkcompiler target.", 156 "gn_targets_depend_on": ["default"], 157 }, 158 "unittest": { 159 "flags": ["unittest", "ut"], 160 "description": 161 "Compile and run unittest of arkcompiler target. " 162 "Add --gn-args=\"run_with_qemu=true\" to command when running unittest of non-host type with qemu.", 163 "gn_targets_depend_on": ["unittest_packages"], 164 }, 165 "gn_target": { 166 "flags": ["<name of target in \"*.gn*\" file>"], # any other flags 167 "description": 168 "Build for arkcompiler target assigned by user. Targets include group(ets_runtime), " 169 "ohos_executable(ark_js_vm), ohos_shared_library(libark_jsruntime), " 170 "ohos_static_library(static_icuuc), ohos_source_set(libark_jsruntime_set), " 171 "ohos_unittest(EcmaVm_001_Test), action(EcmaVm_001_TestAction) and other target of user-defined " 172 "template type in \"*.gn*\" file.", 173 "gn_targets_depend_on": [], # not need, depend on deps of itself in "*.gn*" file 174 }, 175 }, 176 "option": { 177 "clean": { 178 "flags": ["--clean", "-clean"], 179 "description": 180 "Clean the root-out-dir(x64.release-->out/x64.release) execept for file args.gn. " 181 "Then exit.", 182 }, 183 "clean-continue": { 184 "flags": ["--clean-continue", "-clean-continue"], 185 "description": 186 "Clean the root-out-dir(x64.release-->out/x64.release) execept for file args.gn. " 187 "Then continue to build.", 188 }, 189 "gn-args": { 190 "flags": ["--gn-args=*", "-gn-args=*"], 191 "description": 192 "Pass args(*) to gn command. Example: python3 ark.py x64.release " 193 "--gn-args=\"bool_declared_in_src_gn=true string_declared_in_src_gn=\\\"abcd\\\" " 194 "list_declared_in_src_gn=[ \\\"element0\\\", \\\"element1\\\" ] print(list_declared_in_src_gn) " 195 "exec_script(\\\"script_in_src\\\", [ \\\"arg_to_script\\\" ])\" .", 196 }, 197 "keepdepfile": { 198 "flags": ["--keepdepfile", "-keepdepfile"], 199 "description": 200 "Keep depfile(\"*.o.d\") generated by commands(CXX, CC ...) called by ninja during compilation.", 201 }, 202 "verbose": { 203 "flags": ["--verbose", "-verbose"], 204 "description": "Print full commands(CXX, CC, LINK ...) called by ninja during compilation.", 205 }, 206 }, 207 "help": { 208 "flags": ["help", "--help", "--h", "-help", "-h"], 209 "description": "Show the usage of ark.py.", 210 }, 211 } 212 213 # variables which would change with the change of host_os or host_cpu 214 gn_binary_path = "" 215 ninja_binary_path = "" 216 217 # variables which would change with the change of ark.py command 218 has_cleaned = False 219 enable_verbose = False 220 enable_keepdepfile = False 221 222 def get_binaries(self): 223 host_os = sys.platform 224 host_cpu = platform.machine() 225 try: 226 with open(self.PREBUILTS_DOWNLOAD_CONFIG_FILE_PATH) as file_prebuilts_download_config: 227 prebuilts_download_config_dict = json.load(file_prebuilts_download_config) 228 file_prebuilts_download_config.close() 229 for element in prebuilts_download_config_dict[host_os][host_cpu]["copy_config"]: 230 if element["unzip_filename"] == "gn": 231 self.gn_binary_path = os.path.join(element["unzip_dir"], element["unzip_filename"]) 232 elif element["unzip_filename"] == "ninja": 233 self.ninja_binary_path = os.path.join(element["unzip_dir"], element["unzip_filename"]) 234 except Exception as error: 235 print("\nLogic of getting gn binary or ninja binary does not match logic of prebuilts_download." \ 236 "\nCheck func \033[92m{0} of class {1} in file {2}\033[0m against file {3} if the name of this " \ 237 "file had not changed!\n".format( 238 sys._getframe().f_code.co_name, self.__class__.__name__, CURRENT_FILENAME, 239 self.PREBUILTS_DOWNLOAD_CONFIG_FILE_PATH)) 240 raise error 241 if self.gn_binary_path == "" or self.ninja_binary_path == "": 242 print("\nLogic of prebuilts_download may be wrong." \ 243 "\nCheck \033[92mdata in file {0}\033[0m against func {1} of class {2} in file {3}!\n".format( 244 self.PREBUILTS_DOWNLOAD_CONFIG_FILE_PATH, sys._getframe().f_code.co_name, self.__class__.__name__, 245 CURRENT_FILENAME)) 246 sys.exit(0) 247 if not os.path.isfile(self.gn_binary_path) or not os.path.isfile(self.ninja_binary_path): 248 print("\nStep for prebuilts_download may be ommited. (\033[92m./prebuilts_download.sh\033[0m)" \ 249 "\nCheck \033[92mwhether gn binary and ninja binary are under directory prebuilts\033[0m!\n".format()) 250 sys.exit(0) 251 return 252 253 @staticmethod 254 def is_dict_flags_match_arg(dict_to_match: dict, arg_to_match: str) -> bool: 255 for flag in dict_to_match["flags"]: 256 if fnmatch(arg_to_match, flag): 257 return True 258 return False 259 260 def which_dict_flags_match_arg(self, dict_including_dicts_to_match: dict, arg_to_match: str) -> str: 261 for key in dict_including_dicts_to_match.keys(): 262 if self.is_dict_flags_match_arg(dict_including_dicts_to_match[key], arg_to_match): 263 return key 264 return "" 265 266 def dict_in_os_cpu_mode_match_arg(self, arg: str) -> [bool, str, str]: 267 os_cpu_part = "" 268 mode_part = "" 269 match_success = True 270 key_to_dict_in_os_cpu_matched_arg = "" 271 key_to_dict_in_mode_matched_arg = "" 272 arg_to_list = arg.split(self.DELIMITER_BETWEEN_OS_CPU_MODE_FOR_COMMAND) 273 if len(arg_to_list) == 1: 274 os_cpu_part = arg_to_list[0] 275 mode_part = "release" 276 key_to_dict_in_os_cpu_matched_arg = self.which_dict_flags_match_arg(self.ARG_DICT["os_cpu"], os_cpu_part) 277 key_to_dict_in_mode_matched_arg = self.which_dict_flags_match_arg(self.ARG_DICT["mode"], mode_part) 278 elif len(arg_to_list) == 2: 279 os_cpu_part = arg_to_list[0] 280 mode_part = arg_to_list[1] 281 key_to_dict_in_os_cpu_matched_arg = self.which_dict_flags_match_arg(self.ARG_DICT["os_cpu"], os_cpu_part) 282 key_to_dict_in_mode_matched_arg = self.which_dict_flags_match_arg(self.ARG_DICT["mode"], mode_part) 283 else: 284 print("\"\033[92m{0}\033[0m\" combined with more than 2 flags is not supported.".format(arg)) 285 if (key_to_dict_in_os_cpu_matched_arg == "") | (key_to_dict_in_mode_matched_arg == ""): 286 match_success = False 287 return [match_success, key_to_dict_in_os_cpu_matched_arg, key_to_dict_in_mode_matched_arg] 288 289 def get_help_msg_of_dict(self, dict_in: dict, indentation_str_current: str, indentation_str_per_level: str) -> str: 290 help_msg = "".format() 291 for key in dict_in.keys(): 292 if isinstance(dict_in[key], dict): 293 help_msg += "{0}{1}:\n".format(indentation_str_current, key) 294 help_msg += self.get_help_msg_of_dict( 295 dict_in[key], indentation_str_current + indentation_str_per_level, indentation_str_per_level) 296 elif key == "flags": 297 help_msg += "{0}{1}: \033[92m{2}\033[0m\n".format(indentation_str_current, key, " ".join(dict_in[key])) 298 elif key == "description": 299 help_msg += "{0}{1}: {2}\n".format(indentation_str_current, key, dict_in[key]) 300 return help_msg 301 302 def get_help_msg_of_all(self) -> str: 303 help_msg = "".format() 304 # Command template 305 help_msg += "\033[32mCommand template:\033[0m\n{}\n\n".format( 306 " python3 ark.py \033[92m[os_cpu].[mode] [gn_target] [option]\033[0m\n" 307 " python3 ark.py \033[92m[os_cpu].[mode] [test262] [none, file or dir] [option]\033[0m\n" 308 " python3 ark.py \033[92m[os_cpu].[mode] [unittest] [option]\033[0m") 309 # Command examples 310 help_msg += "\033[32mCommand examples:\033[0m\n{}\n\n".format( 311 " python3 ark.py \033[92mx64.release\033[0m\n" 312 " python3 ark.py \033[92mx64.release ets_runtime\033[0m\n" 313 " python3 ark.py \033[92mx64.release ark_js_vm es2panda\033[0m\n" 314 " python3 ark.py \033[92mx64.release ark_js_vm es2panda --clean\033[0m\n" 315 " python3 ark.py \033[92mx64.release test262\033[0m\n" 316 " python3 ark.py \033[92mx64.release test262 built-ins/Array\033[0m\n" 317 " python3 ark.py \033[92mx64.release test262 built-ins/Array/name.js\033[0m\n" 318 " python3 ark.py \033[92mx64.release unittest\033[0m") 319 # Arguments 320 help_msg += "\033[32mArguments:\033[0m\n{}".format( 321 self.get_help_msg_of_dict( 322 self.ARG_DICT, self.INDENTATION_STRING_PER_LEVEL, self.INDENTATION_STRING_PER_LEVEL)) 323 return help_msg 324 325 def clean(self, out_path: str): 326 if not os.path.exists(out_path): 327 print("Path \"{}\" does not exist! No need to clean it.".format(out_path)) 328 else: 329 print("=== clean start ===") 330 code = _call("{0} clean {1}".format(self.gn_binary_path, out_path)) 331 if code != 0: 332 print("=== clean failed! ===\n") 333 sys.exit(code) 334 print("=== clean success! ===\n") 335 return 336 337 def build_for_gn_target(self, out_path: str, gn_args: list, arg_list: list, log_file_name: str): 338 # prepare log file 339 build_log_path = os.path.join(out_path, log_file_name) 340 str_to_build_log = "================================\nbuild_time: {0}\nbuild_target: {1}\n\n".format( 341 str_of_time_now(), " ".join(arg_list)) 342 _write(build_log_path, str_to_build_log, "a") 343 # gn command 344 print("=== gn gen start ===") 345 code = call_with_output( 346 "{0} gen {1} --args=\"{2}\"".format( 347 self.gn_binary_path, out_path, " ".join(gn_args).replace("\"", "\\\"")), 348 build_log_path) 349 if code != 0: 350 print("=== gn gen failed! ===\n") 351 sys.exit(code) 352 else: 353 print("=== gn gen success! ===\n") 354 # ninja command 355 # Always add " -d keeprsp" to ninja command to keep response file("*.rsp"), thus we could get shared libraries 356 # of an excutable from its response file. 357 ninja_cmd = \ 358 self.ninja_binary_path + \ 359 (" -v" if self.enable_verbose else "") + \ 360 (" -d keepdepfile" if self.enable_keepdepfile else "") + \ 361 " -d keeprsp" + \ 362 " -C {}".format(out_path) + \ 363 " {}".format(" ".join(arg_list)) 364 code = call_with_output(ninja_cmd, build_log_path) 365 if code != 0: 366 print("=== ninja failed! ===\n") 367 sys.exit(code) 368 else: 369 print("=== ninja success! ===\n") 370 return 371 372 def build_for_test262(self, out_path, gn_args: list, arg_list: list, log_file_name: str): 373 args_to_test262_cmd = "" 374 if len(arg_list) == 0: 375 args_to_test262_cmd = "--es2021 all" 376 elif len(arg_list) == 1: 377 if ".js" in arg_list[0]: 378 args_to_test262_cmd = "--file test262/data/test_es2021/{}".format(arg_list[0]) 379 else: 380 args_to_test262_cmd = "--dir test262/data/test_es2021/{}".format(arg_list[0]) 381 else: 382 print("\033[92m\"test262\" not support multiple additional arguments.\033[0m\n".format()) 383 sys.exit(0) 384 self.build_for_gn_target( 385 out_path, gn_args, self.ARG_DICT["target"]["test262"]["gn_targets_depend_on"], log_file_name) 386 387 test262_cmd = "cd arkcompiler/ets_frontend && python3 test262/run_test262.py {0} --timeout 180000" \ 388 " --libs-dir ../../prebuilts/clang/ohos/linux-x86_64/llvm/lib" \ 389 " --ark-tool=../../{1}/arkcompiler/ets_runtime/ark_js_vm" \ 390 " --ark-frontend-binary=../../{1}/arkcompiler/ets_frontend/es2abc" \ 391 " --merge-abc-binary=../../{1}/arkcompiler/ets_frontend/merge_abc" \ 392 " --ark-frontend=es2panda".format(args_to_test262_cmd, out_path) 393 test262_log_path = os.path.join(out_path, log_file_name) 394 str_to_test262_log = "================================\ntest262_time: {0}\ntest262_target: {1}\n\n".format( 395 str_of_time_now(), args_to_test262_cmd) 396 _write(test262_log_path, str_to_test262_log, "a") 397 print("=== test262 start ===") 398 code = call_with_output(test262_cmd, test262_log_path) 399 if code != 0: 400 print("=== test262 fail! ===\n") 401 sys.exit(code) 402 print("=== test262 success! ===\n") 403 return 404 405 def build_for_unittest(self, out_path: str, gn_args: list, log_file_name:str): 406 self.build_for_gn_target( 407 out_path, gn_args, self.ARG_DICT["target"]["unittest"]["gn_targets_depend_on"], 408 log_file_name) 409 return 410 411 def build(self, out_path: str, gn_args: list, arg_list: list): 412 if not os.path.exists(out_path): 413 print("# mkdir -p {}".format(out_path)) 414 os.makedirs(out_path) 415 if len(arg_list) == 0: 416 self.build_for_gn_target(out_path, gn_args, ["default"], self.GN_TARGET_LOG_FILE_NAME) 417 elif self.is_dict_flags_match_arg(self.ARG_DICT["target"]["test262"], arg_list[0]): 418 self.build_for_test262(out_path, gn_args, arg_list[1:], self.TEST262_LOG_FILE_NAME) 419 elif self.is_dict_flags_match_arg(self.ARG_DICT["target"]["unittest"], arg_list[0]): 420 if len(arg_list) > 1: 421 print("\033[92m\"unittest\" not support additional arguments.\033[0m\n".format()) 422 sys.exit(0) 423 self.build_for_unittest(out_path, gn_args, self.UNITTEST_LOG_FILE_NAME) 424 else: 425 self.build_for_gn_target(out_path, gn_args, arg_list, self.GN_TARGET_LOG_FILE_NAME) 426 return 427 428 def match_options(self, arg_list: list, out_path: str) -> [list, list]: 429 arg_list_ret = [] 430 gn_args_ret = [] 431 for arg in arg_list: 432 # match [option][clean] flag 433 if self.is_dict_flags_match_arg(self.ARG_DICT["option"]["clean"], arg): 434 self.clean(out_path) 435 sys.exit(0) 436 # match [option][clean-continue] flag 437 elif self.is_dict_flags_match_arg(self.ARG_DICT["option"]["clean-continue"], arg): 438 if not self.has_cleaned: 439 self.clean(out_path) 440 self.has_cleaned = True 441 # match [option][gn-args] flag 442 elif self.is_dict_flags_match_arg(self.ARG_DICT["option"]["gn-args"], arg): 443 gn_args_ret.append(arg[(arg.find("=") + 1):]) 444 # match [option][keepdepfile] flag 445 elif self.is_dict_flags_match_arg(self.ARG_DICT["option"]["keepdepfile"], arg): 446 if not self.enable_keepdepfile: 447 self.enable_keepdepfile = True 448 # match [option][verbose] flag 449 elif self.is_dict_flags_match_arg(self.ARG_DICT["option"]["verbose"], arg): 450 if not self.enable_verbose: 451 self.enable_verbose = True 452 # make a new list with flag that do not match any flag in [option] 453 else: 454 arg_list_ret.append(arg) 455 return [arg_list_ret, gn_args_ret] 456 457 def start_for_matched_os_cpu_mode(self, os_cpu_key: str, mode_key: str, arg_list: list): 458 # get binary gn and ninja 459 self.get_binaries() 460 # get out_path 461 name_of_out_dir_of_second_level = \ 462 self.ARG_DICT["os_cpu"][os_cpu_key]["prefix_of_name_of_out_dir_of_second_level"] + \ 463 self.DELIMITER_FOR_SECOND_OUT_DIR_NAME + \ 464 self.ARG_DICT["mode"][mode_key]["suffix_of_name_of_out_dir_of_second_level"] 465 out_path = os.path.join(self.NAME_OF_OUT_DIR_OF_FIRST_LEVEL, name_of_out_dir_of_second_level) 466 # match [option] flag 467 [arg_list, gn_args] = self.match_options(arg_list, out_path) 468 # get expression which would be written to args.gn file 469 gn_args.extend(self.ARG_DICT["os_cpu"][os_cpu_key]["gn_args"]) 470 gn_args.extend(self.ARG_DICT["mode"][mode_key]["gn_args"]) 471 # start to build 472 self.build(out_path, gn_args, arg_list) 473 return 474 475 def __main__(self, arg_list: list): 476 # delete duplicate arg in arg_list 477 arg_list = list(dict.fromkeys(arg_list)) 478 # match [help] flag 479 if len(arg_list) == 0 or ( 480 True in [self.is_dict_flags_match_arg(self.ARG_DICT["help"], arg) for arg in arg_list]): 481 print(self.get_help_msg_of_all()) 482 return 483 # match [[os_cpu].[mode]] flag 484 [match_success, key_to_dict_in_os_cpu, key_to_dict_in_mode] = self.dict_in_os_cpu_mode_match_arg(arg_list[0]) 485 if match_success: 486 self.start_for_matched_os_cpu_mode(key_to_dict_in_os_cpu, key_to_dict_in_mode, arg_list[1:]) 487 else: 488 print("\033[92mThe command is not supported! Help message shows below.\033[0m\n{}".format( 489 self.get_help_msg_of_all())) 490 return 491 492 493if __name__ == "__main__": 494 ark_py = ArkPy() 495 ark_py.__main__(sys.argv[1:]) 496