1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2023 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16 17from __future__ import print_function 18 19import os 20import os.path 21import sys 22import argparse 23import glob 24import json 25import re 26import shutil 27import subprocess 28 29# Rust path 30RUST_PATH = '//third_party/rust/' 31 32# import content added to all generated BUILD.gn files. 33IMPORT_CONTENT = '//build/templates/rust/ohos.gni' 34 35# The name of the temporary output directory. 36TARGET_TEMP = 'target_temp' 37 38# Header added to all generated BUILD.gn files. 39BUILD_GN_HEADER = ( 40 '# Copyright (c) 2023 Huawei Device Co., Ltd.\n' + 41 '# Licensed under the Apache License, Version 2.0 (the "License");\n' + 42 '# you may not use this file except in compliance with the License.\n' + 43 '# You may obtain a copy of the License at\n' + 44 '#\n' + 45 '# http://www.apache.org/licenses/LICENSE-2.0\n' + 46 '#\n' + 47 '# Unless required by applicable law or agreed to in writing, software\n' + 48 '# distributed under the License is distributed on an "AS IS" BASIS,\n' + 49 '# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' + 50 '# See the License for the specific language governing permissions and\n' + 51 '# limitations under the License.\n') 52 53# Message to be displayed when this script is called without the --run flag. 54DRY_RUN_CONTENT = ( 55 'Dry-run: This script uses ./' + TARGET_TEMP + ' for output directory,\n' + 56 'runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n' + 57 'and writes to BUILD.gn in the current and subdirectories.\n\n' + 58 'To do do all of the above, use the --run flag.\n' + 59 'See --help for other flags, and more usage notes in this script.\n') 60 61# Rust package name with suffix -d1.d2.d3(+.*)?. 62VERSION_SUFFIX_RE = re.compile(r'^(.*)-[0-9]+\.[0-9]+\.[0-9]+(?:\+.*)?$') 63 64# Crate types corresponding to a library 65LIBRARY_CRATE_TYPES = ['staticlib', 'cdylib', 'lib', 'rlib', 'dylib', 'proc-macro'] 66 67 68def escape_quotes(s: str): 69 # replace '"' with '\\"' 70 return s.replace('"', '\\"') 71 72 73def file_base_name(path: str): 74 return os.path.splitext(os.path.basename(path))[0] 75 76 77def pkg_to_crate_name(s: str): 78 return s.replace('-', '_').replace('.', '_') 79 80 81def get_base_name(path: str): 82 return pkg_to_crate_name(file_base_name(path)) 83 84 85def get_crate_name(crate): 86 # to sort crates in a list 87 return crate.crate_name 88 89 90def get_designated_pkg_info(lines: list, designated: str): 91 package = re.compile(r'^ *\[package\]') 92 designated_re = re.compile('^ *' + designated + ' *= * "([^"]*)') 93 is_package = False 94 for line in lines: 95 if is_package: 96 if designated_re.match(line): 97 line = eval(repr(line).replace(f'\\"', '')) 98 return designated_re.match(line).group(1) 99 else: 100 is_package = package.match(line) is not None 101 return '' 102 103 104def is_build_script(name: str): 105 # Judge whether it is build script. 106 return name.startswith('build_script_') 107 108 109def is_dependent_path(path: str): 110 # Absolute('/') or dependent('.../') paths are not main files of this crate. 111 return path.startswith('/') or path.startswith('.../') 112 113 114def unquote(s): 115 # remove quotes around str 116 if s and len(s) > 1 and s[0] == '"' and s[-1] == '"': 117 return s[1:-1] 118 return s 119 120 121def remove_version_suffix(s): 122 # remove -d1.d2.d3 suffix 123 if VERSION_SUFFIX_RE.match(s): 124 return VERSION_SUFFIX_RE.match(s).group(1) 125 return s 126 127 128def short_out_name(pkg: str, s: str): 129 # replace /.../pkg-*/out/* with .../out/* 130 return re.sub('^/.*/' + pkg + '-[0-9a-f]*/out/', '.../out/', s) 131 132 133class Crate(object): 134 """Information of a Rust crate to collect/emit for an BUILD.gn module.""" 135 136 def __init__(self, runner, outfile_name: str): 137 # Remembered global runner 138 self.runner = runner 139 self.debug = runner.args.debug 140 self.cargo_dir = '' # directory of my Cargo.toml 141 self.outfile = None # open file handle of outfile_name during dump* 142 self.outfile_name = outfile_name # path to BUILD.gn 143 # GN module properties derived from rustc parameters. 144 self.module_type = '' # lib,crate_name,test etc. 145 self.root_pkg_name = '' # parent package name of a sub/test packge 146 # Save parsed status 147 self.error_infos = '' # all errors found during parsing 148 self.line = '' # original rustc command line parameters 149 self.line_num = 1 # runner told input source line number 150 # Parameters collected from rustc command line. 151 self.cap_lints = '' 152 self.crate_name = '' 153 self.edition = '2015' # cargo default is 2015, you can specify the edition as 2018 or 2021 154 self.emit_list = '' # --emit=dep-info,metadata,link 155 self.main_src = '' 156 self.target = '' 157 self.cfgs = list() 158 self.core_deps = list() # first part of self.deps elements 159 self.crate_types = list() 160 self.deps = list() 161 self.features = list() 162 self.ignore_options = list() 163 self.srcs = list() # main_src or merged multiple source files 164 self.shared_libs = list() # -l dylib=wayland-client, -l z 165 self.static_libs = list() # -l static=host_cpuid 166 # Parameters collected from Cargo.toml. 167 self.cargo_pkg_version = '' # value extracted from Cargo.toml version field 168 self.cargo_pkg_authors = '' # value extracted from Cargo.toml authors field 169 self.cargo_pkg_name = '' # value extracted from Cargo.toml name field 170 self.cargo_pkg_description = '' # value extracted from Cargo.toml description field 171 # Parameters related to build.rs. 172 self.build_root = '' 173 self.checked_out_files = False # to check only once 174 self.build_script_outputs = [] # output files generated by build.rs 175 176 def write(self, s: str): 177 # convenient way to output one line at a time with EOL. 178 self.outfile.write(s + '\n') 179 180 def parse_rustc(self, line_num: int, line: str): 181 """Find important rustc arguments to convert to BUILD.gn properties.""" 182 self.line_num = line_num 183 self.line = line 184 args = line.split() # Loop through every argument of rustc. 185 self.parse_args(args) 186 if not self.crate_name: 187 self.error_infos += 'ERROR: missing --crate-name\n' 188 if not self.crate_types: 189 if 'test' in self.cfgs: 190 self.crate_types.append('test') 191 else: 192 self.error_infos += 'ERROR: missing --crate-type or --test\n' 193 elif len(self.crate_types) > 1: 194 if 'lib' in self.crate_types and 'rlib' in self.crate_types: 195 self.error_infos += 'ERROR: cannot generate both lib and rlib crate types\n' 196 if 'test' in self.crate_types: 197 self.error_infos += 'ERROR: cannot handle both --crate-type and --test\n' 198 if not self.main_src: 199 self.error_infos += 'ERROR: missing main source file\n' 200 else: 201 self.srcs.append(self.main_src) 202 if self.cargo_dir: 203 self.get_root_pkg_name() 204 if not self.root_pkg_name: 205 self.root_pkg_name = self.crate_name 206 207 # Process crate with build.rs 208 if not self.skip_crate(): 209 if not self.runner.args.no_pkg_info: 210 self.find_pkg_info() 211 self.find_build_root() 212 if self.runner.args.copy_out: 213 self.copy_out_files() 214 elif self.find_out_files() and self.has_used_out_dir(): 215 self.copy_out_files() 216 217 self.cfgs = sorted(set(self.cfgs)) 218 self.core_deps = sorted(set(self.core_deps)) 219 self.crate_types = sorted(set(self.crate_types)) 220 self.deps = sorted(set(self.deps)) 221 self.features = sorted(set(self.features)) 222 self.ignore_options = sorted(set(self.ignore_options)) 223 self.static_libs = sorted(set(self.static_libs)) 224 self.shared_libs = sorted(set(self.shared_libs)) 225 self.decide_module_type() 226 return self 227 228 def parse_args(self, args): 229 num = 0 230 while num < len(args): 231 arg = args[num] 232 if arg == '--crate-name': 233 num += 1 234 self.crate_name = args[num] 235 elif arg == '--crate-type': 236 num += 1 237 self.crate_types.append(args[num]) 238 elif arg == '--cfg': 239 num += 1 240 self.deal_cfg(args[num]) 241 elif arg == '-C': 242 num += 1 243 self.add_ignore_options_flag(args[num]) # codegen options 244 elif arg.startswith('-C'): 245 self.add_ignore_options_flag(arg[2:]) 246 elif arg == '--cap-lints': 247 num += 1 248 self.cap_lints = args[num] 249 elif arg.startswith('--edition='): 250 self.edition = arg.replace('--edition=', '') 251 elif arg.startswith('--emit='): 252 self.emit_list = arg.replace('--emit=', '') 253 elif arg == '--extern': 254 num += 1 255 self.deal_extern(args[num]) 256 elif (arg.startswith('--error-format=') or arg.startswith('--json=') or 257 arg.startswith('\'-Aclippy')): 258 _ = arg # ignored 259 elif arg == '-L': 260 num += 1 261 self.set_root_pkg_name(args[num]) 262 elif arg == '-l': 263 num += 1 264 self.deal_static_and_dylib(args[num]) 265 elif arg == '--out-dir' or arg == '--color': # ignored 266 num += 1 267 elif arg == '--target': 268 num += 1 269 self.target = args[num] 270 elif arg == '--test': 271 self.crate_types.append('test') 272 elif not arg.startswith('-'): 273 self.set_main_src(args[num]) 274 else: 275 self.error_infos += 'ERROR: unknown ' + arg + '\n' 276 num += 1 277 278 def deal_cfg(self, arg: str): 279 if arg.startswith('\'feature='): 280 feature = unquote(arg.replace('\'feature=', '')[:-1]) 281 # 'runtime' feature removed because it conflicts with static 282 if feature == 'runtime': 283 feature = 'static' 284 self.features.append(feature) 285 else: 286 self.cfgs.append(arg) 287 288 def add_ignore_options_flag(self, flag: str): 289 """Ignore options not used in GN.""" 290 # 'codegen-units' is set in GN global config or by default 291 # 'embed-bitcode' is ignored; we might control LTO with other .gn flag 292 # 'prefer-dynamic' does not work with common flag -C lto 293 if not (flag.startswith('codegen-units=') or flag.startswith('debuginfo=') or 294 flag.startswith('embed-bitcode=') or flag.startswith('extra-filename=') or 295 flag.startswith('incremental=') or flag.startswith('metadata=') or 296 flag == 'prefer-dynamic'): 297 self.ignore_options.append(flag) 298 299 def deal_extern(self, arg: str): 300 deps = re.sub('=/[^ ]*/deps/', ' = ', arg) 301 self.deps.append(deps) 302 self.core_deps.append(re.sub(' = .*', '', deps)) 303 304 def set_root_pkg_name(self, arg: str): 305 if arg.startswith('dependency=') and arg.endswith('/deps'): 306 if '/' + TARGET_TEMP + '/' in arg: 307 self.root_pkg_name = re.sub('^.*/', '', 308 re.sub('/' + TARGET_TEMP + '/.*/deps$', '', arg)) 309 else: 310 self.root_pkg_name = re.sub('^.*/', '', 311 re.sub('/[^/]+/[^/]+/deps$', '', arg)) 312 self.root_pkg_name = remove_version_suffix(self.root_pkg_name) 313 314 def deal_static_and_dylib(self, arg: str): 315 if arg.startswith('static='): 316 self.static_libs.append(re.sub('static=', '', arg)) 317 elif arg.startswith('dylib='): 318 self.shared_libs.append(re.sub('dylib=', '', arg)) 319 else: 320 self.shared_libs.append(arg) 321 322 def set_main_src(self, arg: str): 323 self.main_src = re.sub(r'^/[^ ]*/registry/src/', '.../', arg) 324 self.main_src = re.sub(r'^\.\.\./github.com-[0-9a-f]*/', '.../', self.main_src) 325 self.find_cargo_dir() 326 if self.cargo_dir: 327 if self.runner.args.no_subdir: 328 # all .gn content to /dev/null 329 self.outfile_name = '/dev/null' 330 elif not self.runner.args.one_file: 331 # Use Cargo.toml to write BUILD.gn in the subdirectory. 332 self.outfile_name = os.path.join(self.cargo_dir, 'BUILD.gn') 333 self.main_src = self.main_src[len(self.cargo_dir) + 1:] 334 335 def find_cargo_dir(self): 336 """Deepest directory with Cargo.toml and contains the main_src.""" 337 if not is_dependent_path(self.main_src): 338 dir_name = os.path.dirname(self.main_src) 339 while dir_name: 340 if dir_name.endswith('.'): 341 dir_name = os.path.dirname(dir_name) 342 continue 343 if os.path.exists(os.path.join(dir_name, 'Cargo.toml')): 344 self.cargo_dir = dir_name 345 return 346 dir_name = os.path.dirname(dir_name) 347 348 def skip_crate(self): 349 """Return crate_name or a message if this crate should be skipped.""" 350 # Some Rust packages include extra unwanted crates. 351 # This set contains all such excluded crate names. 352 excluded_crates = set(['protobuf_bin_gen_rust_do_not_use']) 353 if (is_build_script(self.crate_name) or 354 self.crate_name in excluded_crates): 355 return self.crate_name 356 if is_dependent_path(self.main_src): 357 return 'dependent crate' 358 return '' 359 360 def get_root_pkg_name(self): 361 """Read name of [package] in ./Cargo.toml.""" 362 cargo_toml_path = './Cargo.toml' 363 if self.cargo_dir: 364 cargo_toml_path = os.path.join( 365 os.path.join('.', self.cargo_dir), 'Cargo.toml') 366 if not os.path.exists(cargo_toml_path): 367 return 368 with open(cargo_toml_path, 'r') as infile: 369 self.root_pkg_name = get_designated_pkg_info(infile, 'name') 370 return 371 372 def find_pkg_info(self): 373 """Read package info of [package] in ./Cargo.toml.""" 374 cargo_toml_path = './Cargo.toml' 375 if self.cargo_dir: 376 cargo_toml_path = os.path.join( 377 os.path.join('.', self.cargo_dir), 'Cargo.toml') 378 if not os.path.exists(cargo_toml_path): 379 return 380 with open(cargo_toml_path, 'r') as infile: 381 if self.root_pkg_name: 382 self.cargo_pkg_name = self.root_pkg_name 383 else: 384 self.cargo_pkg_name = get_designated_pkg_info(infile, 'name') 385 infile.seek(0) 386 self.cargo_pkg_version = get_designated_pkg_info(infile, 'version') 387 infile.seek(0) 388 pkg_description = get_designated_pkg_info(infile, 'description') 389 pkg_description = pkg_description.replace('\n', '').replace(r'\n', '').strip() 390 self.cargo_pkg_description = pkg_description 391 infile.seek(0) 392 authors_re = re.compile(' *authors *= * \[(.*?)\]', re.S) 393 authors_section = authors_re.search(infile.read()) 394 if authors_section: 395 authors = authors_section.group(1) 396 authors = authors.replace('\n', '').replace(' ', ' ').replace('"', '').strip() 397 if authors.endswith(','): 398 authors = authors[:-1] 399 self.cargo_pkg_authors = authors 400 401 def find_build_root(self): 402 """Read build of [package] in ./Cargo.toml.""" 403 cargo_toml_path = './Cargo.toml' 404 if self.cargo_dir: 405 cargo_toml_path = os.path.join( 406 os.path.join('.', self.cargo_dir), 'Cargo.toml') 407 if not os.path.exists(cargo_toml_path): 408 return 409 with open(cargo_toml_path, 'r') as infile: 410 self.build_root = get_designated_pkg_info(infile, 'build') 411 if not self.build_root: 412 build_rs_path = './build.rs' 413 if self.cargo_dir: 414 build_rs_path = os.path.join(os.path.join('.', self.cargo_dir), 'build.rs') 415 if os.path.exists(build_rs_path): 416 self.build_root = 'build.rs' 417 418 def find_out_files(self): 419 # normal_output_list has build.rs output for normal crates 420 normal_output_list = glob.glob( 421 TARGET_TEMP + '/*/*/build/' + self.root_pkg_name + '-*/out/*') 422 # other_output_list has build.rs output for proc-macro crates 423 other_output_list = glob.glob( 424 TARGET_TEMP + '/*/build/' + self.root_pkg_name + '-*/out/*') 425 return normal_output_list + other_output_list 426 427 def has_used_out_dir(self): 428 '''Returns true if env!('OUT_DIR') is found.''' 429 cmd = ['grep', '-rl', '--exclude', 'build.rs', '--include', '*.rs', "env!('OUT_DIR')", '*'] 430 if self.cargo_dir: 431 cmd = ['grep', '-rl', '--exclude', os.path.join(self.cargo_dir, 'build.rs'), '--include', \ 432 '*.rs', "env!('OUT_DIR')", '*'] 433 return subprocess.call(cmd, shell=False) == 0 434 435 def copy_out_files(self): 436 """Copy build.rs output files to ./out and set up build_script_outputs.""" 437 if self.checked_out_files: 438 return 439 self.checked_out_files = True 440 cargo_out_files = self.find_out_files() 441 out_files = set() 442 out_path = 'out' 443 if self.cargo_dir: 444 out_path = os.path.join(self.cargo_dir, out_path) 445 if cargo_out_files: 446 os.makedirs(out_path, exist_ok=True) 447 for path in cargo_out_files: 448 file_name = path.split('/')[-1] 449 out_files.add(file_name) 450 self.build_script_outputs = sorted(out_files) 451 452 def decide_module_type(self): 453 # Use the first crate type for the default/first module. 454 crate_type = self.crate_types[0] if self.crate_types else '' 455 self.decide_one_module_type(crate_type) 456 457 def decide_one_module_type(self, crate_type: str): 458 """Decide which GN module type to use.""" 459 if crate_type == 'bin': 460 self.module_type = self.crate_name 461 elif crate_type in LIBRARY_CRATE_TYPES: 462 self.module_type = 'lib' 463 elif crate_type == 'test': 464 self.module_type = 'test' 465 else: 466 self.module_type = '' 467 468 def merge_crate(self, other, outfile_name: str): 469 """Try to merge crate into self.""" 470 # Cargo build --tests could recompile a library for tests. 471 # We need to merge such duplicated calls to rustc, with the 472 # algorithm in is_should_merge. 473 should_merge = self.is_should_merge(other) 474 should_merge_test = False 475 if not should_merge: 476 should_merge_test = self.merge_test(other) 477 if should_merge or should_merge_test: 478 self.runner.init_gn_file(outfile_name) 479 # to write debug info 480 with open(outfile_name, 'a') as outfile: 481 self.outfile = outfile 482 other.outfile = outfile 483 self.execute_merge(other, should_merge_test) 484 return True 485 return False 486 487 def is_should_merge(self, other): 488 return (self.crate_name == other.crate_name and 489 self.crate_types == other.crate_types and 490 self.main_src == other.main_src and 491 self.root_pkg_name == other.root_pkg_name and 492 not self.skip_crate() and self.is_same_flags(other)) 493 494 def merge_test(self, other): 495 """Returns true if self and other are tests of same root_pkg_name.""" 496 # Before merger, each test has its own crate_name. A merged test uses 497 # its source file base name as output file name, so a test is mergeable 498 # only if its base name equals to its crate name. 499 return (self.crate_types == other.crate_types and self.crate_types == ['test'] and 500 self.root_pkg_name == other.root_pkg_name and not self.skip_crate() and 501 other.crate_name == get_base_name(other.main_src) and 502 (len(self.srcs) > 1 or (self.crate_name == get_base_name(self.main_src))) and 503 self.is_same_flags(other)) 504 505 def is_same_flags(self, other): 506 return (not self.error_infos and not other.error_infos and 507 self.cap_lints == other.cap_lints and self.cfgs == other.cfgs and 508 self.core_deps == other.core_deps and self.edition == other.edition and 509 self.emit_list == other.emit_list and self.features == other.features and 510 self.ignore_options == other.ignore_options and 511 self.static_libs == other.static_libs and 512 self.shared_libs == other.shared_libs) 513 514 def execute_merge(self, other, should_merge_test): 515 """Merge attributes of other to self.""" 516 if self.debug: 517 self.write('\n// Before merge definition(self):') 518 self.dump_debug_info() 519 self.write('\n// Before merge definition(other):') 520 other.dump_debug_info() 521 if not self.target: 522 # okay to keep only the first target triple 523 self.target = other.target 524 self.decide_module_type() 525 if should_merge_test: 526 if (self.runner.should_ignore_test(self.main_src) and 527 not self.runner.should_ignore_test(other.main_src)): 528 self.main_src = other.main_src 529 self.srcs.append(other.main_src) 530 self.crate_name = pkg_to_crate_name(self.root_pkg_name) 531 if self.debug: 532 self.write('\n// After merge definition:') 533 self.dump_debug_info() 534 535 def dump(self): 536 """Dump all error/debug/module code to the output .gn file.""" 537 self.runner.init_gn_file(self.outfile_name) 538 with open(self.outfile_name, 'a') as outfile: 539 self.outfile = outfile 540 if self.error_infos: 541 self.dump_line() 542 self.write(self.error_infos) 543 elif self.skip_crate(): 544 self.dump_skip_crate(self.skip_crate()) 545 else: 546 if self.debug: 547 self.dump_debug_info() 548 self.dump_gn_module() 549 550 def dump_debug_info(self): 551 """Dump parsed data, when cargo2gn is called with --debug.""" 552 553 def dump(name, value): 554 self.write('//%12s = %s' % (name, value)) 555 556 def dump_list(fmt, values): 557 for v in values: 558 self.write(fmt % v) 559 560 def opt_dump(name, value): 561 if value: 562 dump(name, value) 563 564 self.dump_line() 565 dump('crate_name', self.crate_name) 566 dump('crate_types', self.crate_types) 567 opt_dump('edition', self.edition) 568 opt_dump('emit_list', self.emit_list) 569 dump('main_src', self.main_src) 570 dump('module_type', self.module_type) 571 opt_dump('target', self.target) 572 opt_dump('cap_lints', self.cap_lints) 573 dump_list('// cfg = %s', self.cfgs) 574 dump_list('// cfg = \'feature "%s"\'', self.features) 575 dump_list('// codegen = %s', self.ignore_options) 576 dump_list('// deps = %s', self.deps) 577 dump_list('// -l (dylib) = %s', self.shared_libs) 578 dump_list('// -l static = %s', self.static_libs) 579 580 def dump_line(self): 581 self.write('\n// Line ' + str(self.line_num) + ' ' + self.line) 582 583 def dump_skip_crate(self, kind: str): 584 if self.debug: 585 self.write('\n// IGNORED: ' + kind + ' ' + self.main_src) 586 return self 587 588 def dump_gn_module(self): 589 """Dump one or more GN module definition, depending on crate_types.""" 590 if len(self.crate_types) == 1: 591 self.dump_single_type_gn_module() 592 return 593 if 'test' in self.crate_types: 594 self.write('\nERROR: multiple crate types cannot include test type') 595 return 596 # Dump one GN module per crate_type. 597 for crate_type in self.crate_types: 598 self.decide_one_module_type(crate_type) 599 self.dump_one_gn_module(crate_type) 600 601 def dump_single_type_gn_module(self): 602 """Dump one simple GN module, which has only one crate_type.""" 603 crate_type = self.crate_types[0] 604 if crate_type != 'test': 605 self.dump_one_gn_module(crate_type) 606 return 607 # Dump one test module per source file. 608 self.srcs = [ 609 src for src in self.srcs if not self.runner.should_ignore_test(src)] 610 if len(self.srcs) > 1: 611 self.srcs = sorted(set(self.srcs)) 612 saved_srcs = self.srcs 613 for src in saved_srcs: 614 self.srcs = [src] 615 saved_main_src = self.main_src 616 self.main_src = src 617 self.decide_one_module_type(crate_type) 618 self.dump_one_gn_module(crate_type) 619 self.main_src = saved_main_src 620 self.srcs = saved_srcs 621 622 def dump_one_gn_module(self, crate_type: str): 623 """Dump one GN module definition.""" 624 if not self.module_type: 625 self.write('\nERROR: unknown crate_type ' + crate_type) 626 return 627 self.write('\nohos_cargo_crate("' + self.module_type + '") {') 628 self.dump_gn_first_properties(crate_type) 629 self.dump_gn_core_properties() 630 self.write('}') 631 632 def dump_gn_first_properties(self, crate_type: str): 633 if crate_type != 'bin': 634 self.write(' crate_name = "' + self.crate_name + '"') 635 if crate_type: 636 if crate_type == 'lib': 637 crate_type = 'rlib' 638 self.write(' crate_type = "' + crate_type + '"') 639 if self.main_src: 640 self.write(' crate_root = "' + self.main_src + '"') 641 if self.crate_name.startswith('lib'): 642 self.write(' output_name = "lib' + self.crate_name + '"') 643 self.write('') 644 645 def dump_gn_core_properties(self): 646 self.dump_sources_list() 647 if self.edition: 648 self.write(' edition = "' + self.edition + '"') 649 if not self.runner.args.no_pkg_info: 650 if self.cargo_pkg_version: 651 self.write(' cargo_pkg_version = "' + 652 self.cargo_pkg_version + '"') 653 if self.cargo_pkg_authors: 654 self.write(' cargo_pkg_authors = "' + 655 self.cargo_pkg_authors + '"') 656 if self.cargo_pkg_name: 657 self.write(' cargo_pkg_name = "' + 658 self.cargo_pkg_name + '"') 659 if self.cargo_pkg_description: 660 self.write(' cargo_pkg_description = "' + 661 self.cargo_pkg_description + '"') 662 if self.deps: 663 self.dump_gn_deps() 664 if self.build_root and self.root_pkg_name in self.runner.build_deps: 665 self.dump_gn_build_deps() 666 self.dump_gn_property_list('features', '"%s"', self.features) 667 if self.build_root: 668 self.write(' build_root = "' + self.build_root + '"') 669 build_sources = list() 670 build_sources.append(self.build_root) 671 self.dump_gn_property_list('build_sources', '"%s"', build_sources) 672 if self.build_script_outputs: 673 self.dump_gn_property_list( 674 'build_script_outputs', '"%s"', self.build_script_outputs) 675 676 def dump_sources_list(self): 677 """Dump the srcs list, for defaults or regular modules.""" 678 if len(self.srcs) > 1: 679 srcs = sorted(set(self.srcs)) # make a copy and dedup 680 for num in range(len(self.srcs)): 681 srcs[num] = srcs[num] 682 else: 683 srcs = [self.main_src] 684 self.dump_gn_property_list('sources', '"%s"', srcs) 685 686 def dump_gn_deps(self): 687 """Dump the deps.""" 688 rust_deps = list() 689 deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$') 690 for lib in self.deps: 691 libname_groups = deps_libname.match(lib) 692 if libname_groups is not None: 693 lib_name = libname_groups.group(1) 694 else: 695 lib_name = re.sub(' .*$', '', lib) 696 if lib_name in self.runner.args.dependency_blocklist: 697 continue 698 if lib.endswith('.rlib') or lib.endswith('.rmeta') or lib.endswith('.so'): 699 # On MacOS .rmeta is used when Linux uses .rlib or .rmeta. 700 rust_lib = self.get_rust_lib(lib_name) 701 if rust_lib: 702 rust_lib += ':lib' 703 rust_deps.append(rust_lib) 704 elif lib != 'proc_macro': 705 # --extern proc_macro is special and ignored 706 rust_deps.append('// unknown type of lib: '.join(lib)) 707 if rust_deps: 708 self.dump_gn_property_list('deps', '"%s"', rust_deps) 709 710 def dump_gn_build_deps(self): 711 """Dump the build deps.""" 712 rust_build_deps = list() 713 build_deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$') 714 build_deps = self.runner.build_deps.get(self.root_pkg_name) 715 if not build_deps: 716 return 717 for lib in build_deps: 718 libname_groups = build_deps_libname.match(lib) 719 if libname_groups is not None: 720 lib_name = libname_groups.group(1) 721 else: 722 lib_name = re.sub(' .*$', '', lib) 723 if lib_name in self.runner.args.dependency_blocklist: 724 continue 725 if lib.endswith('.rlib') or lib.endswith('.rmeta') or lib.endswith('.so'): 726 # On MacOS .rmeta is used when Linux uses .rlib or .rmeta. 727 rust_lib = self.get_rust_lib(lib_name) 728 if rust_lib: 729 rust_build_deps.append(rust_lib + ':lib') 730 elif lib != 'proc_macro': 731 # --extern proc_macro is special and ignored 732 rust_build_deps.append('// unknown type of lib: '.join(lib)) 733 if rust_build_deps: 734 self.dump_gn_property_list('build_deps', '"%s"', rust_build_deps) 735 736 def dump_gn_property_list(self, name: str, fmt, values): 737 if not values: 738 return 739 if len(values) > 1: 740 self.write(' ' + name + ' = [') 741 self.dump_gn_property_list_items(fmt, values) 742 self.write(' ]') 743 else: 744 self.write(' ' + name + ' = [' + 745 (fmt % escape_quotes(values[0])) + ']') 746 747 def dump_gn_property_list_items(self, fmt, values): 748 for v in values: 749 # fmt has quotes, so we need escape_quotes(v) 750 self.write(' ' + (fmt % escape_quotes(v)) + ',') 751 752 def get_rust_lib(self, lib_name: str): 753 rust_lib = '' 754 if lib_name: 755 crate_name = pkg_to_crate_name(lib_name) 756 deps_libname = self.runner.deps_libname_map.get(crate_name) 757 if deps_libname: 758 rust_lib = RUST_PATH + deps_libname 759 return rust_lib 760 761 762class Runner(object): 763 """Main class to parse cargo -v output""" 764 765 def __init__(self, args): 766 self.gn_files = set() # Remember all output BUILD.gn files. 767 self.root_pkg_name = '' # name of package in ./Cargo.toml 768 self.args = args 769 self.dry_run = not args.run 770 self.skip_cargo = args.skipcargo 771 self.cargo_path = ['./cargo'] # path to cargo 772 self.crates = list() # all crates 773 self.error_infos = '' # all error infos 774 self.test_error_infos = '' # all test error infos 775 self.warning_files = set() # all warning files 776 self.set_cargo_path() 777 # Default operation is cargo clean, followed by build or user given operation. 778 if args.cargo: 779 self.cargo = ['clean'] + args.cargo 780 else: 781 # Use the same target for both host and default device builds. 782 self.cargo = ['clean', 'build --target x86_64-unknown-linux-gnu'] 783 self.empty_tests = set() 784 self.empty_unittests = False 785 self.build_deps = {} 786 self.deps_libname_map = {} 787 788 def set_cargo_path(self): 789 """Find cargo in the --cargo_bin and set cargo path""" 790 if self.args.cargo_bin: 791 self.cargo_path = [os.path.join(self.args.cargo_bin, 'cargo')] 792 if os.path.isfile(self.cargo_path): 793 print('INFO: using cargo in ' + self.args.cargo_bin) 794 return 795 else: 796 sys.exit('ERROR: cannot find cargo in ' + self.args.cargo_bin) 797 else: 798 sys.exit('ERROR: the prebuilt cargo is not available; please use the --cargo_bin flag.') 799 return 800 801 def run_cargo(self): 802 """Run cargo -v and save its output to ./cargo.out.""" 803 if self.skip_cargo: 804 return self 805 cargo_toml = './Cargo.toml' 806 cargo_out = './cargo.out' 807 if not os.access(cargo_toml, os.R_OK): 808 print('ERROR: Cannot find ', cargo_toml) 809 return self 810 cargo_lock = './Cargo.lock' 811 cargo_lock_save = './cargo.lock.save' 812 have_cargo_lock = os.path.exists(cargo_lock) 813 if not self.dry_run: 814 if os.path.exists(cargo_out): 815 os.remove(cargo_out) 816 if not self.args.use_cargo_lock and have_cargo_lock: 817 os.rename(cargo_lock, cargo_lock_save) 818 # set up search PATH for cargo to find the correct rustc 819 save_path = os.environ['PATH'] 820 os.environ['PATH'] = os.path.dirname(self.cargo_path) + ':' + save_path 821 # Add [workspace] to Cargo.toml if it is non-existent. 822 is_add_workspace = False 823 if self.args.add_workspace: 824 with open(cargo_toml, 'r') as in_file: 825 cargo_toml_lines = in_file.readlines() 826 if '[workspace]\n' in cargo_toml_lines: 827 print('WARNING: found [workspace] in Cargo.toml') 828 else: 829 with open(cargo_toml, 'w') as out_file: 830 out_file.write('[workspace]\n') 831 is_add_workspace = True 832 self.deal_cargo_cmd(cargo_out) 833 # restore original Cargo.toml 834 if is_add_workspace: 835 with open(cargo_toml, 'w') as out_file: 836 out_file.writelines(cargo_toml_lines) 837 if not self.dry_run: 838 if not have_cargo_lock: # restore to no Cargo.lock state 839 if os.path.exists(cargo_lock): 840 os.remove(cargo_lock) 841 elif not self.args.use_cargo_lock: # restore saved Cargo.lock 842 os.rename(cargo_lock_save, cargo_lock) 843 os.environ['PATH'] = save_path 844 return self 845 846 def deal_cargo_cmd(self, cargo_out: str): 847 cargo_cmd_v_flag = ['-vv'] if self.args.vv else ['-v'] 848 cargo_cmd_target_dir = ['--target-dir', TARGET_TEMP] 849 cargo_cmd_redir = ['>>', cargo_out, '2>&1'] 850 for cargo in self.cargo: 851 cargo_cmd = [self.cargo_path] + cargo_cmd_v_flag 852 features = [] 853 if cargo != 'clean': 854 if self.args.features is not None: 855 features.append(' --no-default-features') 856 if self.args.features: 857 features += [' --features ', self.args.features] 858 cargo_cmd += [cargo] + features + cargo_cmd_target_dir + cargo_cmd_redir 859 if self.args.rustflags and cargo != 'clean': 860 cargo_cmd = ['RUSTFLAGS="' + self.args.rustflags] + cargo_cmd 861 self.run_cargo_cmd(cargo_cmd, cargo_out) 862 863 def run_cargo_cmd(self, cargo_cmd: str, cargo_out: str): 864 if self.dry_run: 865 print('Dry-run skip:', cargo_cmd) 866 else: 867 with open(cargo_out, 'a') as file: 868 file.write('### Running: ' + ''.join(cargo_cmd) + '\n') 869 ret = subprocess.run(cargo_cmd, shell=False) 870 if ret.returncode != 0: 871 print('ERROR: There was an error while running cargo.' + 872 ' See the cargo.out file for details.') 873 874 def generate_gn(self): 875 """Parse cargo.out and generate BUILD.gn files.""" 876 cargo_out = 'cargo.out' # The file name used to save cargo build -v output. 877 errors_line = 'Errors in ' + cargo_out + ':' 878 if self.dry_run: 879 print('Dry-run skip: read', cargo_out, 'write BUILD.gn') 880 elif os.path.exists(cargo_out): 881 self.find_root_pkg() 882 with open(cargo_out, 'r') as cargo_out: 883 self.parse(cargo_out, 'BUILD.gn') 884 self.crates.sort(key=get_crate_name) 885 for crate in self.crates: 886 crate.dump() 887 if self.error_infos: 888 self.append_to_gn('\n' + errors_line + '\n' + self.error_infos) 889 if self.test_error_infos: 890 self.append_to_gn('\n// Errors when listing tests:\n' + 891 self.test_error_infos) 892 return self 893 894 def find_root_pkg(self): 895 """Read name of [package] in ./Cargo.toml.""" 896 if os.path.exists('./Cargo.toml'): 897 return 898 with open('./Cargo.toml', 'r') as infile: 899 get_designated_pkg_info(infile, 'name') 900 901 def parse(self, infile: str, outfile_name: str): 902 """Parse rustc, test, and warning messages in infile, return a list of Crates.""" 903 # cargo test --list output of the start of running a binary. 904 cargo_test_list_start_re = re.compile('^\s*Running (.*) \(.*\)$') 905 # cargo test --list output of the end of running a binary. 906 cargo_test_list_end_re = re.compile('^(\d+) tests, (\d+) benchmarks$') 907 compiling_pat = re.compile('^ +Compiling (.*)$') 908 current_test_name = None 909 for line in infile: 910 # We read the file in two passes, where the first simply checks for empty tests. 911 # Otherwise we would add and merge tests before seeing they're empty. 912 if cargo_test_list_start_re.match(line): 913 current_test_name = cargo_test_list_start_re.match(line).group(1) 914 elif current_test_name and cargo_test_list_end_re.match(line): 915 match = cargo_test_list_end_re.match(line) 916 if int(match.group(1)) + int(match.group(2)) == 0: 917 self.add_empty_test(current_test_name) 918 current_test_name = None 919 #Get Compiling information 920 if compiling_pat.match(line): 921 self.add_deps_libname_map(compiling_pat.match(line).group(1)) 922 infile.seek(0) 923 self.parse_cargo_out(infile, outfile_name) 924 925 def add_empty_test(self, name: str): 926 if name == 'unittests': 927 self.empty_unittests = True 928 else: 929 self.empty_tests.add(name) 930 931 def add_deps_libname_map(self, line: str): 932 line_list = line.split() 933 if len(line_list) > 1: 934 self.deps_libname_map[pkg_to_crate_name(line_list[0])] = line_list[0] 935 936 def parse_cargo_out(self, infile: str, outfile_name: str): 937 # Cargo -v output of a call to rustc. 938 rustc_re = re.compile('^ +Running `rustc (.*)`$') 939 # Cargo -vv output of a call to rustc could be split into multiple lines. 940 # Assume that the first line will contain some CARGO_* env definition. 941 rustc_vv_re = re.compile('^ +Running `.*CARGO_.*=.*$') 942 # Rustc output of file location path pattern for a warning message. 943 warning_output_file_re = re.compile('^ *--> ([^:]*):[0-9]+') 944 cargo_to_gn_running_re = re.compile('^### Running: .*$') 945 line_num = 0 946 previous_warning = False # true if the previous line was warning 947 rustc_line = '' # previous line matching rustc_vv_re 948 in_tests = False 949 for line in infile: 950 line_num += 1 951 if line.startswith('warning: '): 952 previous_warning = True 953 rustc_line = self.assert_empty_rustc_line(rustc_line) 954 continue 955 new_rustc_line = '' 956 if rustc_re.match(line): 957 args_line = rustc_re.match(line).group(1) 958 self.add_crate(Crate(self, outfile_name).parse_rustc(line_num, args_line)) 959 self.assert_empty_rustc_line(rustc_line) 960 elif rustc_line or rustc_vv_re.match(line): 961 new_rustc_line = self.deal_rustc_command( 962 line_num, rustc_line, line, outfile_name) 963 elif previous_warning and warning_output_file_re.match(line): 964 file_path = warning_output_file_re.match(line).group(1) 965 if file_path[0] != '/': # ignore absolute path 966 self.warning_files.add(file_path) 967 self.assert_empty_rustc_line(rustc_line) 968 elif line.startswith('error: ') or line.startswith('error[E'): 969 if not self.args.ignore_cargo_errors: 970 self.add_error_infos(in_tests, line) 971 elif cargo_to_gn_running_re.match(line): 972 in_tests = "cargo test" in line and "--list" in line 973 previous_warning = False 974 rustc_line = new_rustc_line 975 976 def assert_empty_rustc_line(self, line: str): 977 # report error if line is not empty 978 if line: 979 self.append_to_gn('ERROR -vv line: ' + line) 980 return '' 981 982 def append_to_gn(self, line: str): 983 self.init_gn_file('BUILD.gn') 984 with open('BUILD.gn', 'a') as outfile: 985 outfile.write(line) 986 print(line) 987 988 def init_gn_file(self, name: str): 989 # name could be BUILD.gn or sub_dir_path/BUILD.gn 990 if name in self.gn_files: 991 return 992 self.gn_files.add(name) 993 if os.path.exists(name): 994 os.remove(name) 995 with open(name, 'w') as outfile: 996 outfile.write(BUILD_GN_HEADER) 997 outfile.write('\n') 998 outfile.write('import("%s")\n' % IMPORT_CONTENT) 999 1000 def add_error_infos(self, in_tests: str, line: str): 1001 if in_tests: 1002 self.test_error_infos += '// '.join(line) 1003 else: 1004 self.error_infos += line 1005 1006 def deal_rustc_command(self, line_num: str, rustc_line: str, line: str, outfile_name: str): 1007 """Process a rustc command line from cargo -vv output.""" 1008 # cargo build -vv output can have multiple lines for a rustc command due to '\n' in strings 1009 # for environment variables. strip removes leading spaces and '\n' at the end 1010 new_rustc_line = (rustc_line.strip() + line) if rustc_line else line 1011 # The combined -vv output rustc command line pattern. 1012 rustc_vv_cmd_args = re.compile('^ *Running `.*CARGO_.*=.* rustc (.*)`$') 1013 if not line.endswith('`\n') or (new_rustc_line.count('`') % 2) != 0: 1014 return new_rustc_line 1015 if rustc_vv_cmd_args.match(new_rustc_line): 1016 args = rustc_vv_cmd_args.match(new_rustc_line).group(1) 1017 self.add_crate(Crate(self, outfile_name).parse_rustc(line_num, args)) 1018 else: 1019 self.assert_empty_rustc_line(new_rustc_line) 1020 return '' 1021 1022 def add_crate(self, new_crate): 1023 """Merge crate with someone in crates, or append to it. Return crates.""" 1024 if (is_build_script(new_crate.crate_name) and 1025 not is_dependent_path(new_crate.main_src) and 1026 new_crate.root_pkg_name and len(new_crate.deps) > 0): 1027 self.build_deps[new_crate.root_pkg_name] = new_crate.deps 1028 if new_crate.skip_crate(): 1029 # include debug info of all crates 1030 if self.args.debug: 1031 self.crates.append(new_crate) 1032 else: 1033 for crate in self.crates: 1034 if crate.merge_crate(new_crate, 'BUILD.gn'): 1035 return 1036 # If not merged, decide module type and name now. 1037 new_crate.decide_module_type() 1038 self.crates.append(new_crate) 1039 1040 def should_ignore_test(self, src): 1041 # cargo test outputs the source file for integration tests but "unittests" for unit tests. 1042 # To figure out to which crate this corresponds, we check if the current source file is 1043 # the main source of a non-test crate, e.g., a library or a binary. 1044 return (src in self.empty_tests or src in self.args.test_blocklist or 1045 (self.empty_unittests and 1046 src in [c.main_src for c in self.crates if c.crate_types != ['test']])) 1047 1048 1049def get_arg_parser(): 1050 """Parse main arguments.""" 1051 argparser = argparse.ArgumentParser('cargo2gn') 1052 argparser.add_argument('--add-workspace', action='store_true', default=False, 1053 help=('append [workspace] to Cargo.toml before calling cargo, to treat' + 1054 ' current directory as root of package source; otherwise the relative' + 1055 ' source file path in generated .gn file will be from the parent directory.')) 1056 argparser.add_argument('--cargo', action='append', metavar='args_string', 1057 help=('extra cargo build -v args in a string, ' + 1058 'each --cargo flag calls cargo build -v once')) 1059 argparser.add_argument('--cargo-bin', type=str, 1060 help='use cargo in the cargo_bin directory instead of the prebuilt one') 1061 argparser.add_argument('--config', type=str, 1062 help=('Load command-line options from the given config file. ' + 1063 'Options in this file will override those passed on the command line.')) 1064 argparser.add_argument('--copy-out', action='store_true', default=False, 1065 help=('only for root directory, copy build.rs output to ./out/* and ' + 1066 'add a genrule to copy ./out/*.')) 1067 argparser.add_argument('--debug', action='store_true', default=False, 1068 help='dump debug info into BUILD.gn') 1069 argparser.add_argument('--dependency-blocklist', nargs='*', default=[], 1070 help='Do not emit the given dependencies (without lib prefixes).') 1071 argparser.add_argument('--features', type=str, 1072 help=('pass features to cargo build, ' + 1073 'empty string means no default features')) 1074 argparser.add_argument('--ignore-cargo-errors', action='store_true', default=False, 1075 help='do not append cargo/rustc error messages to BUILD.gn') 1076 argparser.add_argument('--no-pkg-info', action='store_true', default=False, 1077 help='Do not attempt to determine the package info automatically.') 1078 argparser.add_argument('--no-subdir', action='store_true', default=False, 1079 help='do not output anything for sub-directories') 1080 argparser.add_argument('--one-file', action='store_true', default=False, 1081 help=('output all into one BUILD.gn, default will generate one BUILD.gn ' + 1082 'per Cargo.toml in subdirectories')) 1083 argparser.add_argument('--run', action='store_true', default=False, 1084 help='run it, default is dry-run') 1085 argparser.add_argument('--rustflags', type=str, help='passing flags to rustc') 1086 argparser.add_argument('--skipcargo', action='store_true', default=False, 1087 help='skip cargo command, parse cargo.out, and generate BUILD.gn') 1088 argparser.add_argument('--test-blocklist', nargs='*', default=[], 1089 help=('Do not emit the given tests. ' + 1090 'Pass the path to the test file to exclude.')) 1091 argparser.add_argument('--use-cargo-lock', action='store_true', default=False, 1092 help=('run cargo build with existing Cargo.lock ' + 1093 '(used when some latest dependent crates failed)')) 1094 argparser.add_argument('--vv', action='store_true', default=False, 1095 help='run cargo with -vv instead of default -v') 1096 return argparser 1097 1098 1099def get_parse_args(argparser): 1100 """Parses command-line options.""" 1101 args = argparser.parse_args() 1102 # Use the values specified in a config file if one was found. 1103 if args.config: 1104 with open(args.config, 'r') as file: 1105 config_data = json.load(file) 1106 args_dict = vars(args) 1107 for arg in config_data: 1108 args_dict[arg.replace('-', '_')] = config_data[arg] 1109 return args 1110 1111 1112def main(): 1113 argparser = get_arg_parser() 1114 args = get_parse_args(argparser) 1115 if not args.run: # default is dry-run 1116 print(DRY_RUN_CONTENT) 1117 Runner(args).run_cargo().generate_gn() 1118 1119 1120if __name__ == '__main__': 1121 main() 1122