1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2023 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16 17from __future__ import print_function 18 19import os 20import os.path 21import sys 22import argparse 23import glob 24import json 25import re 26import shutil 27 28# Rust path 29RUST_PATH = '//third_party/rust/' 30 31# import content added to all generated BUILD.gn files. 32IMPORT_CONTENT = '//build/templates/rust/ohos.gni' 33 34# The name of the temporary output directory. 35TARGET_TEMP = 'target_temp' 36 37# Header added to all generated BUILD.gn files. 38BUILD_GN_HEADER = ( 39 '# Copyright (c) 2023 Huawei Device Co., Ltd.\n' + 40 '# Licensed under the Apache License, Version 2.0 (the "License");\n' + 41 '# you may not use this file except in compliance with the License.\n' + 42 '# You may obtain a copy of the License at\n' + 43 '#\n' + 44 '# http://www.apache.org/licenses/LICENSE-2.0\n' + 45 '#\n' + 46 '# Unless required by applicable law or agreed to in writing, software\n' + 47 '# distributed under the License is distributed on an "AS IS" BASIS,\n' + 48 '# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' + 49 '# See the License for the specific language governing permissions and\n' + 50 '# limitations under the License.\n') 51 52# Message to be displayed when this script is called without the --run flag. 53DRY_RUN_CONTENT = ( 54 'Dry-run: This script uses ./' + TARGET_TEMP + ' for output directory,\n' + 55 'runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n' + 56 'and writes to BUILD.gn in the current and subdirectories.\n\n' + 57 'To do do all of the above, use the --run flag.\n' + 58 'See --help for other flags, and more usage notes in this script.\n') 59 60# Rust package name with suffix -d1.d2.d3(+.*)?. 61VERSION_SUFFIX_RE = re.compile(r'^(.*)-[0-9]+\.[0-9]+\.[0-9]+(?:\+.*)?$') 62 63# Crate types corresponding to a library 64LIBRARY_CRATE_TYPES = ['staticlib', 'cdylib', 'lib', 'rlib', 'dylib', 'proc-macro'] 65 66 67def escape_quotes(s): 68 # replace '"' with '\\"' 69 return s.replace('"', '\\"') 70 71 72def file_base_name(path): 73 return os.path.splitext(os.path.basename(path))[0] 74 75 76def pkg_to_crate_name(s): 77 return s.replace('-', '_').replace('.', '_') 78 79 80def get_base_name(path): 81 return pkg_to_crate_name(file_base_name(path)) 82 83 84def get_crate_name(crate): 85 # to sort crates in a list 86 return crate.crate_name 87 88 89def get_designated_pkg_info(lines, designated): 90 package = re.compile(r'^ *\[package\]') 91 designated_re = re.compile('^ *' + designated + ' *= * "([^"]*)') 92 is_package = False 93 for line in lines: 94 if is_package: 95 if designated_re.match(line): 96 line = eval(repr(line).replace(f'\\"', '')) 97 return designated_re.match(line).group(1) 98 else: 99 is_package = package.match(line) is not None 100 return '' 101 102 103def is_build_script(name): 104 # Judge whether it is build script. 105 return name.startswith('build_script_') 106 107 108def is_dependent_path(path): 109 # Absolute('/') or dependent('.../') paths are not main files of this crate. 110 return path.startswith('/') or path.startswith('.../') 111 112 113def unquote(s): 114 # remove quotes around str 115 if s and len(s) > 1 and s[0] == '"' and s[-1] == '"': 116 return s[1:-1] 117 return s 118 119 120def remove_version_suffix(s): 121 # remove -d1.d2.d3 suffix 122 if VERSION_SUFFIX_RE.match(s): 123 return VERSION_SUFFIX_RE.match(s).group(1) 124 return s 125 126 127def short_out_name(pkg, s): 128 # replace /.../pkg-*/out/* with .../out/* 129 return re.sub('^/.*/' + pkg + '-[0-9a-f]*/out/', '.../out/', s) 130 131 132class Crate(object): 133 """Information of a Rust crate to collect/emit for an BUILD.gn module.""" 134 135 def __init__(self, runner, outfile_name): 136 # Remembered global runner 137 self.runner = runner 138 self.debug = runner.args.debug 139 self.cargo_dir = '' # directory of my Cargo.toml 140 self.outfile = None # open file handle of outfile_name during dump* 141 self.outfile_name = outfile_name # path to BUILD.gn 142 # GN module properties derived from rustc parameters. 143 self.module_type = '' # lib,crate_name,test etc. 144 self.root_pkg_name = '' # parent package name of a sub/test packge 145 # Save parsed status 146 self.error_infos = '' # all errors found during parsing 147 self.line = '' # original rustc command line parameters 148 self.line_num = 1 # runner told input source line number 149 # Parameters collected from rustc command line. 150 self.cap_lints = '' 151 self.crate_name = '' 152 self.edition = '2015' # cargo default is 2015, you can specify the edition as 2018 or 2021 153 self.emit_list = '' # --emit=dep-info,metadata,link 154 self.main_src = '' 155 self.target = '' 156 self.cfgs = list() 157 self.core_deps = list() # first part of self.deps elements 158 self.crate_types = list() 159 self.deps = list() 160 self.features = list() 161 self.ignore_options = list() 162 self.srcs = list() # main_src or merged multiple source files 163 self.shared_libs = list() # -l dylib=wayland-client, -l z 164 self.static_libs = list() # -l static=host_cpuid 165 # Parameters collected from Cargo.toml. 166 self.cargo_pkg_version = '' # value extracted from Cargo.toml version field 167 self.cargo_pkg_authors = '' # value extracted from Cargo.toml authors field 168 self.cargo_pkg_name = '' # value extracted from Cargo.toml name field 169 self.cargo_pkg_description = '' # value extracted from Cargo.toml description field 170 # Parameters related to build.rs. 171 self.build_root = '' 172 self.checked_out_files = False # to check only once 173 self.build_script_outputs = [] # output files generated by build.rs 174 175 def write(self, s): 176 # convenient way to output one line at a time with EOL. 177 self.outfile.write(s + '\n') 178 179 def parse_rustc(self, line_num, line): 180 """Find important rustc arguments to convert to BUILD.gn properties.""" 181 self.line_num = line_num 182 self.line = line 183 args = line.split() # Loop through every argument of rustc. 184 self.parse_args(args) 185 if not self.crate_name: 186 self.error_infos += 'ERROR: missing --crate-name\n' 187 if not self.crate_types: 188 if 'test' in self.cfgs: 189 self.crate_types.append('test') 190 else: 191 self.error_infos += 'ERROR: missing --crate-type or --test\n' 192 elif len(self.crate_types) > 1: 193 if 'lib' in self.crate_types and 'rlib' in self.crate_types: 194 self.error_infos += 'ERROR: cannot generate both lib and rlib crate types\n' 195 if 'test' in self.crate_types: 196 self.error_infos += 'ERROR: cannot handle both --crate-type and --test\n' 197 if not self.main_src: 198 self.error_infos += 'ERROR: missing main source file\n' 199 else: 200 self.srcs.append(self.main_src) 201 if self.cargo_dir: 202 self.get_root_pkg_name() 203 if not self.root_pkg_name: 204 self.root_pkg_name = self.crate_name 205 206 # Process crate with build.rs 207 if not self.skip_crate(): 208 if not self.runner.args.no_pkg_info: 209 self.find_pkg_info() 210 self.find_build_root() 211 if self.runner.args.copy_out: 212 self.copy_out_files() 213 elif self.find_out_files() and self.has_used_out_dir(): 214 self.copy_out_files() 215 216 self.cfgs = sorted(set(self.cfgs)) 217 self.core_deps = sorted(set(self.core_deps)) 218 self.crate_types = sorted(set(self.crate_types)) 219 self.deps = sorted(set(self.deps)) 220 self.features = sorted(set(self.features)) 221 self.ignore_options = sorted(set(self.ignore_options)) 222 self.static_libs = sorted(set(self.static_libs)) 223 self.shared_libs = sorted(set(self.shared_libs)) 224 self.decide_module_type() 225 return self 226 227 def parse_args(self, args): 228 num = 0 229 while num < len(args): 230 arg = args[num] 231 if arg == '--crate-name': 232 num += 1 233 self.crate_name = args[num] 234 elif arg == '--crate-type': 235 num += 1 236 self.crate_types.append(args[num]) 237 elif arg == '--cfg': 238 num += 1 239 self.deal_cfg(args[num]) 240 elif arg == '-C': 241 num += 1 242 self.add_ignore_options_flag(args[num]) # codegen options 243 elif arg.startswith('-C'): 244 self.add_ignore_options_flag(arg[2:]) 245 elif arg == '--cap-lints': 246 num += 1 247 self.cap_lints = args[num] 248 elif arg.startswith('--edition='): 249 self.edition = arg.replace('--edition=', '') 250 elif arg.startswith('--emit='): 251 self.emit_list = arg.replace('--emit=', '') 252 elif arg == '--extern': 253 num += 1 254 self.deal_extern(args[num]) 255 elif (arg.startswith('--error-format=') or arg.startswith('--json=') or 256 arg.startswith('\'-Aclippy')): 257 _ = arg # ignored 258 elif arg == '-L': 259 num += 1 260 self.set_root_pkg_name(args[num]) 261 elif arg == '-l': 262 num += 1 263 self.deal_static_and_dylib(args[num]) 264 elif arg == '--out-dir' or arg == '--color': # ignored 265 num += 1 266 elif arg == '--target': 267 num += 1 268 self.target = args[num] 269 elif arg == '--test': 270 self.crate_types.append('test') 271 elif not arg.startswith('-'): 272 self.set_main_src(args[num]) 273 else: 274 self.error_infos += 'ERROR: unknown ' + arg + '\n' 275 num += 1 276 277 def deal_cfg(self, arg): 278 if arg.startswith('\'feature='): 279 feature = unquote(arg.replace('\'feature=', '')[:-1]) 280 # 'runtime' feature removed because it conflicts with static 281 if feature == 'runtime': 282 feature = 'static' 283 self.features.append(feature) 284 else: 285 self.cfgs.append(arg) 286 287 def add_ignore_options_flag(self, flag): 288 """Ignore options not used in GN.""" 289 # 'codegen-units' is set in GN global config or by default 290 # 'embed-bitcode' is ignored; we might control LTO with other .gn flag 291 # 'prefer-dynamic' does not work with common flag -C lto 292 if not (flag.startswith('codegen-units=') or flag.startswith('debuginfo=') or 293 flag.startswith('embed-bitcode=') or flag.startswith('extra-filename=') or 294 flag.startswith('incremental=') or flag.startswith('metadata=') or 295 flag == 'prefer-dynamic'): 296 self.ignore_options.append(flag) 297 298 def deal_extern(self, arg): 299 deps = re.sub('=/[^ ]*/deps/', ' = ', arg) 300 self.deps.append(deps) 301 self.core_deps.append(re.sub(' = .*', '', deps)) 302 303 def set_root_pkg_name(self, arg): 304 if arg.startswith('dependency=') and arg.endswith('/deps'): 305 if '/' + TARGET_TEMP + '/' in arg: 306 self.root_pkg_name = re.sub('^.*/', '', 307 re.sub('/' + TARGET_TEMP + '/.*/deps$', '', arg)) 308 else: 309 self.root_pkg_name = re.sub('^.*/', '', 310 re.sub('/[^/]+/[^/]+/deps$', '', arg)) 311 self.root_pkg_name = remove_version_suffix(self.root_pkg_name) 312 313 def deal_static_and_dylib(self, arg): 314 if arg.startswith('static='): 315 self.static_libs.append(re.sub('static=', '', arg)) 316 elif arg.startswith('dylib='): 317 self.shared_libs.append(re.sub('dylib=', '', arg)) 318 else: 319 self.shared_libs.append(arg) 320 321 def set_main_src(self, arg): 322 self.main_src = re.sub(r'^/[^ ]*/registry/src/', '.../', arg) 323 self.main_src = re.sub(r'^\.\.\./github.com-[0-9a-f]*/', '.../', self.main_src) 324 self.find_cargo_dir() 325 if self.cargo_dir: 326 if self.runner.args.no_subdir: 327 # all .gn content to /dev/null 328 self.outfile_name = '/dev/null' 329 elif not self.runner.args.one_file: 330 # Use Cargo.toml to write BUILD.gn in the subdirectory. 331 self.outfile_name = os.path.join(self.cargo_dir, 'BUILD.gn') 332 self.main_src = self.main_src[len(self.cargo_dir) + 1:] 333 334 def find_cargo_dir(self): 335 """Deepest directory with Cargo.toml and contains the main_src.""" 336 if not is_dependent_path(self.main_src): 337 dir_name = os.path.dirname(self.main_src) 338 while dir_name: 339 if dir_name.endswith('.'): 340 dir_name = os.path.dirname(dir_name) 341 continue 342 if os.path.exists(os.path.join(dir_name, 'Cargo.toml')): 343 self.cargo_dir = dir_name 344 return 345 dir_name = os.path.dirname(dir_name) 346 347 def skip_crate(self): 348 """Return crate_name or a message if this crate should be skipped.""" 349 # Some Rust packages include extra unwanted crates. 350 # This set contains all such excluded crate names. 351 excluded_crates = set(['protobuf_bin_gen_rust_do_not_use']) 352 if (is_build_script(self.crate_name) or 353 self.crate_name in excluded_crates): 354 return self.crate_name 355 if is_dependent_path(self.main_src): 356 return 'dependent crate' 357 return '' 358 359 def get_root_pkg_name(self): 360 """Read name of [package] in ./Cargo.toml.""" 361 cargo_toml_path = './Cargo.toml' 362 if self.cargo_dir: 363 cargo_toml_path = os.path.join( 364 os.path.join('.', self.cargo_dir), 'Cargo.toml') 365 if not os.path.exists(cargo_toml_path): 366 return 367 with open(cargo_toml_path, 'r') as infile: 368 self.root_pkg_name = get_designated_pkg_info(infile, 'name') 369 return 370 371 def find_pkg_info(self): 372 """Read package info of [package] in ./Cargo.toml.""" 373 cargo_toml_path = './Cargo.toml' 374 if self.cargo_dir: 375 cargo_toml_path = os.path.join( 376 os.path.join('.', self.cargo_dir), 'Cargo.toml') 377 if not os.path.exists(cargo_toml_path): 378 return 379 with open(cargo_toml_path, 'r') as infile: 380 if self.root_pkg_name: 381 self.cargo_pkg_name = self.root_pkg_name 382 else: 383 self.cargo_pkg_name = get_designated_pkg_info(infile, 'name') 384 infile.seek(0) 385 self.cargo_pkg_version = get_designated_pkg_info(infile, 'version') 386 infile.seek(0) 387 pkg_description = get_designated_pkg_info(infile, 'description') 388 pkg_description = pkg_description.replace('\n', '').replace(r'\n', '').strip() 389 self.cargo_pkg_description = pkg_description 390 infile.seek(0) 391 authors_re = re.compile(' *authors *= * \[(.*?)\]', re.S) 392 authors_section = authors_re.search(infile.read()) 393 if authors_section: 394 authors = authors_section.group(1) 395 authors = authors.replace('\n', '').replace(' ', ' ').replace('"', '').strip() 396 if authors.endswith(','): 397 authors = authors[:-1] 398 self.cargo_pkg_authors = authors 399 400 def find_build_root(self): 401 """Read build of [package] in ./Cargo.toml.""" 402 cargo_toml_path = './Cargo.toml' 403 if self.cargo_dir: 404 cargo_toml_path = os.path.join( 405 os.path.join('.', self.cargo_dir), 'Cargo.toml') 406 if not os.path.exists(cargo_toml_path): 407 return 408 with open(cargo_toml_path, 'r') as infile: 409 self.build_root = get_designated_pkg_info(infile, 'build') 410 if not self.build_root: 411 build_rs_path = './build.rs' 412 if self.cargo_dir: 413 build_rs_path = os.path.join(os.path.join('.', self.cargo_dir), 'build.rs') 414 if os.path.exists(build_rs_path): 415 self.build_root = 'build.rs' 416 417 def find_out_files(self): 418 # normal_output_list has build.rs output for normal crates 419 normal_output_list = glob.glob( 420 TARGET_TEMP + '/*/*/build/' + self.root_pkg_name + '-*/out/*') 421 # other_output_list has build.rs output for proc-macro crates 422 other_output_list = glob.glob( 423 TARGET_TEMP + '/*/build/' + self.root_pkg_name + '-*/out/*') 424 return normal_output_list + other_output_list 425 426 def has_used_out_dir(self): 427 """Returns true if env!("OUT_DIR") is found.""" 428 cmd = 'grep -rl --exclude build.rs --include \\*.rs \'env!("OUT_DIR")\' * > /dev/null' 429 if self.cargo_dir: 430 cmd = 'grep -rl --exclude ' 431 cmd += os.path.join(self.cargo_dir, 'build.rs') 432 cmd += ' --include \\*.rs \'env!("OUT_DIR")\' * > /dev/null' 433 return 0 == os.system(cmd) 434 435 def copy_out_files(self): 436 """Copy build.rs output files to ./out and set up build_script_outputs.""" 437 if self.checked_out_files: 438 return 439 self.checked_out_files = True 440 cargo_out_files = self.find_out_files() 441 out_files = set() 442 out_path = 'out' 443 if self.cargo_dir: 444 out_path = os.path.join(self.cargo_dir, out_path) 445 if cargo_out_files: 446 os.makedirs(out_path, exist_ok=True) 447 for path in cargo_out_files: 448 file_name = path.split('/')[-1] 449 out_files.add(file_name) 450 self.build_script_outputs = sorted(out_files) 451 452 def decide_module_type(self): 453 # Use the first crate type for the default/first module. 454 crate_type = self.crate_types[0] if self.crate_types else '' 455 self.decide_one_module_type(crate_type) 456 457 def decide_one_module_type(self, crate_type): 458 """Decide which GN module type to use.""" 459 if crate_type == 'bin': 460 self.module_type = self.crate_name 461 elif crate_type in LIBRARY_CRATE_TYPES: 462 self.module_type = 'lib' 463 elif crate_type == 'test': 464 self.module_type = 'test' 465 else: 466 self.module_type = '' 467 468 def merge_crate(self, other, outfile_name): 469 """Try to merge crate into self.""" 470 # Cargo build --tests could recompile a library for tests. 471 # We need to merge such duplicated calls to rustc, with the 472 # algorithm in is_should_merge. 473 should_merge = self.is_should_merge(other) 474 should_merge_test = False 475 if not should_merge: 476 should_merge_test = self.merge_test(other) 477 if should_merge or should_merge_test: 478 self.runner.init_gn_file(outfile_name) 479 # to write debug info 480 with open(outfile_name, 'a') as outfile: 481 self.outfile = outfile 482 other.outfile = outfile 483 self.execute_merge(other, should_merge_test) 484 return True 485 return False 486 487 def is_should_merge(self, other): 488 return (self.crate_name == other.crate_name and 489 self.crate_types == other.crate_types and 490 self.main_src == other.main_src and 491 self.root_pkg_name == other.root_pkg_name and 492 not self.skip_crate() and self.is_same_flags(other)) 493 494 def merge_test(self, other): 495 """Returns true if self and other are tests of same root_pkg_name.""" 496 # Before merger, each test has its own crate_name. A merged test uses 497 # its source file base name as output file name, so a test is mergeable 498 # only if its base name equals to its crate name. 499 return (self.crate_types == other.crate_types and self.crate_types == ['test'] and 500 self.root_pkg_name == other.root_pkg_name and not self.skip_crate() and 501 other.crate_name == get_base_name(other.main_src) and 502 (len(self.srcs) > 1 or (self.crate_name == get_base_name(self.main_src))) and 503 self.is_same_flags(other)) 504 505 def is_same_flags(self, other): 506 return (not self.error_infos and not other.error_infos and 507 self.cap_lints == other.cap_lints and self.cfgs == other.cfgs and 508 self.core_deps == other.core_deps and self.edition == other.edition and 509 self.emit_list == other.emit_list and self.features == other.features and 510 self.ignore_options == other.ignore_options and 511 self.static_libs == other.static_libs and 512 self.shared_libs == other.shared_libs) 513 514 def execute_merge(self, other, should_merge_test): 515 """Merge attributes of other to self.""" 516 if self.debug: 517 self.write('\n// Before merge definition(self):') 518 self.dump_debug_info() 519 self.write('\n// Before merge definition(other):') 520 other.dump_debug_info() 521 if not self.target: 522 # okay to keep only the first target triple 523 self.target = other.target 524 self.decide_module_type() 525 if should_merge_test: 526 if (self.runner.should_ignore_test(self.main_src) and 527 not self.runner.should_ignore_test(other.main_src)): 528 self.main_src = other.main_src 529 self.srcs.append(other.main_src) 530 self.crate_name = pkg_to_crate_name(self.root_pkg_name) 531 if self.debug: 532 self.write('\n// After merge definition:') 533 self.dump_debug_info() 534 535 def dump(self): 536 """Dump all error/debug/module code to the output .gn file.""" 537 self.runner.init_gn_file(self.outfile_name) 538 with open(self.outfile_name, 'a') as outfile: 539 self.outfile = outfile 540 if self.error_infos: 541 self.dump_line() 542 self.write(self.error_infos) 543 elif self.skip_crate(): 544 self.dump_skip_crate(self.skip_crate()) 545 else: 546 if self.debug: 547 self.dump_debug_info() 548 self.dump_gn_module() 549 550 def dump_debug_info(self): 551 """Dump parsed data, when cargo2gn is called with --debug.""" 552 553 def dump(name, value): 554 self.write('//%12s = %s' % (name, value)) 555 556 def dump_list(fmt, values): 557 for v in values: 558 self.write(fmt % v) 559 560 def opt_dump(name, value): 561 if value: 562 dump(name, value) 563 564 self.dump_line() 565 dump('crate_name', self.crate_name) 566 dump('crate_types', self.crate_types) 567 opt_dump('edition', self.edition) 568 opt_dump('emit_list', self.emit_list) 569 dump('main_src', self.main_src) 570 dump('module_type', self.module_type) 571 opt_dump('target', self.target) 572 opt_dump('cap_lints', self.cap_lints) 573 dump_list('// cfg = %s', self.cfgs) 574 dump_list('// cfg = \'feature "%s"\'', self.features) 575 dump_list('// codegen = %s', self.ignore_options) 576 dump_list('// deps = %s', self.deps) 577 dump_list('// -l (dylib) = %s', self.shared_libs) 578 dump_list('// -l static = %s', self.static_libs) 579 580 def dump_line(self): 581 self.write('\n// Line ' + str(self.line_num) + ' ' + self.line) 582 583 def dump_skip_crate(self, kind): 584 if self.debug: 585 self.write('\n// IGNORED: ' + kind + ' ' + self.main_src) 586 return self 587 588 def dump_gn_module(self): 589 """Dump one or more GN module definition, depending on crate_types.""" 590 if len(self.crate_types) == 1: 591 self.dump_single_type_gn_module() 592 return 593 if 'test' in self.crate_types: 594 self.write('\nERROR: multiple crate types cannot include test type') 595 return 596 # Dump one GN module per crate_type. 597 for crate_type in self.crate_types: 598 self.decide_one_module_type(crate_type) 599 self.dump_one_gn_module(crate_type) 600 601 def dump_single_type_gn_module(self): 602 """Dump one simple GN module, which has only one crate_type.""" 603 crate_type = self.crate_types[0] 604 if crate_type != 'test': 605 self.dump_one_gn_module(crate_type) 606 return 607 # Dump one test module per source file. 608 self.srcs = [ 609 src for src in self.srcs if not self.runner.should_ignore_test(src)] 610 if len(self.srcs) > 1: 611 self.srcs = sorted(set(self.srcs)) 612 saved_srcs = self.srcs 613 for src in saved_srcs: 614 self.srcs = [src] 615 saved_main_src = self.main_src 616 self.main_src = src 617 self.decide_one_module_type(crate_type) 618 self.dump_one_gn_module(crate_type) 619 self.main_src = saved_main_src 620 self.srcs = saved_srcs 621 622 def dump_one_gn_module(self, crate_type): 623 """Dump one GN module definition.""" 624 if not self.module_type: 625 self.write('\nERROR: unknown crate_type ' + crate_type) 626 return 627 self.write('\nohos_cargo_crate("' + self.module_type + '") {') 628 self.dump_gn_first_properties(crate_type) 629 self.dump_gn_core_properties() 630 self.write('}') 631 632 def dump_gn_first_properties(self, crate_type): 633 if crate_type != 'bin': 634 self.write(' crate_name = "' + self.crate_name + '"') 635 if crate_type: 636 if crate_type == 'lib': 637 crate_type = 'rlib' 638 self.write(' crate_type = "' + crate_type + '"') 639 if self.main_src: 640 self.write(' crate_root = "' + self.main_src + '"') 641 if self.crate_name.startswith('lib'): 642 self.write(' output_name = "lib' + self.crate_name + '"') 643 self.write('') 644 645 def dump_gn_core_properties(self): 646 self.dump_sources_list() 647 if self.edition: 648 self.write(' edition = "' + self.edition + '"') 649 if not self.runner.args.no_pkg_info: 650 if self.cargo_pkg_version: 651 self.write(' cargo_pkg_version = "' + 652 self.cargo_pkg_version + '"') 653 if self.cargo_pkg_authors: 654 self.write(' cargo_pkg_authors = "' + 655 self.cargo_pkg_authors + '"') 656 if self.cargo_pkg_name: 657 self.write(' cargo_pkg_name = "' + 658 self.cargo_pkg_name + '"') 659 if self.cargo_pkg_description: 660 self.write(' cargo_pkg_description = "' + 661 self.cargo_pkg_description + '"') 662 if self.deps: 663 self.dump_gn_deps() 664 if self.build_root and self.root_pkg_name in self.runner.build_deps: 665 self.dump_gn_build_deps() 666 self.dump_gn_property_list('features', '"%s"', self.features) 667 if self.build_root: 668 self.write(' build_root = "' + self.build_root + '"') 669 build_sources = list() 670 build_sources.append(self.build_root) 671 self.dump_gn_property_list('build_sources', '"%s"', build_sources) 672 if self.build_script_outputs: 673 self.dump_gn_property_list( 674 'build_script_outputs', '"%s"', self.build_script_outputs) 675 676 def dump_sources_list(self): 677 """Dump the srcs list, for defaults or regular modules.""" 678 if len(self.srcs) > 1: 679 srcs = sorted(set(self.srcs)) # make a copy and dedup 680 for num in range(len(self.srcs)): 681 srcs[num] = srcs[num] 682 else: 683 srcs = [self.main_src] 684 self.dump_gn_property_list('sources', '"%s"', srcs) 685 686 def dump_gn_deps(self): 687 """Dump the deps.""" 688 rust_deps = list() 689 deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$') 690 for lib in self.deps: 691 libname_groups = deps_libname.match(lib) 692 if libname_groups is not None: 693 lib_name = libname_groups.group(1) 694 else: 695 lib_name = re.sub(' .*$', '', lib) 696 if lib_name in self.runner.args.dependency_blocklist: 697 continue 698 if lib.endswith('.rlib') or lib.endswith('.rmeta') or lib.endswith('.so'): 699 # On MacOS .rmeta is used when Linux uses .rlib or .rmeta. 700 rust_lib = self.get_rust_lib(lib_name) 701 if rust_lib: 702 rust_lib += ':lib' 703 rust_deps.append(rust_lib) 704 elif lib != 'proc_macro': 705 # --extern proc_macro is special and ignored 706 rust_deps.append('// unknown type of lib: '.join(lib)) 707 if rust_deps: 708 self.dump_gn_property_list('deps', '"%s"', rust_deps) 709 710 def dump_gn_build_deps(self): 711 """Dump the build deps.""" 712 rust_build_deps = list() 713 build_deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$') 714 build_deps = self.runner.build_deps.get(self.root_pkg_name) 715 if not build_deps: 716 return 717 for lib in build_deps: 718 libname_groups = build_deps_libname.match(lib) 719 if libname_groups is not None: 720 lib_name = libname_groups.group(1) 721 else: 722 lib_name = re.sub(' .*$', '', lib) 723 if lib_name in self.runner.args.dependency_blocklist: 724 continue 725 if lib.endswith('.rlib') or lib.endswith('.rmeta') or lib.endswith('.so'): 726 # On MacOS .rmeta is used when Linux uses .rlib or .rmeta. 727 rust_lib = self.get_rust_lib(lib_name) 728 if rust_lib: 729 rust_build_deps.append(rust_lib + ':lib') 730 elif lib != 'proc_macro': 731 # --extern proc_macro is special and ignored 732 rust_build_deps.append('// unknown type of lib: '.join(lib)) 733 if rust_build_deps: 734 self.dump_gn_property_list('build_deps', '"%s"', rust_build_deps) 735 736 def dump_gn_property_list(self, name, fmt, values): 737 if not values: 738 return 739 if len(values) > 1: 740 self.write(' ' + name + ' = [') 741 self.dump_gn_property_list_items(fmt, values) 742 self.write(' ]') 743 else: 744 self.write(' ' + name + ' = [' + 745 (fmt % escape_quotes(values[0])) + ']') 746 747 def dump_gn_property_list_items(self, fmt, values): 748 for v in values: 749 # fmt has quotes, so we need escape_quotes(v) 750 self.write(' ' + (fmt % escape_quotes(v)) + ',') 751 752 def get_rust_lib(self, lib_name): 753 rust_lib = '' 754 if lib_name: 755 crate_name = pkg_to_crate_name(lib_name) 756 deps_libname = self.runner.deps_libname_map.get(crate_name) 757 if deps_libname: 758 rust_lib = RUST_PATH + deps_libname 759 return rust_lib 760 761 762class Runner(object): 763 """Main class to parse cargo -v output""" 764 765 def __init__(self, args): 766 self.gn_files = set() # Remember all output BUILD.gn files. 767 self.root_pkg_name = '' # name of package in ./Cargo.toml 768 self.args = args 769 self.dry_run = not args.run 770 self.skip_cargo = args.skipcargo 771 self.cargo_path = './cargo' # path to cargo 772 self.crates = list() # all crates 773 self.error_infos = '' # all error infos 774 self.test_error_infos = '' # all test error infos 775 self.warning_files = set() # all warning files 776 self.set_cargo_path() 777 # Default operation is cargo clean, followed by build or user given operation. 778 if args.cargo: 779 self.cargo = ['clean'] + args.cargo 780 else: 781 # Use the same target for both host and default device builds. 782 self.cargo = ['clean', 'build --target x86_64-unknown-linux-gnu'] 783 self.empty_tests = set() 784 self.empty_unittests = False 785 self.build_deps = {} 786 self.deps_libname_map = {} 787 788 def set_cargo_path(self): 789 """Find cargo in the --cargo_bin and set cargo path""" 790 if self.args.cargo_bin: 791 self.cargo_path = os.path.join(self.args.cargo_bin, 'cargo') 792 if os.path.isfile(self.cargo_path): 793 print('INFO: using cargo in ' + self.args.cargo_bin) 794 return 795 else: 796 sys.exit('ERROR: cannot find cargo in ' + self.args.cargo_bin) 797 else: 798 sys.exit('ERROR: the prebuilt cargo is not available; please use the --cargo_bin flag.') 799 return 800 801 def run_cargo(self): 802 """Run cargo -v and save its output to ./cargo.out.""" 803 if self.skip_cargo: 804 return self 805 cargo_toml = './Cargo.toml' 806 cargo_out = './cargo.out' 807 if not os.access(cargo_toml, os.R_OK): 808 print('ERROR: Cannot find ', cargo_toml) 809 return self 810 cargo_lock = './Cargo.lock' 811 cargo_lock_save = './cargo.lock.save' 812 have_cargo_lock = os.path.exists(cargo_lock) 813 if not self.dry_run: 814 if os.path.exists(cargo_out): 815 os.remove(cargo_out) 816 if not self.args.use_cargo_lock and have_cargo_lock: 817 os.rename(cargo_lock, cargo_lock_save) 818 # set up search PATH for cargo to find the correct rustc 819 save_path = os.environ['PATH'] 820 os.environ['PATH'] = os.path.dirname(self.cargo_path) + ':' + save_path 821 # Add [workspace] to Cargo.toml if it is non-existent. 822 is_add_workspace = False 823 if self.args.add_workspace: 824 with open(cargo_toml, 'r') as in_file: 825 cargo_toml_lines = in_file.readlines() 826 if '[workspace]\n' in cargo_toml_lines: 827 print('WARNING: found [workspace] in Cargo.toml') 828 else: 829 with open(cargo_toml, 'w') as out_file: 830 out_file.write('[workspace]\n') 831 is_add_workspace = True 832 self.deal_cargo_cmd(cargo_out) 833 # restore original Cargo.toml 834 if is_add_workspace: 835 with open(cargo_toml, 'w') as out_file: 836 out_file.writelines(cargo_toml_lines) 837 if not self.dry_run: 838 if not have_cargo_lock: # restore to no Cargo.lock state 839 if os.path.exists(cargo_lock): 840 os.remove(cargo_lock) 841 elif not self.args.use_cargo_lock: # restore saved Cargo.lock 842 os.rename(cargo_lock_save, cargo_lock) 843 os.environ['PATH'] = save_path 844 return self 845 846 def deal_cargo_cmd(self, cargo_out): 847 cargo_cmd_v_flag = ' -vv ' if self.args.vv else ' -v ' 848 cargo_cmd_target_dir = ' --target-dir ' + TARGET_TEMP 849 cargo_cmd_redir = ' >> ' + cargo_out + ' 2>&1' 850 for cargo in self.cargo: 851 cargo_cmd = self.cargo_path + cargo_cmd_v_flag 852 features = '' 853 if cargo != 'clean': 854 if self.args.features is not None: 855 features = ' --no-default-features' 856 if self.args.features: 857 features += ' --features ' + self.args.features 858 cargo_cmd += cargo + features + cargo_cmd_target_dir + cargo_cmd_redir 859 if self.args.rustflags and cargo != 'clean': 860 cargo_cmd = 'RUSTFLAGS="' + self.args.rustflags + '" ' + cargo_cmd 861 self.run_cargo_cmd(cargo_cmd, cargo_out) 862 863 def run_cargo_cmd(self, cargo_cmd, cargo_out): 864 if self.dry_run: 865 print('Dry-run skip:', cargo_cmd) 866 else: 867 with open(cargo_out, 'a') as file: 868 file.write('### Running: ' + cargo_cmd + '\n') 869 ret = os.system(cargo_cmd) 870 if ret != 0: 871 print('ERROR: There was an error while running cargo.' + 872 ' See the cargo.out file for details.') 873 874 def generate_gn(self): 875 """Parse cargo.out and generate BUILD.gn files.""" 876 cargo_out = 'cargo.out' # The file name used to save cargo build -v output. 877 errors_line = 'Errors in ' + cargo_out + ':' 878 if self.dry_run: 879 print('Dry-run skip: read', cargo_out, 'write BUILD.gn') 880 elif os.path.exists(cargo_out): 881 self.find_root_pkg() 882 with open(cargo_out, 'r') as cargo_out: 883 self.parse(cargo_out, 'BUILD.gn') 884 self.crates.sort(key=get_crate_name) 885 for crate in self.crates: 886 crate.dump() 887 if self.error_infos: 888 self.append_to_gn('\n' + errors_line + '\n' + self.error_infos) 889 if self.test_error_infos: 890 self.append_to_gn('\n// Errors when listing tests:\n' + 891 self.test_error_infos) 892 return self 893 894 def find_root_pkg(self): 895 """Read name of [package] in ./Cargo.toml.""" 896 if os.path.exists('./Cargo.toml'): 897 return 898 with open('./Cargo.toml', 'r') as infile: 899 get_designated_pkg_info(infile, 'name') 900 901 def parse(self, infile, outfile_name): 902 """Parse rustc, test, and warning messages in infile, return a list of Crates.""" 903 # cargo test --list output of the start of running a binary. 904 cargo_test_list_start_re = re.compile('^\s*Running (.*) \(.*\)$') 905 # cargo test --list output of the end of running a binary. 906 cargo_test_list_end_re = re.compile('^(\d+) tests, (\d+) benchmarks$') 907 compiling_pat = re.compile('^ +Compiling (.*)$') 908 current_test_name = None 909 for line in infile: 910 # We read the file in two passes, where the first simply checks for empty tests. 911 # Otherwise we would add and merge tests before seeing they're empty. 912 if cargo_test_list_start_re.match(line): 913 current_test_name = cargo_test_list_start_re.match(line).group(1) 914 elif current_test_name and cargo_test_list_end_re.match(line): 915 match = cargo_test_list_end_re.match(line) 916 if int(match.group(1)) + int(match.group(2)) == 0: 917 self.add_empty_test(current_test_name) 918 current_test_name = None 919 #Get Compiling information 920 if compiling_pat.match(line): 921 self.add_deps_libname_map(compiling_pat.match(line).group(1)) 922 infile.seek(0) 923 self.parse_cargo_out(infile, outfile_name) 924 925 def add_empty_test(self, name): 926 if name == 'unittests': 927 self.empty_unittests = True 928 else: 929 self.empty_tests.add(name) 930 931 def add_deps_libname_map(self, line): 932 line_list = line.split() 933 if len(line_list) > 1: 934 self.deps_libname_map[pkg_to_crate_name(line_list[0])] = line_list[0] 935 936 def parse_cargo_out(self, infile, outfile_name): 937 # Cargo -v output of a call to rustc. 938 rustc_re = re.compile('^ +Running `rustc (.*)`$') 939 # Cargo -vv output of a call to rustc could be split into multiple lines. 940 # Assume that the first line will contain some CARGO_* env definition. 941 rustc_vv_re = re.compile('^ +Running `.*CARGO_.*=.*$') 942 # Rustc output of file location path pattern for a warning message. 943 warning_output_file_re = re.compile('^ *--> ([^:]*):[0-9]+') 944 cargo_to_gn_running_re = re.compile('^### Running: .*$') 945 line_num = 0 946 previous_warning = False # true if the previous line was warning 947 rustc_line = '' # previous line matching rustc_vv_re 948 in_tests = False 949 for line in infile: 950 line_num += 1 951 if line.startswith('warning: '): 952 previous_warning = True 953 rustc_line = self.assert_empty_rustc_line(rustc_line) 954 continue 955 new_rustc_line = '' 956 if rustc_re.match(line): 957 args_line = rustc_re.match(line).group(1) 958 self.add_crate(Crate(self, outfile_name).parse_rustc(line_num, args_line)) 959 self.assert_empty_rustc_line(rustc_line) 960 elif rustc_line or rustc_vv_re.match(line): 961 new_rustc_line = self.deal_rustc_command( 962 line_num, rustc_line, line, outfile_name) 963 elif previous_warning and warning_output_file_re.match(line): 964 file_path = warning_output_file_re.match(line).group(1) 965 if file_path[0] != '/': # ignore absolute path 966 self.warning_files.add(file_path) 967 self.assert_empty_rustc_line(rustc_line) 968 elif line.startswith('error: ') or line.startswith('error[E'): 969 if not self.args.ignore_cargo_errors: 970 self.add_error_infos(in_tests, line) 971 elif cargo_to_gn_running_re.match(line): 972 in_tests = "cargo test" in line and "--list" in line 973 previous_warning = False 974 rustc_line = new_rustc_line 975 976 def assert_empty_rustc_line(self, line): 977 # report error if line is not empty 978 if line: 979 self.append_to_gn('ERROR -vv line: ' + line) 980 return '' 981 982 def append_to_gn(self, line): 983 self.init_gn_file('BUILD.gn') 984 with open('BUILD.gn', 'a') as outfile: 985 outfile.write(line) 986 print(line) 987 988 def init_gn_file(self, name): 989 # name could be BUILD.gn or sub_dir_path/BUILD.gn 990 if name in self.gn_files: 991 return 992 self.gn_files.add(name) 993 if os.path.exists(name): 994 os.remove(name) 995 with open(name, 'w') as outfile: 996 outfile.write(BUILD_GN_HEADER) 997 outfile.write('\n') 998 outfile.write('import("%s")\n' % IMPORT_CONTENT) 999 1000 def add_error_infos(self, in_tests, line): 1001 if in_tests: 1002 self.test_error_infos += '// '.join(line) 1003 else: 1004 self.error_infos += line 1005 1006 def deal_rustc_command(self, line_num, rustc_line, line, outfile_name): 1007 """Process a rustc command line from cargo -vv output.""" 1008 # cargo build -vv output can have multiple lines for a rustc command due to '\n' in strings 1009 # for environment variables. strip removes leading spaces and '\n' at the end 1010 new_rustc_line = (rustc_line.strip() + line) if rustc_line else line 1011 # The combined -vv output rustc command line pattern. 1012 rustc_vv_cmd_args = re.compile('^ *Running `.*CARGO_.*=.* rustc (.*)`$') 1013 if not line.endswith('`\n') or (new_rustc_line.count('`') % 2) != 0: 1014 return new_rustc_line 1015 if rustc_vv_cmd_args.match(new_rustc_line): 1016 args = rustc_vv_cmd_args.match(new_rustc_line).group(1) 1017 self.add_crate(Crate(self, outfile_name).parse_rustc(line_num, args)) 1018 else: 1019 self.assert_empty_rustc_line(new_rustc_line) 1020 return '' 1021 1022 def add_crate(self, new_crate): 1023 """Merge crate with someone in crates, or append to it. Return crates.""" 1024 if (is_build_script(new_crate.crate_name) and 1025 not is_dependent_path(new_crate.main_src) and 1026 new_crate.root_pkg_name and len(new_crate.deps) > 0): 1027 self.build_deps[new_crate.root_pkg_name] = new_crate.deps 1028 if new_crate.skip_crate(): 1029 # include debug info of all crates 1030 if self.args.debug: 1031 self.crates.append(new_crate) 1032 else: 1033 for crate in self.crates: 1034 if crate.merge_crate(new_crate, 'BUILD.gn'): 1035 return 1036 # If not merged, decide module type and name now. 1037 new_crate.decide_module_type() 1038 self.crates.append(new_crate) 1039 1040 def should_ignore_test(self, src): 1041 # cargo test outputs the source file for integration tests but "unittests" for unit tests. 1042 # To figure out to which crate this corresponds, we check if the current source file is 1043 # the main source of a non-test crate, e.g., a library or a binary. 1044 return (src in self.empty_tests or src in self.args.test_blocklist or 1045 (self.empty_unittests and 1046 src in [c.main_src for c in self.crates if c.crate_types != ['test']])) 1047 1048 1049def get_arg_parser(): 1050 """Parse main arguments.""" 1051 argparser = argparse.ArgumentParser('cargo2gn') 1052 argparser.add_argument('--add-workspace', action='store_true', default=False, 1053 help=('append [workspace] to Cargo.toml before calling cargo, to treat' + 1054 ' current directory as root of package source; otherwise the relative' + 1055 ' source file path in generated .gn file will be from the parent directory.')) 1056 argparser.add_argument('--cargo', action='append', metavar='args_string', 1057 help=('extra cargo build -v args in a string, ' + 1058 'each --cargo flag calls cargo build -v once')) 1059 argparser.add_argument('--cargo-bin', type=str, 1060 help='use cargo in the cargo_bin directory instead of the prebuilt one') 1061 argparser.add_argument('--config', type=str, 1062 help=('Load command-line options from the given config file. ' + 1063 'Options in this file will override those passed on the command line.')) 1064 argparser.add_argument('--copy-out', action='store_true', default=False, 1065 help=('only for root directory, copy build.rs output to ./out/* and ' + 1066 'add a genrule to copy ./out/*.')) 1067 argparser.add_argument('--debug', action='store_true', default=False, 1068 help='dump debug info into BUILD.gn') 1069 argparser.add_argument('--dependency-blocklist', nargs='*', default=[], 1070 help='Do not emit the given dependencies (without lib prefixes).') 1071 argparser.add_argument('--features', type=str, 1072 help=('pass features to cargo build, ' + 1073 'empty string means no default features')) 1074 argparser.add_argument('--ignore-cargo-errors', action='store_true', default=False, 1075 help='do not append cargo/rustc error messages to BUILD.gn') 1076 argparser.add_argument('--no-pkg-info', action='store_true', default=False, 1077 help='Do not attempt to determine the package info automatically.') 1078 argparser.add_argument('--no-subdir', action='store_true', default=False, 1079 help='do not output anything for sub-directories') 1080 argparser.add_argument('--one-file', action='store_true', default=False, 1081 help=('output all into one BUILD.gn, default will generate one BUILD.gn ' + 1082 'per Cargo.toml in subdirectories')) 1083 argparser.add_argument('--run', action='store_true', default=False, 1084 help='run it, default is dry-run') 1085 argparser.add_argument('--rustflags', type=str, help='passing flags to rustc') 1086 argparser.add_argument('--skipcargo', action='store_true', default=False, 1087 help='skip cargo command, parse cargo.out, and generate BUILD.gn') 1088 argparser.add_argument('--test-blocklist', nargs='*', default=[], 1089 help=('Do not emit the given tests. ' + 1090 'Pass the path to the test file to exclude.')) 1091 argparser.add_argument('--use-cargo-lock', action='store_true', default=False, 1092 help=('run cargo build with existing Cargo.lock ' + 1093 '(used when some latest dependent crates failed)')) 1094 argparser.add_argument('--vv', action='store_true', default=False, 1095 help='run cargo with -vv instead of default -v') 1096 return argparser 1097 1098 1099def get_parse_args(argparser): 1100 """Parses command-line options.""" 1101 args = argparser.parse_args() 1102 # Use the values specified in a config file if one was found. 1103 if args.config: 1104 with open(args.config, 'r') as file: 1105 config_data = json.load(file) 1106 args_dict = vars(args) 1107 for arg in config_data: 1108 args_dict[arg.replace('-', '_')] = config_data[arg] 1109 return args 1110 1111 1112def main(): 1113 argparser = get_arg_parser() 1114 args = get_parse_args(argparser) 1115 if not args.run: # default is dry-run 1116 print(DRY_RUN_CONTENT) 1117 Runner(args).run_cargo().generate_gn() 1118 1119 1120if __name__ == '__main__': 1121 main() 1122