1# Copyright 2024 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4""" 5Generates the rust build script output into a well-structured JSON format. 6""" 7import argparse 8from typing import Set, List, Dict 9import glob 10import multiprocessing.dummy 11import json 12import tempfile 13import re 14import subprocess 15import os 16import sys 17 18REPOSITORY_ROOT = os.path.abspath( 19 os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)) 20sys.path.insert(0, REPOSITORY_ROOT) 21 22# The current value of 500 is a heuristic that seems to work. If command 23# line length limitation is exceeded, reduce this number. 24_MAX_TARGETS_PER_NINJA_EXECUTION = 500 25 26import components.cronet.tools.utils as cronet_utils # pylint: disable=wrong-import-position 27import components.cronet.gn2bp.gen_android_bp as cronet_gn2bp # pylint: disable=wrong-import-position 28 29# TODO: Move ARCHS to cronet_utils. 30_ARCHS = ["x86", "x64", "arm", "arm64", "riscv64"] 31# TODO: Move _OUT_DIR to cronet_utils. 32_OUT_DIR = os.path.join(REPOSITORY_ROOT, "out") 33 34 35def _find_all_cargo_flags_files(out_dir: str) -> Set[str]: 36 return set(glob.glob(f"{out_dir}/gen/**/cargo_flags.rs", recursive=True)) 37 38 39def _find_all_host_cargo_flags_files(out_dir: str) -> Set[str]: 40 return set( 41 glob.glob(f"{out_dir}/clang_*/gen/**/cargo_flags.rs", recursive=True)) 42 43 44def _build_rust_build_script_actions(out_path: str): 45 """Builds build script actions, first GN is used to query 46 all actions that are available to build, then the actions are 47 filtered to only actions that has "build_script_output" in its 48 name which indicates that it builds a build_script. 49 50 The build script actions are split into chunks of _MAX_TARGETS_PER_NINJA_EXECUTION 51 where each ninja execution will build _MAX_TARGETS_PER_NINJA_EXECUTION actions. 52 This is to avoid hitting the command-line maximum length limitation. 53 54 Args: 55 out_path: the GN output directory. 56 57 Raises: 58 Exception: If ninja execution has failed or querying GN failed. 59 """ 60 all_actions_process = subprocess.run( 61 [cronet_utils.GN_PATH, 'ls', out_path, '--as=output', '--type=action'], 62 check=True, 63 capture_output=True, 64 text=True) 65 all_actions_process.check_returncode() 66 build_script_actions = [ 67 action for action in all_actions_process.stdout.split("\n") 68 # Skip roboelectric actions. 69 if "build_script_output" in action and "robolectric" not in action 70 ] 71 # Split the build script actions into chunk of _MAX_TARGETS_PER_NINJA_EXECUTION. 72 # This is needed in order not to exceed the command-line length. 73 build_script_actions_chunk = [ 74 build_script_actions[i:i + _MAX_TARGETS_PER_NINJA_EXECUTION] for i in 75 range(0, len(build_script_actions), _MAX_TARGETS_PER_NINJA_EXECUTION) 76 ] 77 for chunk in build_script_actions_chunk: 78 if cronet_utils.build_all(out_path, chunk) != 0: 79 raise Exception( 80 f"Failed to build the following build actions scripts chunk: {chunk}." 81 ) 82 83 84def _get_target_name_from_file(file_name: str) -> str: 85 """Extracts the target name which generated the file from 86 the directory path. 87 88 The file_name format is "[X]/gen/path/to/BUILDGN/target_name/file_name" 89 90 [X] is clang_* if this is a host version of the target, otherwise 91 [X] is an empty string. 92 93 94 Args: 95 file_name: Path to cargo_flags.rs file relative in GN output dir 96 97 Returns: 98 GN target label that generated the file. 99 """ 100 # Remove everything before gen/ directory 101 file_name = re.sub(".*gen\/", "", file_name) 102 dirs = file_name.split("/") 103 build_gn_path = "/".join(dirs[0:-2]) 104 target_name = dirs[-2] 105 return f"//{build_gn_path}:{target_name}" 106 107 108def _generate_build_script_outputs_for_host() -> Dict[str, List[str]]: 109 return _generate_build_script_outputs_for_arch("x64", True) 110 111 112def _generate_build_script_outputs_for_arch(arch: str, 113 host_variant: bool = False 114 ) -> Dict[str, List[str]]: 115 # gn desc behaves completely differently when the output 116 # directory is outside of chromium/src, some paths will 117 # stop having // in the beginning of their labels 118 # eg (//A/B will become A/B), this mostly apply to files 119 # that are generated through actions and not targets. 120 # This is why the temporary directory has to be generated 121 # beneath the repository root until gn2bp is tweaked to 122 # deal with this small differences. 123 target_name_to_build_script_output = {} 124 with tempfile.TemporaryDirectory(dir=_OUT_DIR) as gn_out_dir: 125 cronet_utils.gn(gn_out_dir, ' '.join(cronet_utils.get_gn_args_for_aosp(arch))) 126 _build_rust_build_script_actions(gn_out_dir) 127 build_script_output_files = _find_all_host_cargo_flags_files( 128 gn_out_dir) if host_variant else _find_all_cargo_flags_files(gn_out_dir) 129 130 for build_script_output_file in build_script_output_files: 131 target_name = _get_target_name_from_file(build_script_output_file) 132 target_name_to_build_script_output[target_name] = cronet_utils.read_file( 133 os.path.join(gn_out_dir, 134 build_script_output_file)).rstrip("\n").split("\n") 135 return target_name_to_build_script_output 136 137 138def _generate_build_scripts_outputs( 139 archs: List[str], 140 targets: List[str]) -> Dict[str, Dict[str, List[str]]]: 141 build_scripts_output_per_arch = {} 142 with multiprocessing.dummy.Pool(len(archs)) as pool: 143 results = [(arch, 144 pool.apply_async(_generate_build_script_outputs_for_arch, 145 (arch, ))) for arch in archs] 146 for (arch, result) in results: 147 build_script_output = result.get() 148 for (target_name, output) in build_script_output.items(): 149 if targets and target_name not in targets: 150 continue 151 if target_name not in build_scripts_output_per_arch: 152 build_scripts_output_per_arch[target_name] = {} 153 build_scripts_output_per_arch[target_name][arch] = output 154 155 # Generate host-specific build script outputs 156 build_script_output = _generate_build_script_outputs_for_host() 157 for (target_name, output) in build_script_output.items(): 158 if targets and target_name not in targets: 159 continue 160 if target_name not in build_scripts_output_per_arch: 161 build_scripts_output_per_arch[target_name] = {} 162 build_scripts_output_per_arch[target_name]["host"] = output 163 return build_scripts_output_per_arch 164 165 166def dump_build_scripts_outputs_to_file( 167 output_file_path: str, 168 archs: List[str], 169 targets_to_build: List[str] = None) -> None: 170 """Dumps a JSON formatted string that maps from target 171 name to build scripts output. 172 173 Args: 174 output_file_path: Path of the file to write the output to 175 archs: List of archs to compile for 176 targets_to_build: If specified, only those targets build_script will 177 be present in the final output. Otherwise, everything will be available. 178 """ 179 with open(output_file_path, "w") as output_file: 180 output_file.write( 181 json.dumps(_generate_build_scripts_outputs(archs, targets_to_build), 182 indent=2, 183 sort_keys=True)) 184 185 186def main(): 187 parser = argparse.ArgumentParser( 188 description= 189 'Generates a JSON dictionary containing the mapping between GN target labels to Rust build script output' 190 ) 191 parser.add_argument( 192 '--output', 193 type=str, 194 help='Path to file for which the output will be written to', 195 required=True) 196 args = parser.parse_args() 197 dump_build_scripts_outputs_to_file(args.output, _ARCHS) 198 199 200if __name__ == '__main__': 201 sys.exit(main()) 202