1# -*- bazel-starlark -*- 2# Copyright 2023 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Siso configuration for rewriting remote calls into reproxy config.""" 6 7load("@builtin//encoding.star", "json") 8load("@builtin//lib/gn.star", "gn") 9load("@builtin//struct.star", "module") 10load("./rewrapper_cfg.star", "rewrapper_cfg") 11load("./clang_code_coverage_wrapper.star", "clang_code_coverage_wrapper") 12 13__filegroups = {} 14 15def __parse_rewrapper_cmdline(ctx, cmd): 16 if not "rewrapper" in cmd.args[0]: 17 return [], "", False 18 19 # Example command: 20 # ../../buildtools/reclient/rewrapper 21 # -cfg=../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 22 # -exec_root=/path/to/your/chromium/src/ 23 # ../../third_party/llvm-build/Release+Asserts/bin/clang++ 24 # [rest of clang args] 25 # We don't need to care about: 26 # -exec_root: Siso already knows this. 27 wrapped_command_pos = -1 28 cfg_file = None 29 for i, arg in enumerate(cmd.args): 30 if i == 0: 31 continue 32 if arg.startswith("-cfg="): 33 cfg_file = ctx.fs.canonpath(arg.removeprefix("-cfg=")) 34 continue 35 if not arg.startswith("-"): 36 wrapped_command_pos = i 37 break 38 if wrapped_command_pos < 1: 39 fail("couldn't find first non-arg passed to rewrapper for %s" % str(cmd.args)) 40 return cmd.args[wrapped_command_pos:], cfg_file, True 41 42def __rewrite_rewrapper(ctx, cmd): 43 args, cfg_file, wrapped = __parse_rewrapper_cmdline(ctx, cmd) 44 if not wrapped: 45 return 46 if not cfg_file: 47 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 48 ctx.actions.fix( 49 args = args, 50 reproxy_config = json.encode(rewrapper_cfg.parse(ctx, cfg_file)), 51 ) 52 53def __strip_rewrapper(ctx, cmd): 54 args, _, wrapped = __parse_rewrapper_cmdline(ctx, cmd) 55 if not wrapped: 56 return 57 ctx.actions.fix(args = args) 58 59# TODO(b/278225415): change gn so this wrapper (and by extension this handler) becomes unnecessary. 60def __rewrite_clang_code_coverage_wrapper(ctx, cmd): 61 # Example command: 62 # python3 63 # ../../build/toolchain/clang_code_coverage_wrapper.py 64 # --target-os=... 65 # --files_to_instrument=... 66 # ../../buildtools/reclient/rewrapper 67 # -cfg=../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 68 # -exec_root=/path/to/your/chromium/src/ 69 # ../../third_party/llvm-build/Release+Asserts/bin/clang++ 70 # [rest of clang args] 71 # We don't need to care about: 72 # most args to clang_code_coverage_wrapper (need --files_to_instrument as tool_input) 73 # -exec_root: Siso already knows this. 74 rewrapper_pos = -1 75 wrapped_command_pos = -1 76 cfg_file = None 77 for i, arg in enumerate(cmd.args): 78 if i < 2: 79 continue 80 if rewrapper_pos == -1 and not arg.startswith("-"): 81 rewrapper_pos = i 82 continue 83 if rewrapper_pos > 0 and arg.startswith("-cfg="): 84 cfg_file = ctx.fs.canonpath(arg.removeprefix("-cfg=")) 85 continue 86 if rewrapper_pos > 0 and not arg.startswith("-"): 87 wrapped_command_pos = i 88 break 89 if rewrapper_pos < 1: 90 fail("couldn't find rewrapper in %s" % str(cmd.args)) 91 if wrapped_command_pos < 1: 92 fail("couldn't find first non-arg passed to rewrapper for %s" % str(cmd.args)) 93 if not cfg_file: 94 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 95 coverage_wrapper_command = cmd.args[:rewrapper_pos] + cmd.args[wrapped_command_pos:] 96 clang_command = clang_code_coverage_wrapper.run(ctx, list(coverage_wrapper_command)) 97 98 ctx.actions.fix( 99 args = clang_command, 100 reproxy_config = json.encode(rewrapper_cfg.parse(ctx, cfg_file)), 101 ) 102 103def __rewrite_action_remote_py(ctx, cmd): 104 # Example command: 105 # python3 106 # ../../build/util/action_remote.py 107 # ../../buildtools/reclient/rewrapper 108 # --custom_processor=mojom_parser 109 # --cfg=../../buildtools/reclient_cfgs/python/rewrapper_linux.cfg 110 # --exec_root=/path/to/your/chromium/src/ 111 # --input_list_paths=gen/gpu/ipc/common/surface_handle__parser__remote_inputs.rsp 112 # --output_list_paths=gen/gpu/ipc/common/surface_handle__parser__remote_outputs.rsp 113 # python3 114 # ../../mojo/public/tools/mojom/mojom_parser.py 115 # [rest of mojo args] 116 # We don't need to care about: 117 # --exec_root: Siso already knows this. 118 # --custom_processor: Used by action_remote.py to apply mojo handling. 119 # --[input,output]_list_paths: We should always use mojo.star for Siso. 120 wrapped_command_pos = -1 121 cfg_file = None 122 for i, arg in enumerate(cmd.args): 123 if i < 3: 124 continue 125 if arg.startswith("--cfg="): 126 cfg_file = ctx.fs.canonpath(arg.removeprefix("--cfg=")) 127 continue 128 if not arg.startswith("-"): 129 wrapped_command_pos = i 130 break 131 if wrapped_command_pos < 1: 132 fail("couldn't find action command in %s" % str(cmd.args)) 133 ctx.actions.fix( 134 args = cmd.args[wrapped_command_pos:], 135 reproxy_config = json.encode(rewrapper_cfg.parse(ctx, cfg_file)), 136 ) 137 138__handlers = { 139 "rewrite_rewrapper": __rewrite_rewrapper, 140 "strip_rewrapper": __strip_rewrapper, 141 "rewrite_clang_code_coverage_wrapper": __rewrite_clang_code_coverage_wrapper, 142 "rewrite_action_remote_py": __rewrite_action_remote_py, 143} 144 145def __use_remoteexec(ctx): 146 if "args.gn" in ctx.metadata: 147 gn_args = gn.parse_args(ctx.metadata["args.gn"]) 148 if gn_args.get("use_remoteexec") == "true": 149 return True 150 return False 151 152def __step_config(ctx, step_config): 153 # New rules to convert commands calling rewrapper to use reproxy instead. 154 new_rules = [ 155 # mojo/mojom_bindings_generator will not always have rewrapper args. 156 # Use this rule for commands with rewrapper args, the native remote rule is converted above. 157 { 158 "name": "mojo/mojom_bindings_generator_rewrapper", 159 "action": "mojom_(.*_)?__generator", 160 "command_prefix": "python3 ../../build/util/action_remote.py ../../buildtools/reclient/rewrapper --cfg=", 161 "handler": "rewrite_action_remote_py", 162 }, 163 # Handle generic action_remote calls. 164 { 165 "name": "action_remote", 166 "command_prefix": "python3 ../../build/util/action_remote.py ../../buildtools/reclient/rewrapper", 167 "handler": "rewrite_action_remote_py", 168 }, 169 ] 170 171 for rule in step_config["rules"]: 172 # mojo/mojom_parser will always have rewrapper config when use_remoteexec=true. 173 # Mutate the original step rule to rewrite rewrapper and convert its rewrapper config to reproxy config. 174 # Stop handling the rule so that it's not modified below. 175 # TODO(b/292838933): Implement mojom_parser processor in Starlark? 176 if rule["name"] == "mojo/mojom_parser": 177 rule.update({ 178 "command_prefix": "python3 ../../build/util/action_remote.py ../../buildtools/reclient/rewrapper --custom_processor=mojom_parser", 179 "handler": "rewrite_action_remote_py", 180 }) 181 new_rules.append(rule) 182 continue 183 184 # Replace nacl-clang/clang++ rules without command_prefix, because they will incorrectly match rewrapper. 185 # Replace the original step rule with one that only rewrites rewrapper and convert its rewrapper config to reproxy config. 186 if rule["name"].find("nacl-clang") >= 0 and not rule.get("command_prefix"): 187 new_rule = { 188 "name": rule["name"], 189 "action": rule["action"], 190 "handler": "rewrite_rewrapper", 191 } 192 new_rules.append(new_rule) 193 continue 194 195 # clang will always have rewrapper config when use_remoteexec=true. 196 # Remove the native siso handling and replace with custom rewrapper-specific handling. 197 # All other rule values are not reused, instead use rewrapper config via handler. 198 if rule["name"].startswith("clang/") or rule["name"].startswith("clang-cl/"): 199 if not rule.get("action"): 200 fail("clang rule %s found without action" % rule["name"]) 201 new_rule = { 202 "name": rule["name"], 203 "action": rule["action"], 204 "handler": "rewrite_rewrapper", 205 } 206 new_rules.append(new_rule) 207 continue 208 209 # clang-coverage will always have rewrapper config when use_remoteexec=true. 210 # Remove the native siso handling and replace with custom rewrapper-specific handling. 211 # All other rule values are not reused, instead use rewrapper config via handler. 212 # TODO(b/278225415): change gn so this wrapper (and by extension these rules) become unnecessary. 213 if rule["name"].startswith("clang-coverage"): 214 if rule["command_prefix"].find("../../build/toolchain/clang_code_coverage_wrapper.py") < 0: 215 fail("clang-coverage rule %s found without clang_code_coverage_wrapper.py in command_prefix" % rule["name"]) 216 new_rule = { 217 "name": rule["name"], 218 "command_prefix": rule["command_prefix"], 219 "handler": "rewrite_clang_code_coverage_wrapper", 220 } 221 # Insert clang-coverage/ rules at the top. 222 # They are more specific than reproxy clang/ rules, therefore should not be placed after. 223 new_rules.insert(0, new_rule) 224 continue 225 226 # Other rules where it's enough to only convert native remote config to reproxy config. 227 if not rule.get("remote"): 228 continue 229 platform_ref = rule.get("platform_ref") 230 if platform_ref: 231 platform = step_config["platforms"].get(platform_ref) 232 if not platform: 233 fail("Rule %s uses undefined platform '%s'" % (rule["name"], platform_ref)) 234 else: 235 platform = step_config.get("platforms", {}).get("default") 236 if not platform: 237 fail("Rule %s did not set platform_ref but no default platform exists" % rule["name"]) 238 rule["reproxy_config"] = { 239 "platform": platform, 240 "labels": { 241 "type": "tool", 242 }, 243 "inputs": rule.get("inputs", []), 244 "canonicalize_working_dir": rule.get("canonicalize_dir", False), 245 "exec_strategy": "remote", 246 "exec_timeout": rule.get("timeout", "10m"), 247 "download_outputs": True, 248 } 249 new_rules.append(rule) 250 251 step_config["rules"] = new_rules 252 return step_config 253 254reproxy = module( 255 "reproxy", 256 enabled = __use_remoteexec, 257 step_config = __step_config, 258 filegroups = __filegroups, 259 handlers = __handlers, 260) 261