1# -*- bazel-starlark -*- 2# Copyright 2023 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Siso configuration for rewriting remote calls into reproxy config.""" 6 7load("@builtin//encoding.star", "json") 8load("@builtin//lib/gn.star", "gn") 9load("@builtin//path.star", "path") 10load("@builtin//struct.star", "module") 11load("./clang_code_coverage_wrapper.star", "clang_code_coverage_wrapper") 12load("./config.star", "config") 13load("./platform.star", "platform") 14load("./rewrapper_cfg.star", "rewrapper_cfg") 15 16def __filegroups(ctx): 17 return {} 18 19def __parse_rewrapper_cmdline(ctx, cmd): 20 if not "rewrapper" in cmd.args[0]: 21 return [], "", False 22 23 # Example command: 24 # ../../buildtools/reclient/rewrapper 25 # -cfg=../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 26 # -exec_root=/path/to/your/chromium/src/ 27 # ../../third_party/llvm-build/Release+Asserts/bin/clang++ 28 # [rest of clang args] 29 # We don't need to care about: 30 # -exec_root: Siso already knows this. 31 wrapped_command_pos = -1 32 cfg_file = None 33 for i, arg in enumerate(cmd.args): 34 if i == 0: 35 continue 36 if arg.startswith("-cfg="): 37 cfg_file = ctx.fs.canonpath(arg.removeprefix("-cfg=")) 38 continue 39 if not arg.startswith("-"): 40 wrapped_command_pos = i 41 break 42 if wrapped_command_pos < 1: 43 fail("couldn't find first non-arg passed to rewrapper for %s" % str(cmd.args)) 44 if not cfg_file: 45 return cmd.args[wrapped_command_pos:], None, True 46 return cmd.args[wrapped_command_pos:], rewrapper_cfg.parse(ctx, cfg_file), True 47 48def __parse_cros_rewrapper_cmdline(ctx, cmd): 49 # fix cros sdk clang command line and extract rewrapper cfg. 50 # Example command: 51 # ../../build/cros_cache/chrome-sdk/symlinks/amd64-generic+15629.0.0+target_toolchain/bin/x86_64-cros-linux-gnu-clang++ 52 # -MMD -MF obj/third_party/abseil-cpp/absl/base/base/spinlock.o.d 53 # ... 54 # --rewrapper-path /usr/local/google/home/ukai/src/chromium/src/build/args/chromeos/rewrapper_amd64-generic 55 # --rewrapper-cfg ../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 56 # -pipe -march=x86-64 -msse3 ... 57 cfg_file = None 58 skip = "" 59 args = [] 60 toolchainpath = None 61 for i, arg in enumerate(cmd.args): 62 if i == 0: 63 toolchainpath = path.dir(path.dir(ctx.fs.canonpath(arg))) 64 args.append(arg) 65 continue 66 if skip: 67 if skip == "--rewrapper-cfg": 68 cfg_file = ctx.fs.canonpath(arg) 69 skip = "" 70 continue 71 if arg in ("--rewrapper-path", "--rewrapper-cfg"): 72 skip = arg 73 continue 74 args.append(arg) 75 if not cfg_file: 76 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 77 rwcfg = rewrapper_cfg.parse(ctx, cfg_file) 78 inputs = rwcfg.get("inputs", []) 79 inputs.append(toolchainpath) 80 rwcfg["inputs"] = inputs 81 rwcfg["preserve_symlinks"] = True 82 return args, rwcfg 83 84# TODO(b/278225415): change gn so this wrapper (and by extension this handler) becomes unnecessary. 85def __parse_clang_code_coverage_wrapper_cmdline(ctx, cmd): 86 # Example command: 87 # python3 88 # ../../build/toolchain/clang_code_coverage_wrapper.py 89 # --target-os=... 90 # --files_to_instrument=... 91 # ../../buildtools/reclient/rewrapper 92 # -cfg=../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 93 # -exec_root=/path/to/your/chromium/src/ 94 # ../../third_party/llvm-build/Release+Asserts/bin/clang++ 95 # [rest of clang args] 96 # We don't need to care about: 97 # most args to clang_code_coverage_wrapper (need --files_to_instrument as tool_input) 98 # -exec_root: Siso already knows this. 99 rewrapper_pos = -1 100 wrapped_command_pos = -1 101 cfg_file = None 102 for i, arg in enumerate(cmd.args): 103 if i < 2: 104 continue 105 if rewrapper_pos == -1 and not arg.startswith("-"): 106 rewrapper_pos = i 107 continue 108 if rewrapper_pos > 0 and arg.startswith("-cfg="): 109 cfg_file = ctx.fs.canonpath(arg.removeprefix("-cfg=")) 110 continue 111 if rewrapper_pos > 0 and not arg.startswith("-"): 112 wrapped_command_pos = i 113 break 114 if rewrapper_pos < 1: 115 fail("couldn't find rewrapper in %s" % str(cmd.args)) 116 if wrapped_command_pos < 1: 117 fail("couldn't find first non-arg passed to rewrapper for %s" % str(cmd.args)) 118 if not cfg_file: 119 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 120 coverage_wrapper_command = cmd.args[:rewrapper_pos] + cmd.args[wrapped_command_pos:] 121 clang_command = clang_code_coverage_wrapper.run(ctx, list(coverage_wrapper_command)) 122 if len(clang_command) > 1 and "/chrome-sdk/" in clang_command[0]: 123 # TODO: implement cros sdk support under code coverage wrapper 124 fail("need to fix handler for cros sdk under code coverage wrapper") 125 return clang_command, rewrapper_cfg.parse(ctx, cfg_file) 126 127def __rewrite_rewrapper(ctx, cmd): 128 # If clang-coverage, needs different handling. 129 if len(cmd.args) > 2 and "clang_code_coverage_wrapper.py" in cmd.args[1]: 130 args, rwcfg = __parse_clang_code_coverage_wrapper_cmdline(ctx, cmd) 131 elif len(cmd.args) > 1 and "/chrome-sdk/" in cmd.args[0]: 132 args, rwcfg = __parse_cros_rewrapper_cmdline(ctx, cmd) 133 else: 134 # handling for generic rewrapper. 135 args, rwcfg, wrapped = __parse_rewrapper_cmdline(ctx, cmd) 136 if not wrapped: 137 print("command doesn't have rewrapper. %s" % str(cmd.args)) 138 return 139 if not rwcfg: 140 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 141 if cmd.outputs[0] == ctx.fs.canonpath("./obj/third_party/abseil-cpp/absl/functional/any_invocable_test/any_invocable_test.o"): 142 # need longer timeout for any_invocable_test.o crbug.com/1484474 143 rwcfg.update({ 144 "exec_timeout": "4m", 145 }) 146 ctx.actions.fix( 147 args = args, 148 reproxy_config = json.encode(rwcfg), 149 ) 150 151def __strip_rewrapper(ctx, cmd): 152 # If clang-coverage, needs different handling. 153 if len(cmd.args) > 2 and "clang_code_coverage_wrapper.py" in cmd.args[1]: 154 args, _ = __parse_clang_code_coverage_wrapper_cmdline(ctx, cmd) 155 else: 156 args, _, wrapped = __parse_rewrapper_cmdline(ctx, cmd) 157 if not wrapped: 158 print("command doesn't have rewrapper. %s" % str(cmd.args)) 159 return 160 ctx.actions.fix(args = args) 161 162def __rewrite_action_remote_py(ctx, cmd): 163 # Example command: 164 # python3 165 # ../../build/util/action_remote.py 166 # ../../buildtools/reclient/rewrapper 167 # --custom_processor=mojom_parser 168 # --cfg=../../buildtools/reclient_cfgs/python/rewrapper_linux.cfg 169 # --exec_root=/path/to/your/chromium/src/ 170 # --input_list_paths=gen/gpu/ipc/common/surface_handle__parser__remote_inputs.rsp 171 # --output_list_paths=gen/gpu/ipc/common/surface_handle__parser__remote_outputs.rsp 172 # python3 173 # ../../mojo/public/tools/mojom/mojom_parser.py 174 # [rest of mojo args] 175 # We don't need to care about: 176 # --exec_root: Siso already knows this. 177 # --custom_processor: Used by action_remote.py to apply mojo handling. 178 # --[input,output]_list_paths: We should always use mojo.star for Siso. 179 wrapped_command_pos = -1 180 cfg_file = None 181 for i, arg in enumerate(cmd.args): 182 if i < 3: 183 continue 184 185 # TODO: b/300046750 - Fix GN args and/or implement input processor. 186 if arg == "--custom_processor=mojom_parser": 187 print("--custom_processor=mojom_parser is not supported. " + 188 "Running locally. cmd=%s" % " ".join(cmd.args)) 189 return 190 if arg.startswith("--cfg="): 191 cfg_file = ctx.fs.canonpath(arg.removeprefix("--cfg=")) 192 continue 193 if not arg.startswith("-"): 194 wrapped_command_pos = i 195 break 196 if wrapped_command_pos < 1: 197 fail("couldn't find action command in %s" % str(cmd.args)) 198 ctx.actions.fix( 199 args = cmd.args[wrapped_command_pos:], 200 reproxy_config = json.encode(rewrapper_cfg.parse(ctx, cfg_file)), 201 ) 202 203__handlers = { 204 "rewrite_rewrapper": __rewrite_rewrapper, 205 "strip_rewrapper": __strip_rewrapper, 206 "rewrite_action_remote_py": __rewrite_action_remote_py, 207} 208 209def __use_remoteexec(ctx): 210 if "args.gn" in ctx.metadata: 211 gn_args = gn.args(ctx) 212 if gn_args.get("use_remoteexec") == "true": 213 return True 214 return False 215 216def __step_config(ctx, step_config): 217 # New rules to convert commands calling rewrapper to use reproxy instead. 218 new_rules = [ 219 # Disabling remote should always come first. 220 { 221 # TODO(b/281663988): missing headers. 222 "name": "b281663988/missing-headers", 223 "action_outs": [ 224 "./obj/ui/qt/qt5_shim/qt_shim.o", 225 "./obj/ui/qt/qt6_shim/qt_shim.o", 226 "./obj/ui/qt/qt5_shim/qt5_shim_moc.o", 227 "./obj/ui/qt/qt6_shim/qt6_shim_moc.o", 228 "./obj/ui/qt/qt_interface/qt_interface.o", 229 ], 230 "remote": False, 231 "handler": "strip_rewrapper", 232 }, 233 # Handle generic action_remote calls. 234 { 235 "name": "action_remote", 236 "command_prefix": platform.python_bin + " ../../build/util/action_remote.py ../../buildtools/reclient/rewrapper", 237 "handler": "rewrite_action_remote_py", 238 "remote_command": "python3", 239 }, 240 ] 241 242 # Disable racing on builders since bots don't have many CPU cores. 243 # TODO: b/297807325 - Siso wants to handle local execution. 244 # However, Reclient's alerts require racing and local fallback to be 245 # done on Reproxy side. 246 exec_strategy = "racing" 247 if config.get(ctx, "builder"): 248 exec_strategy = "remote_local_fallback" 249 250 for rule in step_config["rules"]: 251 # Replace nacl-clang/clang++ rules without command_prefix, because they will incorrectly match rewrapper. 252 # Replace the original step rule with one that only rewrites rewrapper and convert its rewrapper config to reproxy config. 253 if rule["name"].find("nacl-clang") >= 0 and not rule.get("command_prefix"): 254 new_rule = { 255 "name": rule["name"], 256 "action": rule["action"], 257 "handler": "rewrite_rewrapper", 258 } 259 new_rules.append(new_rule) 260 continue 261 262 # clang will always have rewrapper config when use_remoteexec=true. 263 # Remove the native siso handling and replace with custom rewrapper-specific handling. 264 # All other rule values are not reused, instead use rewrapper config via handler. 265 # (In particular, command_prefix should be avoided because it will be rewrapper.) 266 if rule["name"].startswith("clang/") or rule["name"].startswith("clang-cl/"): 267 if not rule.get("action"): 268 fail("clang rule %s found without action" % rule["name"]) 269 270 if not config.get(ctx, "reproxy-cros"): 271 # TODO: b/314698010 - use reproxy mode once performance issue is fixed. 272 cros_rule = { 273 "name": rule["name"] + "/cros", 274 "action": rule["action"], 275 "command_prefix": "../../build/cros_cache/", 276 "use_remote_exec_wrapper": True, 277 } 278 new_rules.append(cros_rule) 279 280 new_rule = { 281 "name": rule["name"], 282 "action": rule["action"], 283 "handler": "rewrite_rewrapper", 284 } 285 new_rules.append(new_rule) 286 continue 287 288 # clang-coverage/ is handled by the rewrite_rewrapper handler of clang/{cxx, cc} action rules above, so ignore these rules. 289 if rule["name"].startswith("clang-coverage/"): 290 continue 291 292 # Add non-remote rules as-is. 293 if not rule.get("remote"): 294 new_rules.append(rule) 295 continue 296 297 # Finally handle remaining remote rules. It's assumed it is enough to only convert native remote config to reproxy config. 298 platform_ref = rule.get("platform_ref") 299 if platform_ref: 300 p = step_config["platforms"].get(platform_ref) 301 if not p: 302 fail("Rule %s uses undefined platform '%s'" % (rule["name"], platform_ref)) 303 else: 304 p = step_config.get("platforms", {}).get("default") 305 if not p: 306 fail("Rule %s did not set platform_ref but no default platform exists" % rule["name"]) 307 rule["reproxy_config"] = { 308 "platform": p, 309 "labels": { 310 "type": "tool", 311 "siso_rule": rule["name"], 312 }, 313 "canonicalize_working_dir": rule.get("canonicalize_dir", False), 314 "exec_strategy": exec_strategy, 315 "exec_timeout": rule.get("timeout", "10m"), 316 "reclient_timeout": rule.get("timeout", "10m"), 317 "download_outputs": True, 318 } 319 new_rules.append(rule) 320 321 step_config["rules"] = new_rules 322 return step_config 323 324reproxy = module( 325 "reproxy", 326 enabled = __use_remoteexec, 327 step_config = __step_config, 328 filegroups = __filegroups, 329 handlers = __handlers, 330) 331