1# -*- bazel-starlark -*- 2# Copyright 2023 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Siso configuration for rewriting remote calls into reproxy config.""" 6 7load("@builtin//encoding.star", "json") 8load("@builtin//lib/gn.star", "gn") 9load("@builtin//path.star", "path") 10load("@builtin//runtime.star", "runtime") 11load("@builtin//struct.star", "module") 12load("./clang_code_coverage_wrapper.star", "clang_code_coverage_wrapper") 13load("./config.star", "config") 14load("./platform.star", "platform") 15load("./rewrapper_cfg.star", "rewrapper_cfg") 16 17def __filegroups(ctx): 18 return {} 19 20def __parse_rewrapper_cmdline(ctx, cmd): 21 if not "rewrapper" in cmd.args[0]: 22 return [], "", False 23 24 # Example command: 25 # ../../buildtools/reclient/rewrapper 26 # -cfg=../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 27 # -inputs=build/config/unsafe_buffers_paths.txt 28 # -exec_root=/path/to/your/chromium/src/ 29 # ../../third_party/llvm-build/Release+Asserts/bin/clang++ 30 # [rest of clang args] 31 # We don't need to care about: 32 # -exec_root: Siso already knows this. 33 wrapped_command_pos = -1 34 cfg_file = None 35 skip = "" 36 rw_cmd_opts = {} 37 for i, arg in enumerate(cmd.args): 38 if i == 0: 39 continue 40 if arg.startswith("-cfg="): 41 cfg_file = ctx.fs.canonpath(arg.removeprefix("-cfg=")) 42 continue 43 if arg.startswith("-inputs=") or skip == "-inputs": 44 rw_cmd_opts["inputs"] = arg.removeprefix("-inputs=").split(",") 45 skip = "" 46 continue 47 if arg == "-inputs": 48 skip = arg 49 continue 50 if not arg.startswith("-"): 51 wrapped_command_pos = i 52 break 53 if wrapped_command_pos < 1: 54 fail("couldn't find first non-arg passed to rewrapper from %s" % str(cmd.args)) 55 if not cfg_file: 56 fail("couldn't find rewrapper cfg file from %s" % str(cmd.args)) 57 58 # Config options are the lowest prioity. 59 rw_opts = rewrapper_cfg.parse(ctx, cfg_file) 60 61 # TODO: Read RBE_* envvars. 62 if runtime.os == "windows": 63 # Experimenting if longer timeouts resolve slow Windows developer builds. b/335525655 64 rw_opts.update({ 65 "exec_timeout": "4m", 66 "reclient_timeout": "8m", 67 }) 68 69 # Command line options are the highest priority. 70 rw_opts.update(rw_cmd_opts) 71 return cmd.args[wrapped_command_pos:], rw_opts, True 72 73def __parse_cros_rewrapper_cmdline(ctx, cmd): 74 # fix cros sdk clang command line and extract rewrapper cfg. 75 # Example command: 76 # ../../build/cros_cache/chrome-sdk/symlinks/amd64-generic+15629.0.0+target_toolchain/bin/x86_64-cros-linux-gnu-clang++ 77 # -MMD -MF obj/third_party/abseil-cpp/absl/base/base/spinlock.o.d 78 # ... 79 # --rewrapper-path /usr/local/google/home/ukai/src/chromium/src/build/args/chromeos/rewrapper_amd64-generic 80 # --rewrapper-cfg ../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 81 # -pipe -march=x86-64 -msse3 ... 82 cfg_file = None 83 skip = "" 84 args = [] 85 toolchainpath = None 86 for i, arg in enumerate(cmd.args): 87 if i == 0: 88 toolchainpath = path.dir(path.dir(ctx.fs.canonpath(arg))) 89 args.append(arg) 90 continue 91 if skip: 92 if skip == "--rewrapper-cfg": 93 cfg_file = ctx.fs.canonpath(arg) 94 skip = "" 95 continue 96 if arg in ("--rewrapper-path", "--rewrapper-cfg"): 97 skip = arg 98 continue 99 args.append(arg) 100 if not cfg_file: 101 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 102 rwcfg = rewrapper_cfg.parse(ctx, cfg_file) 103 inputs = rwcfg.get("inputs", []) 104 inputs.extend([ 105 path.join(toolchainpath, "bin"), 106 path.join(toolchainpath, "lib"), 107 path.join(toolchainpath, "usr/bin"), 108 path.join(toolchainpath, "usr/lib64/clang"), 109 # TODO: b/320189180 - Simple Chrome builds should use libraries under usr/lib64. 110 # But, Ninja/Reclient also don't use them unexpectedly. 111 ]) 112 rwcfg["inputs"] = inputs 113 rwcfg["preserve_symlinks"] = True 114 return args, rwcfg 115 116# TODO(b/278225415): change gn so this wrapper (and by extension this handler) becomes unnecessary. 117def __parse_clang_code_coverage_wrapper_cmdline(ctx, cmd): 118 # Example command: 119 # python3 120 # ../../build/toolchain/clang_code_coverage_wrapper.py 121 # --target-os=... 122 # --files_to_instrument=... 123 # ../../buildtools/reclient/rewrapper 124 # -cfg=../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 125 # -inputs=build/config/unsafe_buffers_paths.txt 126 # -exec_root=/path/to/your/chromium/src/ 127 # ../../third_party/llvm-build/Release+Asserts/bin/clang++ 128 # [rest of clang args] 129 # We don't need to care about: 130 # most args to clang_code_coverage_wrapper (need --files_to_instrument as tool_input) 131 # -exec_root: Siso already knows this. 132 rewrapper_pos = -1 133 wrapped_command_pos = -1 134 cfg_file = None 135 skip = None 136 rw_ops = {} 137 for i, arg in enumerate(cmd.args): 138 if i < 2: 139 continue 140 if rewrapper_pos == -1 and not arg.startswith("-"): 141 rewrapper_pos = i 142 continue 143 if rewrapper_pos > 0 and arg.startswith("-cfg="): 144 cfg_file = ctx.fs.canonpath(arg.removeprefix("-cfg=")) 145 continue 146 if arg.startswith("-inputs=") or skip == "-inputs": 147 rw_ops["inputs"] = arg.removeprefix("-inputs=").split(",") 148 skip = "" 149 continue 150 if arg == "-inputs": 151 skip = arg 152 continue 153 if rewrapper_pos > 0 and not arg.startswith("-"): 154 wrapped_command_pos = i 155 break 156 if rewrapper_pos < 1: 157 fail("couldn't find rewrapper in %s" % str(cmd.args)) 158 if wrapped_command_pos < 1: 159 fail("couldn't find first non-arg passed to rewrapper for %s" % str(cmd.args)) 160 if not cfg_file: 161 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 162 coverage_wrapper_command = cmd.args[:rewrapper_pos] + cmd.args[wrapped_command_pos:] 163 clang_command = clang_code_coverage_wrapper.run(ctx, list(coverage_wrapper_command)) 164 if len(clang_command) > 1 and "/chrome-sdk/" in clang_command[0]: 165 # TODO: implement cros sdk support under code coverage wrapper 166 fail("need to fix handler for cros sdk under code coverage wrapper") 167 rw_cfg_opts = rewrapper_cfg.parse(ctx, cfg_file) 168 169 # Command line options have higher priority than the ones in the cfg file. 170 rw_cfg_opts.update(rw_ops) 171 return clang_command, rw_cfg_opts 172 173def __rewrite_rewrapper(ctx, cmd, use_large = False): 174 # If clang-coverage, needs different handling. 175 if len(cmd.args) > 2 and "clang_code_coverage_wrapper.py" in cmd.args[1]: 176 args, rwcfg = __parse_clang_code_coverage_wrapper_cmdline(ctx, cmd) 177 elif len(cmd.args) > 1 and "/chrome-sdk/" in cmd.args[0]: 178 args, rwcfg = __parse_cros_rewrapper_cmdline(ctx, cmd) 179 else: 180 # handling for generic rewrapper. 181 args, rwcfg, wrapped = __parse_rewrapper_cmdline(ctx, cmd) 182 if not wrapped: 183 print("command doesn't have rewrapper. %s" % str(cmd.args)) 184 return 185 if not rwcfg: 186 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 187 if use_large: 188 platform = rwcfg.get("platform", {}) 189 if platform.get("OSFamily") == "Windows": 190 # Since there is no large Windows workers, it needs to run locally. 191 ctx.actions.fix(args = args) 192 return 193 if platform: 194 action_key = None 195 for key in rwcfg["platform"]: 196 if key.startswith("label:action_"): 197 action_key = key 198 break 199 if action_key: 200 rwcfg["platform"].pop(action_key) 201 else: 202 rwcfg["platform"] = {} 203 rwcfg["platform"].update({ 204 "label:action_large": "1", 205 }) 206 207 # Some large compiles take longer than the default timeout 2m. 208 rwcfg["exec_timeout"] = "4m" 209 rwcfg["reclient_timeout"] = "4m" 210 ctx.actions.fix( 211 args = args, 212 reproxy_config = json.encode(rwcfg), 213 ) 214 215def __rewrite_rewrapper_large(ctx, cmd): 216 return __rewrite_rewrapper(ctx, cmd, use_large = True) 217 218def __strip_rewrapper(ctx, cmd): 219 # If clang-coverage, needs different handling. 220 if len(cmd.args) > 2 and "clang_code_coverage_wrapper.py" in cmd.args[1]: 221 args, _ = __parse_clang_code_coverage_wrapper_cmdline(ctx, cmd) 222 else: 223 args, _, wrapped = __parse_rewrapper_cmdline(ctx, cmd) 224 if not wrapped: 225 print("command doesn't have rewrapper. %s" % str(cmd.args)) 226 return 227 ctx.actions.fix(args = args) 228 229__handlers = { 230 "rewrite_rewrapper": __rewrite_rewrapper, 231 "rewrite_rewrapper_large": __rewrite_rewrapper_large, 232 "strip_rewrapper": __strip_rewrapper, 233} 234 235def __use_reclient(ctx): 236 use_remoteexec = False 237 use_reclient = None 238 if "args.gn" in ctx.metadata: 239 gn_args = gn.args(ctx) 240 if gn_args.get("use_remoteexec") == "true": 241 use_remoteexec = True 242 if gn_args.get("use_reclient") == "false": 243 use_reclient = False 244 if use_reclient == None: 245 use_reclient = use_remoteexec 246 return use_reclient 247 248def __step_config(ctx, step_config): 249 # New rules to convert commands calling rewrapper to use reproxy instead. 250 new_rules = [] 251 252 # Disable racing on builders since bots don't have many CPU cores. 253 # TODO: b/297807325 - Siso wants to handle local execution. 254 # However, Reclient's alerts require racing and local fallback to be 255 # done on Reproxy side. 256 exec_strategy = "racing" 257 if config.get(ctx, "builder"): 258 exec_strategy = "remote_local_fallback" 259 260 for rule in step_config["rules"]: 261 # Replace nacl-clang/clang++ rules without command_prefix, because they will incorrectly match rewrapper. 262 # Replace the original step rule with one that only rewrites rewrapper and convert its rewrapper config to reproxy config. 263 if rule["name"].find("nacl-clang") >= 0 and not rule.get("command_prefix"): 264 new_rule = { 265 "name": rule["name"], 266 "action": rule["action"], 267 "handler": "rewrite_rewrapper", 268 } 269 new_rules.append(new_rule) 270 continue 271 272 # clang cxx/cc/objcxx/objc will always have rewrapper config when use_remoteexec=true. 273 # Remove the native siso handling and replace with custom rewrapper-specific handling. 274 # All other rule values are not reused, instead use rewrapper config via handler. 275 # (In particular, command_prefix should be avoided because it will be rewrapper.) 276 if (rule["name"].startswith("clang/cxx") or rule["name"].startswith("clang/cc") or 277 rule["name"].startswith("clang-cl/cxx") or rule["name"].startswith("clang-cl/cc") or 278 rule["name"].startswith("clang/objc")): 279 if not rule.get("action"): 280 fail("clang rule %s found without action" % rule["name"]) 281 282 new_rule = { 283 "name": rule["name"], 284 "action": rule["action"], 285 "exclude_input_patterns": rule.get("exclude_input_patterns"), 286 "handler": "rewrite_rewrapper", 287 "input_root_absolute_path": rule.get("input_root_absolute_path"), 288 } 289 new_rules.append(new_rule) 290 continue 291 292 # clang-coverage/ is handled by the rewrite_rewrapper handler of clang/{cxx, cc} action rules above, so ignore these rules. 293 if rule["name"].startswith("clang-coverage/"): 294 continue 295 296 # Add non-remote rules as-is. 297 if not rule.get("remote"): 298 new_rules.append(rule) 299 continue 300 301 # Finally handle remaining remote rules. It's assumed it is enough to only convert native remote config to reproxy config. 302 platform_ref = rule.get("platform_ref") 303 if platform_ref: 304 p = step_config["platforms"].get(platform_ref) 305 if not p: 306 fail("Rule %s uses undefined platform '%s'" % (rule["name"], platform_ref)) 307 else: 308 p = step_config.get("platforms", {}).get("default") 309 if not p: 310 fail("Rule %s did not set platform_ref but no default platform exists" % rule["name"]) 311 rule["reproxy_config"] = { 312 "platform": p, 313 "labels": { 314 "type": "tool", 315 "siso_rule": rule["name"], 316 }, 317 "canonicalize_working_dir": rule.get("canonicalize_dir", False), 318 "exec_strategy": exec_strategy, 319 "exec_timeout": rule.get("timeout", "10m"), 320 "reclient_timeout": rule.get("timeout", "10m"), 321 "download_outputs": True, 322 } 323 new_rules.append(rule) 324 325 step_config["rules"] = new_rules 326 return step_config 327 328reproxy = module( 329 "reproxy", 330 enabled = __use_reclient, 331 step_config = __step_config, 332 filegroups = __filegroups, 333 handlers = __handlers, 334) 335