1#!/usr/bin/env python3 2 3# Copyright 2021 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7import argparse 8import pathlib 9import subprocess 10import os 11import sys 12import re 13 14# Set up path to be able to import action_helpers. 15sys.path.append( 16 os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 17 os.pardir, 'build')) 18import action_helpers 19 20# This script wraps rustc for (currently) these reasons: 21# * To work around some ldflags escaping performed by ninja/gn 22# * To remove dependencies on some environment variables from the .d file. 23# * To enable use of .rsp files. 24# * To work around two gn bugs on Windows 25# 26# LDFLAGS ESCAPING 27# 28# This script performs a simple function to work around some of the 29# parameter escaping performed by ninja/gn. 30# 31# rustc invocations are given access to {{rustflags}} and {{ldflags}}. 32# We want to pass {{ldflags}} into rustc, using -Clink-args="{{ldflags}}". 33# Unfortunately, ninja assumes that each item in {{ldflags}} is an 34# independent command-line argument and will have escaped them appropriately 35# for use on a bare command line, instead of in a string. 36# 37# This script converts such {{ldflags}} into individual -Clink-arg=X 38# arguments to rustc. 39# 40# RUSTENV dependency stripping 41# 42# When Rust code depends on an environment variable at build-time 43# (using the env! macro), rustc spots that and adds it to the .d file. 44# Ninja then parses that .d file and determines that the environment 45# dependency means that the target always needs to be rebuilt. 46# 47# That's all correct, but _we_ know that some of these environment 48# variables (typically, all of them) are set by .gn files which ninja 49# tracks independently. So we remove them from the .d file. 50# 51# RSP files: 52# 53# We want to put the ninja/gn variables {{rustdeps}} and {{externs}} 54# in an RSP file. Unfortunately, they are space-separated variables 55# but Rust requires a newline-separated input. This script duly makes 56# the adjustment. This works around a gn issue: 57# TODO(https://bugs.chromium.org/p/gn/issues/detail?id=249): fix this 58# 59# WORKAROUND WINDOWS BUGS: 60# 61# On Windows platforms, this temporarily works around some issues in gn. 62# See comments inline, linking to the relevant gn fixes. 63# 64# Usage: 65# rustc_wrapper.py --rustc <path to rustc> --depfile <path to .d file> 66# -- <normal rustc args> LDFLAGS {{ldflags}} RUSTENV {{rustenv}} 67# The LDFLAGS token is discarded, and everything after that is converted 68# to being a series of -Clink-arg=X arguments, until or unless RUSTENV 69# is encountered, after which those are interpreted as environment 70# variables to pass to rustc (and which will be removed from the .d file). 71# 72# Both LDFLAGS and RUSTENV **MUST** be specified, in that order, even if 73# the list following them is empty. 74# 75# TODO(https://github.com/rust-lang/rust/issues/73632): avoid using rustc 76# for linking in the first place. Most of our binaries are linked using 77# clang directly, but there are some types of Rust build product which 78# must currently be created by rustc (e.g. unit test executables). As 79# part of support for using non-rustc linkers, we should arrange to extract 80# such functionality from rustc so that we can make all types of binary 81# using our clang toolchain. That will remove the need for most of this 82# script. 83 84FILE_RE = re.compile("[^:]+: (.+)") 85 86 87# Equivalent of python3.9 built-in 88def remove_lib_suffix_from_l_args(text): 89 if text.startswith("-l") and text.endswith(".lib"): 90 return text[:-len(".lib")] 91 return text 92 93 94def verify_inputs(depline, sources, abs_build_root): 95 """Verify everything used by rustc (found in `depline`) was specified in the 96 GN build rule (found in `sources` or `inputs`). 97 98 TODO(danakj): This allows things in `sources` that were not actually used by 99 rustc since third-party packages sources need to be a union of all build 100 configs/platforms for simplicity in generating build rules. For first-party 101 code we could be more strict and reject things in `sources` that were not 102 consumed. 103 """ 104 105 # str.removeprefix() does not exist before python 3.9. 106 def remove_prefix(text, prefix): 107 if text.startswith(prefix): 108 return text[len(prefix):] 109 return text 110 111 def normalize_path(p): 112 return os.path.relpath(os.path.normpath(remove_prefix( 113 p, abs_build_root))).replace('\\', '/') 114 115 # Collect the files that rustc says are needed. 116 found_files = {} 117 m = FILE_RE.match(depline) 118 if m: 119 files = m.group(1) 120 found_files = {normalize_path(f): f for f in files.split()} 121 # Get which ones are not listed in GN. 122 missing_files = found_files.keys() - sources 123 124 if not missing_files: 125 return True 126 127 # The matching did a bunch of path manipulation to get paths relative to the 128 # build dir such that they would match GN. In errors, we will print out the 129 # exact path that rustc produces for easier debugging and writing of stdlib 130 # config rules. 131 for file_files_key in missing_files: 132 gn_type = "sources" if file_files_key.endswith(".rs") else "inputs" 133 print(f'ERROR: file not in GN {gn_type}: {found_files[file_files_key]}', 134 file=sys.stderr) 135 return False 136 137 138def main(): 139 parser = argparse.ArgumentParser() 140 parser.add_argument('--rustc', required=True, type=pathlib.Path) 141 parser.add_argument('--depfile', required=True, type=pathlib.Path) 142 parser.add_argument('--rsp', type=pathlib.Path, required=True) 143 parser.add_argument('--target-windows', action='store_true') 144 parser.add_argument('args', metavar='ARG', nargs='+') 145 146 args = parser.parse_args() 147 148 remaining_args = args.args 149 150 ldflags_separator = remaining_args.index("LDFLAGS") 151 rustenv_separator = remaining_args.index("RUSTENV", ldflags_separator) 152 # Sometimes we duplicate the SOURCES list into the command line for debugging 153 # issues on the bots. 154 try: 155 sources_separator = remaining_args.index("SOURCES", rustenv_separator) 156 except: 157 sources_separator = None 158 rustc_args = remaining_args[:ldflags_separator] 159 ldflags = remaining_args[ldflags_separator + 1:rustenv_separator] 160 rustenv = remaining_args[rustenv_separator + 1:sources_separator] 161 162 abs_build_root = os.getcwd().replace('\\', '/') + '/' 163 is_windows = sys.platform == 'win32' or args.target_windows 164 165 rustc_args.extend(["-Clink-arg=%s" % arg for arg in ldflags]) 166 167 with open(args.rsp) as rspfile: 168 rsp_args = [l.rstrip() for l in rspfile.read().split(' ') if l.rstrip()] 169 170 sources_separator = rsp_args.index("SOURCES") 171 sources = set(rsp_args[sources_separator + 1:]) 172 rsp_args = rsp_args[:sources_separator] 173 174 if is_windows: 175 # Work around for "-l<foo>.lib", where ".lib" suffix is undesirable. 176 # Full fix will come from https://gn-review.googlesource.com/c/gn/+/12480 177 rsp_args = [remove_lib_suffix_from_l_args(arg) for arg in rsp_args] 178 out_rsp = str(args.rsp) + ".rsp" 179 with open(out_rsp, 'w') as rspfile: 180 # rustc needs the rsp file to be separated by newlines. Note that GN 181 # generates the file separated by spaces: 182 # https://bugs.chromium.org/p/gn/issues/detail?id=249, 183 rspfile.write("\n".join(rsp_args)) 184 rustc_args.append(f'@{out_rsp}') 185 186 env = os.environ.copy() 187 fixed_env_vars = [] 188 for item in rustenv: 189 (k, v) = item.split("=", 1) 190 env[k] = v 191 fixed_env_vars.append(k) 192 193 try: 194 r = subprocess.run([args.rustc, *rustc_args], env=env, check=False) 195 finally: 196 os.remove(out_rsp) 197 if r.returncode != 0: 198 sys.exit(r.returncode) 199 200 final_depfile_lines = [] 201 dirty = False 202 with open(args.depfile, encoding="utf-8") as d: 203 # Figure out which lines we want to keep in the depfile. If it's not the 204 # whole file, we will rewrite the file. 205 env_dep_re = re.compile("# env-dep:(.*)=.*") 206 for line in d: 207 m = env_dep_re.match(line) 208 if m and m.group(1) in fixed_env_vars: 209 dirty = True # We want to skip this line. 210 else: 211 final_depfile_lines.append(line) 212 213 # Verify each dependent file is listed in sources/inputs. 214 for line in final_depfile_lines: 215 if not verify_inputs(line, sources, abs_build_root): 216 return 1 217 218 if dirty: # we made a change, let's write out the file 219 with action_helpers.atomic_output(args.depfile) as output: 220 output.write("\n".join(final_depfile_lines).encode("utf-8")) 221 222 223if __name__ == '__main__': 224 sys.exit(main()) 225