1 // Copyright (C) 2022 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 use anyhow::anyhow;
16 use anyhow::bail;
17 use anyhow::Context;
18 use anyhow::Result;
19 use once_cell::sync::Lazy;
20 use regex::Regex;
21 use std::collections::BTreeMap;
22 use std::path::Path;
23 use std::path::PathBuf;
24
25 /// Combined representation of --crate-type and --test flags.
26 #[derive(Debug, PartialEq, Eq)]
27 pub enum CrateType {
28 // --crate-type types
29 Bin,
30 Lib,
31 RLib,
32 DyLib,
33 CDyLib,
34 StaticLib,
35 ProcMacro,
36 // --test
37 Test,
38 // "--cfg test" without --test. (Assume it is a test with the harness disabled.
39 TestNoHarness,
40 }
41
42 /// Info extracted from `CargoOut` for a crate.
43 ///
44 /// Note that there is a 1-to-many relationship between a Cargo.toml file and these `Crate`
45 /// objects. For example, a Cargo.toml file might have a bin, a lib, and various tests. Each of
46 /// those will be a separate `Crate`. All of them will have the same `package_name`.
47 #[derive(Debug, Default)]
48 pub struct Crate {
49 pub name: String,
50 pub package_name: String,
51 pub version: Option<String>,
52 pub types: Vec<CrateType>,
53 pub target: Option<String>, // --target
54 pub features: Vec<String>, // --cfg feature=
55 pub cfgs: Vec<String>, // non-feature --cfg
56 pub externs: Vec<(String, Option<String>)>, // name => rlib file
57 pub codegens: Vec<String>, // -C
58 pub cap_lints: String,
59 pub static_libs: Vec<String>,
60 pub shared_libs: Vec<String>,
61 pub emit_list: String,
62 pub edition: String,
63 pub package_dir: PathBuf, // canonicalized
64 pub main_src: PathBuf, // relative to package_dir
65 }
66
parse_cargo_out(cargo_out_path: &str, cargo_metadata_path: &str) -> Result<Vec<Crate>>67 pub fn parse_cargo_out(cargo_out_path: &str, cargo_metadata_path: &str) -> Result<Vec<Crate>> {
68 let metadata: WorkspaceMetadata = serde_json::from_str(
69 &std::fs::read_to_string(cargo_metadata_path).context("failed to read cargo.metadata")?,
70 )
71 .context("failed to parse cargo.metadata")?;
72
73 let cargo_out = CargoOut::parse(
74 &std::fs::read_to_string(cargo_out_path).context("failed to read cargo.out")?,
75 )
76 .context("failed to parse cargo.out")?;
77
78 assert!(cargo_out.cc_invocations.is_empty(), "cc not supported yet");
79 assert!(cargo_out.ar_invocations.is_empty(), "ar not supported yet");
80
81 let mut crates = Vec::new();
82 for rustc in cargo_out.rustc_invocations.iter() {
83 let c = Crate::from_rustc_invocation(rustc, &metadata)
84 .with_context(|| format!("failed to process rustc invocation: {rustc}"))?;
85 // Ignore build.rs crates.
86 if c.name.starts_with("build_script_") {
87 continue;
88 }
89 // Ignore crates outside the current directory.
90 let cwd = std::env::current_dir().unwrap().canonicalize().unwrap();
91 if !c.package_dir.starts_with(cwd) {
92 continue;
93 }
94 crates.push(c);
95 }
96 Ok(crates)
97 }
98
99 /// `cargo metadata` output.
100 #[derive(serde::Deserialize)]
101 struct WorkspaceMetadata {
102 packages: Vec<PackageMetadata>,
103 }
104
105 #[derive(serde::Deserialize)]
106 struct PackageMetadata {
107 name: String,
108 version: String,
109 edition: String,
110 manifest_path: String,
111 }
112
113 /// Raw-ish data extracted from cargo.out file.
114 #[derive(Debug, Default)]
115 struct CargoOut {
116 rustc_invocations: Vec<String>,
117
118 // package name => cmd args
119 cc_invocations: BTreeMap<String, String>,
120 ar_invocations: BTreeMap<String, String>,
121
122 // lines starting with "warning: ".
123 // line number => line
124 warning_lines: BTreeMap<usize, String>,
125 warning_files: Vec<String>,
126
127 errors: Vec<String>,
128 test_errors: Vec<String>,
129 }
130
match1(regex: &Regex, s: &str) -> Option<String>131 fn match1(regex: &Regex, s: &str) -> Option<String> {
132 regex.captures(s).and_then(|x| x.get(1)).map(|x| x.as_str().to_string())
133 }
134
match3(regex: &Regex, s: &str) -> Option<(String, String, String)>135 fn match3(regex: &Regex, s: &str) -> Option<(String, String, String)> {
136 regex.captures(s).and_then(|x| match (x.get(1), x.get(2), x.get(3)) {
137 (Some(a), Some(b), Some(c)) => {
138 Some((a.as_str().to_string(), b.as_str().to_string(), c.as_str().to_string()))
139 }
140 _ => None,
141 })
142 }
143
144 impl CargoOut {
145 /// Parse the output of a `cargo build -v` run.
parse(contents: &str) -> Result<CargoOut>146 fn parse(contents: &str) -> Result<CargoOut> {
147 let mut result = CargoOut::default();
148 let mut in_tests = false;
149 let mut lines_iter = contents.lines().enumerate();
150 while let Some((n, line)) = lines_iter.next() {
151 if line.starts_with("warning: ") {
152 result.warning_lines.insert(n, line.to_string());
153 continue;
154 }
155
156 // Cargo -v output of a call to rustc.
157 static RUSTC_REGEX: Lazy<Regex> =
158 Lazy::new(|| Regex::new(r"^ +Running `rustc (.*)`$").unwrap());
159 if let Some(args) = match1(&RUSTC_REGEX, line) {
160 result.rustc_invocations.push(args);
161 continue;
162 }
163 // Cargo -vv output of a call to rustc could be split into multiple lines.
164 // Assume that the first line will contain some CARGO_* env definition.
165 static RUSTC_VV_REGEX: Lazy<Regex> =
166 Lazy::new(|| Regex::new(r"^ +Running `.*CARGO_.*=.*$").unwrap());
167 if RUSTC_VV_REGEX.is_match(line) {
168 // cargo build -vv output can have multiple lines for a rustc command due to
169 // '\n' in strings for environment variables.
170 let mut line = line.to_string();
171 loop {
172 // Use an heuristic to detect the completions of a multi-line command.
173 if line.ends_with('`') && line.chars().filter(|c| *c == '`').count() % 2 == 0 {
174 break;
175 }
176 if let Some((_, next_line)) = lines_iter.next() {
177 line += next_line;
178 continue;
179 }
180 break;
181 }
182 // The combined -vv output rustc command line pattern.
183 static RUSTC_VV_CMD_ARGS: Lazy<Regex> =
184 Lazy::new(|| Regex::new(r"^ *Running `.*CARGO_.*=.* rustc (.*)`$").unwrap());
185 if let Some(args) = match1(&RUSTC_VV_CMD_ARGS, &line) {
186 result.rustc_invocations.push(args);
187 } else {
188 bail!("failed to parse cargo.out line: {}", line);
189 }
190 continue;
191 }
192 // Cargo -vv output of a "cc" or "ar" command; all in one line.
193 static CC_AR_VV_REGEX: Lazy<Regex> = Lazy::new(|| {
194 Regex::new(r#"^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$"#).unwrap()
195 });
196 if let Some((pkg, cmd, args)) = match3(&CC_AR_VV_REGEX, line) {
197 match cmd.as_str() {
198 "ar" => result.ar_invocations.insert(pkg, args),
199 "cc" => result.cc_invocations.insert(pkg, args),
200 _ => unreachable!(),
201 };
202 continue;
203 }
204 // Rustc output of file location path pattern for a warning message.
205 static WARNING_FILE_REGEX: Lazy<Regex> =
206 Lazy::new(|| Regex::new(r"^ *--> ([^:]*):[0-9]+").unwrap());
207 if result.warning_lines.contains_key(&n.saturating_sub(1)) {
208 if let Some(fpath) = match1(&WARNING_FILE_REGEX, line) {
209 result.warning_files.push(fpath);
210 continue;
211 }
212 }
213 if line.starts_with("error: ") || line.starts_with("error[E") {
214 if in_tests {
215 result.test_errors.push(line.to_string());
216 } else {
217 result.errors.push(line.to_string());
218 }
219 continue;
220 }
221 static CARGO2ANDROID_RUNNING_REGEX: Lazy<Regex> =
222 Lazy::new(|| Regex::new(r"^### Running: .*$").unwrap());
223 if CARGO2ANDROID_RUNNING_REGEX.is_match(line) {
224 in_tests = line.contains("cargo test") && line.contains("--list");
225 continue;
226 }
227 }
228
229 // self.find_warning_owners()
230
231 Ok(result)
232 }
233 }
234
235 impl CrateType {
from_str(s: &str) -> CrateType236 fn from_str(s: &str) -> CrateType {
237 match s {
238 "bin" => CrateType::Bin,
239 "lib" => CrateType::Lib,
240 "rlib" => CrateType::RLib,
241 "dylib" => CrateType::DyLib,
242 "cdylib" => CrateType::CDyLib,
243 "staticlib" => CrateType::StaticLib,
244 "proc-macro" => CrateType::ProcMacro,
245 _ => panic!("unexpected --crate-type: {}", s),
246 }
247 }
248 }
249
250 impl Crate {
from_rustc_invocation(rustc: &str, metadata: &WorkspaceMetadata) -> Result<Crate>251 fn from_rustc_invocation(rustc: &str, metadata: &WorkspaceMetadata) -> Result<Crate> {
252 let mut out = Crate::default();
253
254 // split into args
255 let args: Vec<&str> = rustc.split_whitespace().collect();
256 let mut arg_iter = args
257 .iter()
258 // Remove quotes from simple strings, panic for others.
259 .map(|arg| match (arg.chars().next(), arg.chars().skip(1).last()) {
260 (Some('"'), Some('"')) => &arg[1..arg.len() - 1],
261 (Some('\''), Some('\'')) => &arg[1..arg.len() - 1],
262 (Some('"'), _) => panic!("can't handle strings with whitespace"),
263 (Some('\''), _) => panic!("can't handle strings with whitespace"),
264 _ => arg,
265 });
266 // process each arg
267 while let Some(arg) = arg_iter.next() {
268 match arg {
269 "--crate-name" => out.name = arg_iter.next().unwrap().to_string(),
270 "--crate-type" => out
271 .types
272 .push(CrateType::from_str(arg_iter.next().unwrap().to_string().as_str())),
273 "--test" => out.types.push(CrateType::Test),
274 "--target" => out.target = Some(arg_iter.next().unwrap().to_string()),
275 "--cfg" => {
276 // example: feature=\"sink\"
277 let arg = arg_iter.next().unwrap();
278 if let Some(feature) =
279 arg.strip_prefix("feature=\"").and_then(|s| s.strip_suffix('\"'))
280 {
281 out.features.push(feature.to_string());
282 } else {
283 out.cfgs.push(arg.to_string());
284 }
285 }
286 "--extern" => {
287 // example: proc_macro
288 // example: memoffset=/some/path/libmemoffset-2cfda327d156e680.rmeta
289 let arg = arg_iter.next().unwrap();
290 if let Some((name, path)) = arg.split_once('=') {
291 out.externs.push((
292 name.to_string(),
293 Some(path.split('/').last().unwrap().to_string()),
294 ));
295 } else {
296 out.externs.push((arg.to_string(), None));
297 }
298 }
299 _ if arg.starts_with("-C") => {
300 // handle both "-Cfoo" and "-C foo"
301 let arg = if arg == "-C" {
302 arg_iter.next().unwrap()
303 } else {
304 arg.strip_prefix("-C").unwrap()
305 };
306 // 'prefer-dynamic' does not work with common flag -C lto
307 // 'embed-bitcode' is ignored; we might control LTO with other .bp flag
308 // 'codegen-units' is set in Android global config or by default
309 //
310 // TODO: this is business logic. move it out of the parsing code
311 if !arg.starts_with("codegen-units=")
312 && !arg.starts_with("debuginfo=")
313 && !arg.starts_with("embed-bitcode=")
314 && !arg.starts_with("extra-filename=")
315 && !arg.starts_with("incremental=")
316 && !arg.starts_with("metadata=")
317 && arg != "prefer-dynamic"
318 {
319 out.codegens.push(arg.to_string());
320 }
321 }
322 "--cap-lints" => out.cap_lints = arg_iter.next().unwrap().to_string(),
323 "-l" => {
324 let arg = arg_iter.next().unwrap();
325 if let Some(lib) = arg.strip_prefix("static=") {
326 out.static_libs.push(lib.to_string());
327 } else if let Some(lib) = arg.strip_prefix("dylib=") {
328 out.shared_libs.push(lib.to_string());
329 } else {
330 out.shared_libs.push(arg.to_string());
331 }
332 }
333 _ if arg.starts_with("--emit=") => {
334 out.emit_list = arg.strip_prefix("--emit=").unwrap().to_string();
335 }
336 _ if !arg.starts_with('-') => {
337 let src_path = Path::new(arg);
338 // Canonicalize the path because:
339 //
340 // 1. We don't consistently get relative or absolute paths elsewhere. If we
341 // canonicalize everything, it becomes easy to compare paths.
342 //
343 // 2. We don't want to consider symlinks to code outside the cwd as part of the
344 // project (e.g. AOSP's import of crosvm has symlinks from crosvm's own 3p
345 // directory to the android 3p directories).
346 let src_path = src_path
347 .canonicalize()
348 .unwrap_or_else(|e| panic!("failed to canonicalize {src_path:?}: {}", e));
349 out.package_dir = src_path.parent().unwrap().to_path_buf();
350 while !out.package_dir.join("Cargo.toml").try_exists()? {
351 if let Some(parent) = out.package_dir.parent() {
352 out.package_dir = parent.to_path_buf();
353 } else {
354 bail!("No Cargo.toml found in parents of {:?}", src_path);
355 }
356 }
357 out.main_src = src_path.strip_prefix(&out.package_dir).unwrap().to_path_buf();
358 }
359
360 // ignored flags
361 "-L" => {
362 arg_iter.next().unwrap();
363 }
364 "--out-dir" => {
365 arg_iter.next().unwrap();
366 }
367 "--color" => {
368 arg_iter.next().unwrap();
369 }
370 _ if arg.starts_with("--error-format=") => {}
371 _ if arg.starts_with("--edition=") => {}
372 _ if arg.starts_with("--json=") => {}
373 _ if arg.starts_with("-Aclippy") => {}
374 _ if arg.starts_with("-Wclippy") => {}
375 "-W" => {}
376 "-D" => {}
377
378 arg => bail!("unsupported rustc argument: {arg:?}"),
379 }
380 }
381
382 if out.name.is_empty() {
383 bail!("missing --crate-name");
384 }
385 if out.main_src.as_os_str().is_empty() {
386 bail!("missing main source file");
387 }
388 // Must have at least one type.
389 if out.types.is_empty() {
390 if out.cfgs.contains(&"test".to_string()) {
391 out.types.push(CrateType::TestNoHarness);
392 } else {
393 bail!("failed to detect crate type. did not have --crate-type or --test or '--cfg test'");
394 }
395 }
396 if out.types.contains(&CrateType::Test) && out.types.len() != 1 {
397 bail!("cannot specify both --test and --crate-type");
398 }
399 if out.types.contains(&CrateType::Lib) && out.types.contains(&CrateType::RLib) {
400 bail!("cannot both have lib and rlib crate types");
401 }
402
403 // Find the metadata for the crates containing package by matching the manifest's path.
404 let manifest_path = out.package_dir.join("Cargo.toml");
405 let package_metadata = metadata
406 .packages
407 .iter()
408 .find(|p| Path::new(&p.manifest_path).canonicalize().unwrap() == manifest_path)
409 .ok_or_else(|| {
410 anyhow!(
411 "can't find metadata for crate {:?} with manifest path {:?}",
412 out.name,
413 manifest_path,
414 )
415 })?;
416 out.package_name = package_metadata.name.clone();
417 out.version = Some(package_metadata.version.clone());
418 out.edition = package_metadata.edition.clone();
419
420 Ok(out)
421 }
422 }
423