Lines Matching +full:generate +full:- +full:env
1 #!/usr/bin/env python3
3 # Use of this source code is governed by a BSD-style license that can be
6 # pylint: disable=line-too-long
13 gs://chromeos-localmirror/distfiles/rust-pgo-{rust_version}-frontend.profdata{s}.tz
15 gs://chromeos-localmirror/distfiles/rust-pgo-{rust_version}-llvm.profdata{s}.tz
22 you need to generate manifests for dev-lang/rust and dev-lang/rust-host before
25 variable SRC_URI in cros-rustc.eclass which refer to profdata files.
30 $ ./pgo_rust.py generate # generate profdata files
31 $ ./pgo_rust.py benchmark-pgo # benchmark with PGO
32 $ ./pgo_rust.py benchmark-nopgo # benchmark without PGO
33 $ ./pgo_rust.py upload-profdata # upload profdata to localmirror
39 gs://chromeos-toolchain-artifacts/rust-pgo/benchmarks/{rust_version}/
46 gs://chromeos-toolchain-artifacts/rust-pgo/crates/{name}-{version}.tar.xz
48 and use `--crate-name` and `--crate-version` to indicate which crate to build
49 to generate profdata (or which crate's generated profdata to use), and
50 `--bench-crate-name` to indicate which crate to build in benchmarks.
61 gs://chromeos-localmirror/distfiles
65 gs://chromeos-localmirror/distfiles/rust-pgo-{rust-version}-llvm.profdata.xz
67 gs://chromeos-localmirror/distfiles/
68 rust-pgo-{rust-version}-frontend.profdata.xz
70 But we can store other data elsewhere, like gs://chromeos-toolchain-artifacts.
74 {GS_BASE}/crates/ - store crates we may use for generating profiles or
78 {bench_crate_name}-{bench_crate_version}-{triple}
80 {GS_BASE}/benchmarks/{rust_version}/{crate_name}-{crate_version}/
81 {bench_crate_name}-{bench_crate_version}-{triple}
87 {LOCAL_BASE}/llvm-profraw/
89 {LOCAL_BASE}/frontend-profraw/
91 {LOCAL_BASE}/profdata/{crate_name}-{crate_version}/llvm.profdata
93 {LOCAL_BASE}/profdata/{crate_name}-{crate_version}/frontend.profdata
96 {bench_crate_name}-{bench_crate_version}-{triple}
98 {LOCAL_BASE}/benchmarks/{rust_version}/{crate_name}-{crate_version}/
99 {bench_crate_name}-{bench_crate_version}-{triple}
101 {LOCAL_BASE}/llvm.profdata - must go here to be used by Rust ebuild
102 {LOCAL_BASE}/frontend.profdata - must go here to be used by Rust ebuild
119 "x86_64-cros-linux-gnu",
120 "x86_64-pc-linux-gnu",
121 "armv7a-cros-linux-gnueabihf",
122 "aarch64-cros-linux-gnu",
125 LOCAL_BASE = Path("/tmp/rust-pgo")
127 GS_BASE = PurePosixPath("/chromeos-toolchain-artifacts/rust-pgo")
129 GS_DISTFILES = PurePosixPath("/chromeos-localmirror/distfiles")
150 env: Optional[Mapping[str, str]] = None,
153 ) -> Optional[str]:
156 if env is None:
160 new_env.update(env)
163 if env is None:
166 logging.info("Running %s in environment %s", args, env)
170 env=new_env,
173 encoding="utf-8",
195 if env is None:
198 logging.info("Ran %s in environment %s\n", args, env)
203 def get_command_output(args: List, **kwargs) -> str:
208 def get_rust_version() -> str:
209 s = get_command_output(["rustc", "--version"])
216 filename_no_extension = f"{crate_name}-{crate_version}"
220 local_path / f"{crate_name}-{crate_version}", ignore_errors=True
234 local_path = LOCAL_BASE / "crates" / f"{crate_name}-{crate_version}"
237 with open(".cargo/config.toml", "w", encoding="utf-8") as f:
241 "[source.crates-io]",
242 'replace-with = "vendored-sources"',
244 "[source.vendored-sources]",
248 f'linker = "{target_triple}-clang"',
252 ' "-Clto=thin",',
253 ' "-Cembed-bitcode=yes",',
261 cargo_cmd = ["cargo", "build", "--release", "--target", target_triple]
268 f"--output={time_file}",
269 "--format=wall time (s) %e\nuser time (s) %U\nmax RSS %M\n",
284 "and generate it"
289 ), "Can't generate both frontend and LLVM profile information"
291 use = "-rust_profile_frontend_use -rust_profile_llvm_use "
304 f"cross-{x}/rust" for x in TARGET_TRIPLES if "-pc-linux-" not in x
307 # -E to preserve environment variables like USE, FEATURES, etc.
311 "-E",
313 "-j",
314 "dev-lang/rust-host",
317 env={"USE": use},
324 # The two `llvm-profdata` programs come from different LLVM versions, and
328 "/usr/bin/llvm-profdata"
330 else "/usr/libexec/rust/llvm-profdata"
337 run([llvm_profdata, "merge", f"--output={dest}"] + files)
342 run(["xz", "--keep", "--compress", "--force", source])
350 run(["gsutil", "cp", "-a", "public-read", source, f"gs:/{dest}"])
357 directory = LOCAL_BASE / "crates" / f"{crate_name}-{crate_version}"
367 def generate(args): function
372 llvm_dir = LOCAL_BASE / "llvm-profraw"
374 frontend_dir = LOCAL_BASE / "frontend-profraw"
394 source_directory=LOCAL_BASE / "llvm-profraw",
398 / f"{args.crate_name}-{args.crate_version}"
421 source_directory=LOCAL_BASE / "frontend-profraw",
425 / f"{args.crate_name}-{args.crate_version}"
449 / f"{args.bench_crate_name}-{args.bench_crate_version}-{triple}"
470 "/mnt/host/source/src/third_party/chromiumos-overlay",
471 "dev-lang/rust/files",
481 / f"{args.crate_name}-{args.crate_version}"
493 / f"{args.crate_name}-{args.crate_version}"
504 LOCAL_BASE / "benchmarks" / f"{args.crate_name}-{args.crate_version}"
515 / f"{args.bench_crate_name}-{args.bench_crate_version}-{triple}"
524 / f"{args.crate_name}-{args.crate_version}{args.suffix}"
535 LOCAL_BASE / "profdata" / f"{args.crate_name}-{args.crate_version}"
544 / f"rust-pgo-{rust_version}-llvm{args.suffix}.profdata.xz"
553 / f"rust-pgo-{rust_version}-frontend{args.suffix}.profdata.xz"
558 def main(argv: List[str]) -> int:
572 "generate",
573 help="Generate LLVM and frontend profdata files by building "
577 parser_generate.set_defaults(func=generate)
579 "--crate-name", default=CRATE_NAME, help="Name of the crate to build"
582 "--crate-version",
588 "benchmark-nopgo",
595 "--bench-crate-name",
600 "--bench-crate-version",
605 "--suffix",
612 "benchmark-pgo",
619 "--bench-crate-name",
624 "--bench-crate-version",
629 "--crate-name",
634 "--crate-version",
639 "--suffix",
646 "upload-profdata", help="Upload the profdata files"
650 "--crate-name",
655 "--crate-version",
660 "--suffix",
669 (LOCAL_BASE / "llvm-profraw").mkdir(parents=True, exist_ok=True)
670 (LOCAL_BASE / "frontend-profraw").mkdir(parents=True, exist_ok=True)