1 use super::link::{self, ensure_removed};
2 use super::lto::{self, SerializedModule};
3 use super::symbol_export::symbol_name_for_instance_in_crate;
4
5 use crate::errors;
6 use crate::traits::*;
7 use crate::{
8 CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind,
9 };
10 use jobserver::{Acquired, Client};
11 use rustc_ast::attr;
12 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
13 use rustc_data_structures::memmap::Mmap;
14 use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
15 use rustc_data_structures::sync::Lrc;
16 use rustc_errors::emitter::Emitter;
17 use rustc_errors::{translation::Translate, DiagnosticId, FatalError, Handler, Level};
18 use rustc_errors::{DiagnosticMessage, Style};
19 use rustc_fs_util::link_or_copy;
20 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
21 use rustc_incremental::{
22 copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess,
23 };
24 use rustc_metadata::fs::copy_to_stdout;
25 use rustc_metadata::EncodedMetadata;
26 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
27 use rustc_middle::middle::exported_symbols::SymbolExportInfo;
28 use rustc_middle::ty::TyCtxt;
29 use rustc_session::cgu_reuse_tracker::CguReuseTracker;
30 use rustc_session::config::{self, CrateType, Lto, OutFileName, OutputFilenames, OutputType};
31 use rustc_session::config::{Passes, SwitchWithOptPath};
32 use rustc_session::Session;
33 use rustc_span::source_map::SourceMap;
34 use rustc_span::symbol::sym;
35 use rustc_span::{BytePos, FileName, InnerSpan, Pos, Span};
36 use rustc_target::spec::{MergeFunctions, SanitizerSet};
37
38 use crate::errors::ErrorCreatingRemarkDir;
39 use std::any::Any;
40 use std::borrow::Cow;
41 use std::fs;
42 use std::io;
43 use std::marker::PhantomData;
44 use std::mem;
45 use std::path::{Path, PathBuf};
46 use std::str;
47 use std::sync::mpsc::{channel, Receiver, Sender};
48 use std::sync::Arc;
49 use std::thread;
50
51 const PRE_LTO_BC_EXT: &str = "pre-lto.bc";
52
53 /// What kind of object file to emit.
54 #[derive(Clone, Copy, PartialEq)]
55 pub enum EmitObj {
56 // No object file.
57 None,
58
59 // Just uncompressed llvm bitcode. Provides easy compatibility with
60 // emscripten's ecc compiler, when used as the linker.
61 Bitcode,
62
63 // Object code, possibly augmented with a bitcode section.
64 ObjectCode(BitcodeSection),
65 }
66
67 /// What kind of llvm bitcode section to embed in an object file.
68 #[derive(Clone, Copy, PartialEq)]
69 pub enum BitcodeSection {
70 // No bitcode section.
71 None,
72
73 // A full, uncompressed bitcode section.
74 Full,
75 }
76
77 /// Module-specific configuration for `optimize_and_codegen`.
78 pub struct ModuleConfig {
79 /// Names of additional optimization passes to run.
80 pub passes: Vec<String>,
81 /// Some(level) to optimize at a certain level, or None to run
82 /// absolutely no optimizations (used for the metadata module).
83 pub opt_level: Option<config::OptLevel>,
84
85 /// Some(level) to optimize binary size, or None to not affect program size.
86 pub opt_size: Option<config::OptLevel>,
87
88 pub pgo_gen: SwitchWithOptPath,
89 pub pgo_use: Option<PathBuf>,
90 pub pgo_sample_use: Option<PathBuf>,
91 pub debug_info_for_profiling: bool,
92 pub instrument_coverage: bool,
93 pub instrument_gcov: bool,
94
95 pub sanitizer: SanitizerSet,
96 pub sanitizer_recover: SanitizerSet,
97 pub sanitizer_memory_track_origins: usize,
98
99 // Flags indicating which outputs to produce.
100 pub emit_pre_lto_bc: bool,
101 pub emit_no_opt_bc: bool,
102 pub emit_bc: bool,
103 pub emit_ir: bool,
104 pub emit_asm: bool,
105 pub emit_obj: EmitObj,
106 pub emit_thin_lto: bool,
107 pub bc_cmdline: String,
108
109 // Miscellaneous flags. These are mostly copied from command-line
110 // options.
111 pub verify_llvm_ir: bool,
112 pub no_prepopulate_passes: bool,
113 pub no_builtins: bool,
114 pub time_module: bool,
115 pub vectorize_loop: bool,
116 pub vectorize_slp: bool,
117 pub merge_functions: bool,
118 pub inline_threshold: Option<u32>,
119 pub emit_lifetime_markers: bool,
120 pub llvm_plugins: Vec<String>,
121 }
122
123 impl ModuleConfig {
new( kind: ModuleKind, sess: &Session, no_builtins: bool, is_compiler_builtins: bool, ) -> ModuleConfig124 fn new(
125 kind: ModuleKind,
126 sess: &Session,
127 no_builtins: bool,
128 is_compiler_builtins: bool,
129 ) -> ModuleConfig {
130 // If it's a regular module, use `$regular`, otherwise use `$other`.
131 // `$regular` and `$other` are evaluated lazily.
132 macro_rules! if_regular {
133 ($regular: expr, $other: expr) => {
134 if let ModuleKind::Regular = kind { $regular } else { $other }
135 };
136 }
137
138 let opt_level_and_size = if_regular!(Some(sess.opts.optimize), None);
139
140 let save_temps = sess.opts.cg.save_temps;
141
142 let should_emit_obj = sess.opts.output_types.contains_key(&OutputType::Exe)
143 || match kind {
144 ModuleKind::Regular => sess.opts.output_types.contains_key(&OutputType::Object),
145 ModuleKind::Allocator => false,
146 ModuleKind::Metadata => sess.opts.output_types.contains_key(&OutputType::Metadata),
147 };
148
149 let emit_obj = if !should_emit_obj {
150 EmitObj::None
151 } else if sess.target.obj_is_bitcode
152 || (sess.opts.cg.linker_plugin_lto.enabled() && !no_builtins)
153 {
154 // This case is selected if the target uses objects as bitcode, or
155 // if linker plugin LTO is enabled. In the linker plugin LTO case
156 // the assumption is that the final link-step will read the bitcode
157 // and convert it to object code. This may be done by either the
158 // native linker or rustc itself.
159 //
160 // Note, however, that the linker-plugin-lto requested here is
161 // explicitly ignored for `#![no_builtins]` crates. These crates are
162 // specifically ignored by rustc's LTO passes and wouldn't work if
163 // loaded into the linker. These crates define symbols that LLVM
164 // lowers intrinsics to, and these symbol dependencies aren't known
165 // until after codegen. As a result any crate marked
166 // `#![no_builtins]` is assumed to not participate in LTO and
167 // instead goes on to generate object code.
168 EmitObj::Bitcode
169 } else if need_bitcode_in_object(sess) {
170 EmitObj::ObjectCode(BitcodeSection::Full)
171 } else {
172 EmitObj::ObjectCode(BitcodeSection::None)
173 };
174
175 ModuleConfig {
176 passes: if_regular!(sess.opts.cg.passes.clone(), vec![]),
177
178 opt_level: opt_level_and_size,
179 opt_size: opt_level_and_size,
180
181 pgo_gen: if_regular!(
182 sess.opts.cg.profile_generate.clone(),
183 SwitchWithOptPath::Disabled
184 ),
185 pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None),
186 pgo_sample_use: if_regular!(sess.opts.unstable_opts.profile_sample_use.clone(), None),
187 debug_info_for_profiling: sess.opts.unstable_opts.debug_info_for_profiling,
188 instrument_coverage: if_regular!(sess.instrument_coverage(), false),
189 instrument_gcov: if_regular!(
190 // compiler_builtins overrides the codegen-units settings,
191 // which is incompatible with -Zprofile which requires that
192 // only a single codegen unit is used per crate.
193 sess.opts.unstable_opts.profile && !is_compiler_builtins,
194 false
195 ),
196
197 sanitizer: if_regular!(sess.opts.unstable_opts.sanitizer, SanitizerSet::empty()),
198 sanitizer_recover: if_regular!(
199 sess.opts.unstable_opts.sanitizer_recover,
200 SanitizerSet::empty()
201 ),
202 sanitizer_memory_track_origins: if_regular!(
203 sess.opts.unstable_opts.sanitizer_memory_track_origins,
204 0
205 ),
206
207 emit_pre_lto_bc: if_regular!(
208 save_temps || need_pre_lto_bitcode_for_incr_comp(sess),
209 false
210 ),
211 emit_no_opt_bc: if_regular!(save_temps, false),
212 emit_bc: if_regular!(
213 save_temps || sess.opts.output_types.contains_key(&OutputType::Bitcode),
214 save_temps
215 ),
216 emit_ir: if_regular!(
217 sess.opts.output_types.contains_key(&OutputType::LlvmAssembly),
218 false
219 ),
220 emit_asm: if_regular!(
221 sess.opts.output_types.contains_key(&OutputType::Assembly),
222 false
223 ),
224 emit_obj,
225 emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto,
226 bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(),
227
228 verify_llvm_ir: sess.verify_llvm_ir(),
229 no_prepopulate_passes: sess.opts.cg.no_prepopulate_passes,
230 no_builtins: no_builtins || sess.target.no_builtins,
231
232 // Exclude metadata and allocator modules from time_passes output,
233 // since they throw off the "LLVM passes" measurement.
234 time_module: if_regular!(true, false),
235
236 // Copy what clang does by turning on loop vectorization at O2 and
237 // slp vectorization at O3.
238 vectorize_loop: !sess.opts.cg.no_vectorize_loops
239 && (sess.opts.optimize == config::OptLevel::Default
240 || sess.opts.optimize == config::OptLevel::Aggressive),
241 vectorize_slp: !sess.opts.cg.no_vectorize_slp
242 && sess.opts.optimize == config::OptLevel::Aggressive,
243
244 // Some targets (namely, NVPTX) interact badly with the
245 // MergeFunctions pass. This is because MergeFunctions can generate
246 // new function calls which may interfere with the target calling
247 // convention; e.g. for the NVPTX target, PTX kernels should not
248 // call other PTX kernels. MergeFunctions can also be configured to
249 // generate aliases instead, but aliases are not supported by some
250 // backends (again, NVPTX). Therefore, allow targets to opt out of
251 // the MergeFunctions pass, but otherwise keep the pass enabled (at
252 // O2 and O3) since it can be useful for reducing code size.
253 merge_functions: match sess
254 .opts
255 .unstable_opts
256 .merge_functions
257 .unwrap_or(sess.target.merge_functions)
258 {
259 MergeFunctions::Disabled => false,
260 MergeFunctions::Trampolines | MergeFunctions::Aliases => {
261 use config::OptLevel::*;
262 match sess.opts.optimize {
263 Aggressive | Default | SizeMin | Size => true,
264 Less | No => false,
265 }
266 }
267 },
268
269 inline_threshold: sess.opts.cg.inline_threshold,
270 emit_lifetime_markers: sess.emit_lifetime_markers(),
271 llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]),
272 }
273 }
274
bitcode_needed(&self) -> bool275 pub fn bitcode_needed(&self) -> bool {
276 self.emit_bc
277 || self.emit_obj == EmitObj::Bitcode
278 || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
279 }
280 }
281
282 /// Configuration passed to the function returned by the `target_machine_factory`.
283 pub struct TargetMachineFactoryConfig {
284 /// Split DWARF is enabled in LLVM by checking that `TM.MCOptions.SplitDwarfFile` isn't empty,
285 /// so the path to the dwarf object has to be provided when we create the target machine.
286 /// This can be ignored by backends which do not need it for their Split DWARF support.
287 pub split_dwarf_file: Option<PathBuf>,
288 }
289
290 impl TargetMachineFactoryConfig {
new( cgcx: &CodegenContext<impl WriteBackendMethods>, module_name: &str, ) -> TargetMachineFactoryConfig291 pub fn new(
292 cgcx: &CodegenContext<impl WriteBackendMethods>,
293 module_name: &str,
294 ) -> TargetMachineFactoryConfig {
295 let split_dwarf_file = if cgcx.target_can_use_split_dwarf {
296 cgcx.output_filenames.split_dwarf_path(
297 cgcx.split_debuginfo,
298 cgcx.split_dwarf_kind,
299 Some(module_name),
300 )
301 } else {
302 None
303 };
304 TargetMachineFactoryConfig { split_dwarf_file }
305 }
306 }
307
308 pub type TargetMachineFactoryFn<B> = Arc<
309 dyn Fn(
310 TargetMachineFactoryConfig,
311 ) -> Result<
312 <B as WriteBackendMethods>::TargetMachine,
313 <B as WriteBackendMethods>::TargetMachineError,
314 > + Send
315 + Sync,
316 >;
317
318 pub type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo)>>>;
319
320 /// Additional resources used by optimize_and_codegen (not module specific)
321 #[derive(Clone)]
322 pub struct CodegenContext<B: WriteBackendMethods> {
323 // Resources needed when running LTO
324 pub prof: SelfProfilerRef,
325 pub lto: Lto,
326 pub save_temps: bool,
327 pub fewer_names: bool,
328 pub time_trace: bool,
329 pub exported_symbols: Option<Arc<ExportedSymbols>>,
330 pub opts: Arc<config::Options>,
331 pub crate_types: Vec<CrateType>,
332 pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
333 pub output_filenames: Arc<OutputFilenames>,
334 pub regular_module_config: Arc<ModuleConfig>,
335 pub metadata_module_config: Arc<ModuleConfig>,
336 pub allocator_module_config: Arc<ModuleConfig>,
337 pub tm_factory: TargetMachineFactoryFn<B>,
338 pub msvc_imps_needed: bool,
339 pub is_pe_coff: bool,
340 pub target_can_use_split_dwarf: bool,
341 pub target_arch: String,
342 pub split_debuginfo: rustc_target::spec::SplitDebuginfo,
343 pub split_dwarf_kind: rustc_session::config::SplitDwarfKind,
344
345 /// Handler to use for diagnostics produced during codegen.
346 pub diag_emitter: SharedEmitter,
347 /// LLVM optimizations for which we want to print remarks.
348 pub remark: Passes,
349 /// Directory into which should the LLVM optimization remarks be written.
350 /// If `None`, they will be written to stderr.
351 pub remark_dir: Option<PathBuf>,
352 /// Worker thread number
353 pub worker: usize,
354 /// The incremental compilation session directory, or None if we are not
355 /// compiling incrementally
356 pub incr_comp_session_dir: Option<PathBuf>,
357 /// Used to update CGU re-use information during the thinlto phase.
358 pub cgu_reuse_tracker: CguReuseTracker,
359 /// Channel back to the main control thread to send messages to
360 pub coordinator_send: Sender<Box<dyn Any + Send>>,
361 }
362
363 impl<B: WriteBackendMethods> CodegenContext<B> {
create_diag_handler(&self) -> Handler364 pub fn create_diag_handler(&self) -> Handler {
365 Handler::with_emitter(true, None, Box::new(self.diag_emitter.clone()))
366 }
367
config(&self, kind: ModuleKind) -> &ModuleConfig368 pub fn config(&self, kind: ModuleKind) -> &ModuleConfig {
369 match kind {
370 ModuleKind::Regular => &self.regular_module_config,
371 ModuleKind::Metadata => &self.metadata_module_config,
372 ModuleKind::Allocator => &self.allocator_module_config,
373 }
374 }
375 }
376
generate_lto_work<B: ExtraBackendMethods>( cgcx: &CodegenContext<B>, needs_fat_lto: Vec<FatLTOInput<B>>, needs_thin_lto: Vec<(String, B::ThinBuffer)>, import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>, ) -> Vec<(WorkItem<B>, u64)>377 fn generate_lto_work<B: ExtraBackendMethods>(
378 cgcx: &CodegenContext<B>,
379 needs_fat_lto: Vec<FatLTOInput<B>>,
380 needs_thin_lto: Vec<(String, B::ThinBuffer)>,
381 import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
382 ) -> Vec<(WorkItem<B>, u64)> {
383 let _prof_timer = cgcx.prof.generic_activity("codegen_generate_lto_work");
384
385 let (lto_modules, copy_jobs) = if !needs_fat_lto.is_empty() {
386 assert!(needs_thin_lto.is_empty());
387 let lto_module =
388 B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
389 (vec![lto_module], vec![])
390 } else {
391 assert!(needs_fat_lto.is_empty());
392 B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules).unwrap_or_else(|e| e.raise())
393 };
394
395 lto_modules
396 .into_iter()
397 .map(|module| {
398 let cost = module.cost();
399 (WorkItem::LTO(module), cost)
400 })
401 .chain(copy_jobs.into_iter().map(|wp| {
402 (
403 WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen {
404 name: wp.cgu_name.clone(),
405 source: wp,
406 }),
407 0,
408 )
409 }))
410 .collect()
411 }
412
413 pub struct CompiledModules {
414 pub modules: Vec<CompiledModule>,
415 pub allocator_module: Option<CompiledModule>,
416 }
417
need_bitcode_in_object(sess: &Session) -> bool418 fn need_bitcode_in_object(sess: &Session) -> bool {
419 let requested_for_rlib = sess.opts.cg.embed_bitcode
420 && sess.crate_types().contains(&CrateType::Rlib)
421 && sess.opts.output_types.contains_key(&OutputType::Exe);
422 let forced_by_target = sess.target.forces_embed_bitcode;
423 requested_for_rlib || forced_by_target
424 }
425
need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool426 fn need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool {
427 if sess.opts.incremental.is_none() {
428 return false;
429 }
430
431 match sess.lto() {
432 Lto::No => false,
433 Lto::Fat | Lto::Thin | Lto::ThinLocal => true,
434 }
435 }
436
start_async_codegen<B: ExtraBackendMethods>( backend: B, tcx: TyCtxt<'_>, target_cpu: String, metadata: EncodedMetadata, metadata_module: Option<CompiledModule>, ) -> OngoingCodegen<B>437 pub fn start_async_codegen<B: ExtraBackendMethods>(
438 backend: B,
439 tcx: TyCtxt<'_>,
440 target_cpu: String,
441 metadata: EncodedMetadata,
442 metadata_module: Option<CompiledModule>,
443 ) -> OngoingCodegen<B> {
444 let (coordinator_send, coordinator_receive) = channel();
445 let sess = tcx.sess;
446
447 let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
448 let no_builtins = attr::contains_name(crate_attrs, sym::no_builtins);
449 let is_compiler_builtins = attr::contains_name(crate_attrs, sym::compiler_builtins);
450
451 let crate_info = CrateInfo::new(tcx, target_cpu);
452
453 let regular_config =
454 ModuleConfig::new(ModuleKind::Regular, sess, no_builtins, is_compiler_builtins);
455 let metadata_config =
456 ModuleConfig::new(ModuleKind::Metadata, sess, no_builtins, is_compiler_builtins);
457 let allocator_config =
458 ModuleConfig::new(ModuleKind::Allocator, sess, no_builtins, is_compiler_builtins);
459
460 let (shared_emitter, shared_emitter_main) = SharedEmitter::new();
461 let (codegen_worker_send, codegen_worker_receive) = channel();
462
463 let coordinator_thread = start_executing_work(
464 backend.clone(),
465 tcx,
466 &crate_info,
467 shared_emitter,
468 codegen_worker_send,
469 coordinator_receive,
470 sess.jobserver.clone(),
471 Arc::new(regular_config),
472 Arc::new(metadata_config),
473 Arc::new(allocator_config),
474 coordinator_send.clone(),
475 );
476
477 OngoingCodegen {
478 backend,
479 metadata,
480 metadata_module,
481 crate_info,
482
483 codegen_worker_receive,
484 shared_emitter_main,
485 coordinator: Coordinator {
486 sender: coordinator_send,
487 future: Some(coordinator_thread),
488 phantom: PhantomData,
489 },
490 output_filenames: tcx.output_filenames(()).clone(),
491 }
492 }
493
copy_all_cgu_workproducts_to_incr_comp_cache_dir( sess: &Session, compiled_modules: &CompiledModules, ) -> FxIndexMap<WorkProductId, WorkProduct>494 fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
495 sess: &Session,
496 compiled_modules: &CompiledModules,
497 ) -> FxIndexMap<WorkProductId, WorkProduct> {
498 let mut work_products = FxIndexMap::default();
499
500 if sess.opts.incremental.is_none() {
501 return work_products;
502 }
503
504 let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir");
505
506 for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) {
507 let mut files = Vec::new();
508 if let Some(object_file_path) = &module.object {
509 files.push(("o", object_file_path.as_path()));
510 }
511 if let Some(dwarf_object_file_path) = &module.dwarf_object {
512 files.push(("dwo", dwarf_object_file_path.as_path()));
513 }
514
515 if let Some((id, product)) =
516 copy_cgu_workproduct_to_incr_comp_cache_dir(sess, &module.name, files.as_slice())
517 {
518 work_products.insert(id, product);
519 }
520 }
521
522 work_products
523 }
524
produce_final_output_artifacts( sess: &Session, compiled_modules: &CompiledModules, crate_output: &OutputFilenames, )525 fn produce_final_output_artifacts(
526 sess: &Session,
527 compiled_modules: &CompiledModules,
528 crate_output: &OutputFilenames,
529 ) {
530 let mut user_wants_bitcode = false;
531 let mut user_wants_objects = false;
532
533 // Produce final compile outputs.
534 let copy_gracefully = |from: &Path, to: &OutFileName| match to {
535 OutFileName::Stdout => {
536 if let Err(e) = copy_to_stdout(from) {
537 sess.emit_err(errors::CopyPath::new(from, to.as_path(), e));
538 }
539 }
540 OutFileName::Real(path) => {
541 if let Err(e) = fs::copy(from, path) {
542 sess.emit_err(errors::CopyPath::new(from, path, e));
543 }
544 }
545 };
546
547 let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| {
548 if compiled_modules.modules.len() == 1 {
549 // 1) Only one codegen unit. In this case it's no difficulty
550 // to copy `foo.0.x` to `foo.x`.
551 let module_name = Some(&compiled_modules.modules[0].name[..]);
552 let path = crate_output.temp_path(output_type, module_name);
553 let output = crate_output.path(output_type);
554 if !output_type.is_text_output() && output.is_tty() {
555 sess.emit_err(errors::BinaryOutputToTty { shorthand: output_type.shorthand() });
556 } else {
557 copy_gracefully(&path, &output);
558 }
559 if !sess.opts.cg.save_temps && !keep_numbered {
560 // The user just wants `foo.x`, not `foo.#module-name#.x`.
561 ensure_removed(sess.diagnostic(), &path);
562 }
563 } else {
564 let extension = crate_output
565 .temp_path(output_type, None)
566 .extension()
567 .unwrap()
568 .to_str()
569 .unwrap()
570 .to_owned();
571
572 if crate_output.outputs.contains_key(&output_type) {
573 // 2) Multiple codegen units, with `--emit foo=some_name`. We have
574 // no good solution for this case, so warn the user.
575 sess.emit_warning(errors::IgnoringEmitPath { extension });
576 } else if crate_output.single_output_file.is_some() {
577 // 3) Multiple codegen units, with `-o some_name`. We have
578 // no good solution for this case, so warn the user.
579 sess.emit_warning(errors::IgnoringOutput { extension });
580 } else {
581 // 4) Multiple codegen units, but no explicit name. We
582 // just leave the `foo.0.x` files in place.
583 // (We don't have to do any work in this case.)
584 }
585 }
586 };
587
588 // Flag to indicate whether the user explicitly requested bitcode.
589 // Otherwise, we produced it only as a temporary output, and will need
590 // to get rid of it.
591 for output_type in crate_output.outputs.keys() {
592 match *output_type {
593 OutputType::Bitcode => {
594 user_wants_bitcode = true;
595 // Copy to .bc, but always keep the .0.bc. There is a later
596 // check to figure out if we should delete .0.bc files, or keep
597 // them for making an rlib.
598 copy_if_one_unit(OutputType::Bitcode, true);
599 }
600 OutputType::LlvmAssembly => {
601 copy_if_one_unit(OutputType::LlvmAssembly, false);
602 }
603 OutputType::Assembly => {
604 copy_if_one_unit(OutputType::Assembly, false);
605 }
606 OutputType::Object => {
607 user_wants_objects = true;
608 copy_if_one_unit(OutputType::Object, true);
609 }
610 OutputType::Mir | OutputType::Metadata | OutputType::Exe | OutputType::DepInfo => {}
611 }
612 }
613
614 // Clean up unwanted temporary files.
615
616 // We create the following files by default:
617 // - #crate#.#module-name#.bc
618 // - #crate#.#module-name#.o
619 // - #crate#.crate.metadata.bc
620 // - #crate#.crate.metadata.o
621 // - #crate#.o (linked from crate.##.o)
622 // - #crate#.bc (copied from crate.##.bc)
623 // We may create additional files if requested by the user (through
624 // `-C save-temps` or `--emit=` flags).
625
626 if !sess.opts.cg.save_temps {
627 // Remove the temporary .#module-name#.o objects. If the user didn't
628 // explicitly request bitcode (with --emit=bc), and the bitcode is not
629 // needed for building an rlib, then we must remove .#module-name#.bc as
630 // well.
631
632 // Specific rules for keeping .#module-name#.bc:
633 // - If the user requested bitcode (`user_wants_bitcode`), and
634 // codegen_units > 1, then keep it.
635 // - If the user requested bitcode but codegen_units == 1, then we
636 // can toss .#module-name#.bc because we copied it to .bc earlier.
637 // - If we're not building an rlib and the user didn't request
638 // bitcode, then delete .#module-name#.bc.
639 // If you change how this works, also update back::link::link_rlib,
640 // where .#module-name#.bc files are (maybe) deleted after making an
641 // rlib.
642 let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe);
643
644 let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units().as_usize() > 1;
645
646 let keep_numbered_objects =
647 needs_crate_object || (user_wants_objects && sess.codegen_units().as_usize() > 1);
648
649 for module in compiled_modules.modules.iter() {
650 if let Some(ref path) = module.object {
651 if !keep_numbered_objects {
652 ensure_removed(sess.diagnostic(), path);
653 }
654 }
655
656 if let Some(ref path) = module.dwarf_object {
657 if !keep_numbered_objects {
658 ensure_removed(sess.diagnostic(), path);
659 }
660 }
661
662 if let Some(ref path) = module.bytecode {
663 if !keep_numbered_bitcode {
664 ensure_removed(sess.diagnostic(), path);
665 }
666 }
667 }
668
669 if !user_wants_bitcode {
670 if let Some(ref allocator_module) = compiled_modules.allocator_module {
671 if let Some(ref path) = allocator_module.bytecode {
672 ensure_removed(sess.diagnostic(), path);
673 }
674 }
675 }
676 }
677
678 // We leave the following files around by default:
679 // - #crate#.o
680 // - #crate#.crate.metadata.o
681 // - #crate#.bc
682 // These are used in linking steps and will be cleaned up afterward.
683 }
684
685 pub(crate) enum WorkItem<B: WriteBackendMethods> {
686 /// Optimize a newly codegened, totally unoptimized module.
687 Optimize(ModuleCodegen<B::Module>),
688 /// Copy the post-LTO artifacts from the incremental cache to the output
689 /// directory.
690 CopyPostLtoArtifacts(CachedModuleCodegen),
691 /// Performs (Thin)LTO on the given module.
692 LTO(lto::LtoModuleCodegen<B>),
693 }
694
695 impl<B: WriteBackendMethods> WorkItem<B> {
module_kind(&self) -> ModuleKind696 pub fn module_kind(&self) -> ModuleKind {
697 match *self {
698 WorkItem::Optimize(ref m) => m.kind,
699 WorkItem::CopyPostLtoArtifacts(_) | WorkItem::LTO(_) => ModuleKind::Regular,
700 }
701 }
702
703 /// Generate a short description of this work item suitable for use as a thread name.
short_description(&self) -> String704 fn short_description(&self) -> String {
705 // `pthread_setname()` on *nix ignores anything beyond the first 15
706 // bytes. Use short descriptions to maximize the space available for
707 // the module name.
708 #[cfg(not(windows))]
709 fn desc(short: &str, _long: &str, name: &str) -> String {
710 // The short label is three bytes, and is followed by a space. That
711 // leaves 11 bytes for the CGU name. How we obtain those 11 bytes
712 // depends on the the CGU name form.
713 //
714 // - Non-incremental, e.g. `regex.f10ba03eb5ec7975-cgu.0`: the part
715 // before the `-cgu.0` is the same for every CGU, so use the
716 // `cgu.0` part. The number suffix will be different for each
717 // CGU.
718 //
719 // - Incremental (normal), e.g. `2i52vvl2hco29us0`: use the whole
720 // name because each CGU will have a unique ASCII hash, and the
721 // first 11 bytes will be enough to identify it.
722 //
723 // - Incremental (with `-Zhuman-readable-cgu-names`), e.g.
724 // `regex.f10ba03eb5ec7975-re_builder.volatile`: use the whole
725 // name. The first 11 bytes won't be enough to uniquely identify
726 // it, but no obvious substring will, and this is a rarely used
727 // option so it doesn't matter much.
728 //
729 assert_eq!(short.len(), 3);
730 let name = if let Some(index) = name.find("-cgu.") {
731 &name[index + 1..] // +1 skips the leading '-'.
732 } else {
733 name
734 };
735 format!("{short} {name}")
736 }
737
738 // Windows has no thread name length limit, so use more descriptive names.
739 #[cfg(windows)]
740 fn desc(_short: &str, long: &str, name: &str) -> String {
741 format!("{long} {name}")
742 }
743
744 match self {
745 WorkItem::Optimize(m) => desc("opt", "optimize module {}", &m.name),
746 WorkItem::CopyPostLtoArtifacts(m) => desc("cpy", "copy LTO artifacts for {}", &m.name),
747 WorkItem::LTO(m) => desc("lto", "LTO module {}", m.name()),
748 }
749 }
750 }
751
752 /// A result produced by the backend.
753 pub(crate) enum WorkItemResult<B: WriteBackendMethods> {
754 Compiled(CompiledModule),
755 NeedsLink(ModuleCodegen<B::Module>),
756 NeedsFatLTO(FatLTOInput<B>),
757 NeedsThinLTO(String, B::ThinBuffer),
758 }
759
760 pub enum FatLTOInput<B: WriteBackendMethods> {
761 Serialized { name: String, buffer: B::ModuleBuffer },
762 InMemory(ModuleCodegen<B::Module>),
763 }
764
765 /// Actual LTO type we end up choosing based on multiple factors.
766 pub enum ComputedLtoType {
767 No,
768 Thin,
769 Fat,
770 }
771
compute_per_cgu_lto_type( sess_lto: &Lto, opts: &config::Options, sess_crate_types: &[CrateType], module_kind: ModuleKind, ) -> ComputedLtoType772 pub fn compute_per_cgu_lto_type(
773 sess_lto: &Lto,
774 opts: &config::Options,
775 sess_crate_types: &[CrateType],
776 module_kind: ModuleKind,
777 ) -> ComputedLtoType {
778 // Metadata modules never participate in LTO regardless of the lto
779 // settings.
780 if module_kind == ModuleKind::Metadata {
781 return ComputedLtoType::No;
782 }
783
784 // If the linker does LTO, we don't have to do it. Note that we
785 // keep doing full LTO, if it is requested, as not to break the
786 // assumption that the output will be a single module.
787 let linker_does_lto = opts.cg.linker_plugin_lto.enabled();
788
789 // When we're automatically doing ThinLTO for multi-codegen-unit
790 // builds we don't actually want to LTO the allocator modules if
791 // it shows up. This is due to various linker shenanigans that
792 // we'll encounter later.
793 let is_allocator = module_kind == ModuleKind::Allocator;
794
795 // We ignore a request for full crate graph LTO if the crate type
796 // is only an rlib, as there is no full crate graph to process,
797 // that'll happen later.
798 //
799 // This use case currently comes up primarily for targets that
800 // require LTO so the request for LTO is always unconditionally
801 // passed down to the backend, but we don't actually want to do
802 // anything about it yet until we've got a final product.
803 let is_rlib = sess_crate_types.len() == 1 && sess_crate_types[0] == CrateType::Rlib;
804
805 match sess_lto {
806 Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin,
807 Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin,
808 Lto::Fat if !is_rlib => ComputedLtoType::Fat,
809 _ => ComputedLtoType::No,
810 }
811 }
812
execute_optimize_work_item<B: ExtraBackendMethods>( cgcx: &CodegenContext<B>, module: ModuleCodegen<B::Module>, module_config: &ModuleConfig, ) -> Result<WorkItemResult<B>, FatalError>813 fn execute_optimize_work_item<B: ExtraBackendMethods>(
814 cgcx: &CodegenContext<B>,
815 module: ModuleCodegen<B::Module>,
816 module_config: &ModuleConfig,
817 ) -> Result<WorkItemResult<B>, FatalError> {
818 let diag_handler = cgcx.create_diag_handler();
819
820 unsafe {
821 B::optimize(cgcx, &diag_handler, &module, module_config)?;
822 }
823
824 // After we've done the initial round of optimizations we need to
825 // decide whether to synchronously codegen this module or ship it
826 // back to the coordinator thread for further LTO processing (which
827 // has to wait for all the initial modules to be optimized).
828
829 let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind);
830
831 // If we're doing some form of incremental LTO then we need to be sure to
832 // save our module to disk first.
833 let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc {
834 let filename = pre_lto_bitcode_filename(&module.name);
835 cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename))
836 } else {
837 None
838 };
839
840 match lto_type {
841 ComputedLtoType::No => finish_intra_module_work(cgcx, module, module_config),
842 ComputedLtoType::Thin => {
843 let (name, thin_buffer) = B::prepare_thin(module);
844 if let Some(path) = bitcode {
845 fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| {
846 panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
847 });
848 }
849 Ok(WorkItemResult::NeedsThinLTO(name, thin_buffer))
850 }
851 ComputedLtoType::Fat => match bitcode {
852 Some(path) => {
853 let (name, buffer) = B::serialize_module(module);
854 fs::write(&path, buffer.data()).unwrap_or_else(|e| {
855 panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
856 });
857 Ok(WorkItemResult::NeedsFatLTO(FatLTOInput::Serialized { name, buffer }))
858 }
859 None => Ok(WorkItemResult::NeedsFatLTO(FatLTOInput::InMemory(module))),
860 },
861 }
862 }
863
execute_copy_from_cache_work_item<B: ExtraBackendMethods>( cgcx: &CodegenContext<B>, module: CachedModuleCodegen, module_config: &ModuleConfig, ) -> WorkItemResult<B>864 fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
865 cgcx: &CodegenContext<B>,
866 module: CachedModuleCodegen,
867 module_config: &ModuleConfig,
868 ) -> WorkItemResult<B> {
869 assert!(module_config.emit_obj != EmitObj::None);
870
871 let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
872
873 let load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
874 let source_file = in_incr_comp_dir(&incr_comp_session_dir, saved_path);
875 debug!(
876 "copying preexisting module `{}` from {:?} to {}",
877 module.name,
878 source_file,
879 output_path.display()
880 );
881 match link_or_copy(&source_file, &output_path) {
882 Ok(_) => Some(output_path),
883 Err(error) => {
884 cgcx.create_diag_handler().emit_err(errors::CopyPathBuf {
885 source_file,
886 output_path,
887 error,
888 });
889 None
890 }
891 }
892 };
893
894 let object = load_from_incr_comp_dir(
895 cgcx.output_filenames.temp_path(OutputType::Object, Some(&module.name)),
896 &module.source.saved_files.get("o").expect("no saved object file in work product"),
897 );
898 let dwarf_object =
899 module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| {
900 let dwarf_obj_out = cgcx
901 .output_filenames
902 .split_dwarf_path(cgcx.split_debuginfo, cgcx.split_dwarf_kind, Some(&module.name))
903 .expect(
904 "saved dwarf object in work product but `split_dwarf_path` returned `None`",
905 );
906 load_from_incr_comp_dir(dwarf_obj_out, &saved_dwarf_object_file)
907 });
908
909 WorkItemResult::Compiled(CompiledModule {
910 name: module.name,
911 kind: ModuleKind::Regular,
912 object,
913 dwarf_object,
914 bytecode: None,
915 })
916 }
917
execute_lto_work_item<B: ExtraBackendMethods>( cgcx: &CodegenContext<B>, module: lto::LtoModuleCodegen<B>, module_config: &ModuleConfig, ) -> Result<WorkItemResult<B>, FatalError>918 fn execute_lto_work_item<B: ExtraBackendMethods>(
919 cgcx: &CodegenContext<B>,
920 module: lto::LtoModuleCodegen<B>,
921 module_config: &ModuleConfig,
922 ) -> Result<WorkItemResult<B>, FatalError> {
923 let module = unsafe { module.optimize(cgcx)? };
924 finish_intra_module_work(cgcx, module, module_config)
925 }
926
finish_intra_module_work<B: ExtraBackendMethods>( cgcx: &CodegenContext<B>, module: ModuleCodegen<B::Module>, module_config: &ModuleConfig, ) -> Result<WorkItemResult<B>, FatalError>927 fn finish_intra_module_work<B: ExtraBackendMethods>(
928 cgcx: &CodegenContext<B>,
929 module: ModuleCodegen<B::Module>,
930 module_config: &ModuleConfig,
931 ) -> Result<WorkItemResult<B>, FatalError> {
932 let diag_handler = cgcx.create_diag_handler();
933
934 if !cgcx.opts.unstable_opts.combine_cgu
935 || module.kind == ModuleKind::Metadata
936 || module.kind == ModuleKind::Allocator
937 {
938 let module = unsafe { B::codegen(cgcx, &diag_handler, module, module_config)? };
939 Ok(WorkItemResult::Compiled(module))
940 } else {
941 Ok(WorkItemResult::NeedsLink(module))
942 }
943 }
944
945 /// Messages sent to the coordinator.
946 pub(crate) enum Message<B: WriteBackendMethods> {
947 /// A jobserver token has become available. Sent from the jobserver helper
948 /// thread.
949 Token(io::Result<Acquired>),
950
951 /// The backend has finished processing a work item for a codegen unit.
952 /// Sent from a backend worker thread.
953 WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
954
955 /// The frontend has finished generating something (backend IR or a
956 /// post-LTO artifact) for a codegen unit, and it should be passed to the
957 /// backend. Sent from the main thread.
958 CodegenDone { llvm_work_item: WorkItem<B>, cost: u64 },
959
960 /// Similar to `CodegenDone`, but for reusing a pre-LTO artifact
961 /// Sent from the main thread.
962 AddImportOnlyModule {
963 module_data: SerializedModule<B::ModuleBuffer>,
964 work_product: WorkProduct,
965 },
966
967 /// The frontend has finished generating everything for all codegen units.
968 /// Sent from the main thread.
969 CodegenComplete,
970
971 /// Some normal-ish compiler error occurred, and codegen should be wound
972 /// down. Sent from the main thread.
973 CodegenAborted,
974 }
975
976 /// A message sent from the coordinator thread to the main thread telling it to
977 /// process another codegen unit.
978 pub struct CguMessage;
979
980 type DiagnosticArgName<'source> = Cow<'source, str>;
981
982 struct Diagnostic {
983 msg: Vec<(DiagnosticMessage, Style)>,
984 args: FxHashMap<DiagnosticArgName<'static>, rustc_errors::DiagnosticArgValue<'static>>,
985 code: Option<DiagnosticId>,
986 lvl: Level,
987 }
988
989 #[derive(PartialEq, Clone, Copy, Debug)]
990 enum MainThreadWorkerState {
991 Idle,
992 Codegenning,
993 LLVMing,
994 }
995
start_executing_work<B: ExtraBackendMethods>( backend: B, tcx: TyCtxt<'_>, crate_info: &CrateInfo, shared_emitter: SharedEmitter, codegen_worker_send: Sender<CguMessage>, coordinator_receive: Receiver<Box<dyn Any + Send>>, jobserver: Client, regular_config: Arc<ModuleConfig>, metadata_config: Arc<ModuleConfig>, allocator_config: Arc<ModuleConfig>, tx_to_llvm_workers: Sender<Box<dyn Any + Send>>, ) -> thread::JoinHandle<Result<CompiledModules, ()>>996 fn start_executing_work<B: ExtraBackendMethods>(
997 backend: B,
998 tcx: TyCtxt<'_>,
999 crate_info: &CrateInfo,
1000 shared_emitter: SharedEmitter,
1001 codegen_worker_send: Sender<CguMessage>,
1002 coordinator_receive: Receiver<Box<dyn Any + Send>>,
1003 jobserver: Client,
1004 regular_config: Arc<ModuleConfig>,
1005 metadata_config: Arc<ModuleConfig>,
1006 allocator_config: Arc<ModuleConfig>,
1007 tx_to_llvm_workers: Sender<Box<dyn Any + Send>>,
1008 ) -> thread::JoinHandle<Result<CompiledModules, ()>> {
1009 let coordinator_send = tx_to_llvm_workers;
1010 let sess = tcx.sess;
1011
1012 let mut each_linked_rlib_for_lto = Vec::new();
1013 drop(link::each_linked_rlib(crate_info, None, &mut |cnum, path| {
1014 if link::ignored_for_lto(sess, crate_info, cnum) {
1015 return;
1016 }
1017 each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
1018 }));
1019
1020 // Compute the set of symbols we need to retain when doing LTO (if we need to)
1021 let exported_symbols = {
1022 let mut exported_symbols = FxHashMap::default();
1023
1024 let copy_symbols = |cnum| {
1025 let symbols = tcx
1026 .exported_symbols(cnum)
1027 .iter()
1028 .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
1029 .collect();
1030 Arc::new(symbols)
1031 };
1032
1033 match sess.lto() {
1034 Lto::No => None,
1035 Lto::ThinLocal => {
1036 exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1037 Some(Arc::new(exported_symbols))
1038 }
1039 Lto::Fat | Lto::Thin => {
1040 exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1041 for &(cnum, ref _path) in &each_linked_rlib_for_lto {
1042 exported_symbols.insert(cnum, copy_symbols(cnum));
1043 }
1044 Some(Arc::new(exported_symbols))
1045 }
1046 }
1047 };
1048
1049 // First up, convert our jobserver into a helper thread so we can use normal
1050 // mpsc channels to manage our messages and such.
1051 // After we've requested tokens then we'll, when we can,
1052 // get tokens on `coordinator_receive` which will
1053 // get managed in the main loop below.
1054 let coordinator_send2 = coordinator_send.clone();
1055 let helper = jobserver
1056 .into_helper_thread(move |token| {
1057 drop(coordinator_send2.send(Box::new(Message::Token::<B>(token))));
1058 })
1059 .expect("failed to spawn helper thread");
1060
1061 let ol =
1062 if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() {
1063 // If we know that we won’t be doing codegen, create target machines without optimisation.
1064 config::OptLevel::No
1065 } else {
1066 tcx.backend_optimization_level(())
1067 };
1068 let backend_features = tcx.global_backend_features(());
1069
1070 let remark_dir = if let Some(ref dir) = sess.opts.unstable_opts.remark_dir {
1071 let result = fs::create_dir_all(dir).and_then(|_| dir.canonicalize());
1072 match result {
1073 Ok(dir) => Some(dir),
1074 Err(error) => sess.emit_fatal(ErrorCreatingRemarkDir { error }),
1075 }
1076 } else {
1077 None
1078 };
1079
1080 let cgcx = CodegenContext::<B> {
1081 crate_types: sess.crate_types().to_vec(),
1082 each_linked_rlib_for_lto,
1083 lto: sess.lto(),
1084 fewer_names: sess.fewer_names(),
1085 save_temps: sess.opts.cg.save_temps,
1086 time_trace: sess.opts.unstable_opts.llvm_time_trace,
1087 opts: Arc::new(sess.opts.clone()),
1088 prof: sess.prof.clone(),
1089 exported_symbols,
1090 remark: sess.opts.cg.remark.clone(),
1091 remark_dir,
1092 worker: 0,
1093 incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
1094 cgu_reuse_tracker: sess.cgu_reuse_tracker.clone(),
1095 coordinator_send,
1096 diag_emitter: shared_emitter.clone(),
1097 output_filenames: tcx.output_filenames(()).clone(),
1098 regular_module_config: regular_config,
1099 metadata_module_config: metadata_config,
1100 allocator_module_config: allocator_config,
1101 tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features),
1102 msvc_imps_needed: msvc_imps_needed(tcx),
1103 is_pe_coff: tcx.sess.target.is_like_windows,
1104 target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(),
1105 target_arch: tcx.sess.target.arch.to_string(),
1106 split_debuginfo: tcx.sess.split_debuginfo(),
1107 split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind,
1108 };
1109
1110 // This is the "main loop" of parallel work happening for parallel codegen.
1111 // It's here that we manage parallelism, schedule work, and work with
1112 // messages coming from clients.
1113 //
1114 // There are a few environmental pre-conditions that shape how the system
1115 // is set up:
1116 //
1117 // - Error reporting can only happen on the main thread because that's the
1118 // only place where we have access to the compiler `Session`.
1119 // - LLVM work can be done on any thread.
1120 // - Codegen can only happen on the main thread.
1121 // - Each thread doing substantial work must be in possession of a `Token`
1122 // from the `Jobserver`.
1123 // - The compiler process always holds one `Token`. Any additional `Tokens`
1124 // have to be requested from the `Jobserver`.
1125 //
1126 // Error Reporting
1127 // ===============
1128 // The error reporting restriction is handled separately from the rest: We
1129 // set up a `SharedEmitter` that holds an open channel to the main thread.
1130 // When an error occurs on any thread, the shared emitter will send the
1131 // error message to the receiver main thread (`SharedEmitterMain`). The
1132 // main thread will periodically query this error message queue and emit
1133 // any error messages it has received. It might even abort compilation if
1134 // it has received a fatal error. In this case we rely on all other threads
1135 // being torn down automatically with the main thread.
1136 // Since the main thread will often be busy doing codegen work, error
1137 // reporting will be somewhat delayed, since the message queue can only be
1138 // checked in between two work packages.
1139 //
1140 // Work Processing Infrastructure
1141 // ==============================
1142 // The work processing infrastructure knows three major actors:
1143 //
1144 // - the coordinator thread,
1145 // - the main thread, and
1146 // - LLVM worker threads
1147 //
1148 // The coordinator thread is running a message loop. It instructs the main
1149 // thread about what work to do when, and it will spawn off LLVM worker
1150 // threads as open LLVM WorkItems become available.
1151 //
1152 // The job of the main thread is to codegen CGUs into LLVM work packages
1153 // (since the main thread is the only thread that can do this). The main
1154 // thread will block until it receives a message from the coordinator, upon
1155 // which it will codegen one CGU, send it to the coordinator and block
1156 // again. This way the coordinator can control what the main thread is
1157 // doing.
1158 //
1159 // The coordinator keeps a queue of LLVM WorkItems, and when a `Token` is
1160 // available, it will spawn off a new LLVM worker thread and let it process
1161 // a WorkItem. When a LLVM worker thread is done with its WorkItem,
1162 // it will just shut down, which also frees all resources associated with
1163 // the given LLVM module, and sends a message to the coordinator that the
1164 // WorkItem has been completed.
1165 //
1166 // Work Scheduling
1167 // ===============
1168 // The scheduler's goal is to minimize the time it takes to complete all
1169 // work there is, however, we also want to keep memory consumption low
1170 // if possible. These two goals are at odds with each other: If memory
1171 // consumption were not an issue, we could just let the main thread produce
1172 // LLVM WorkItems at full speed, assuring maximal utilization of
1173 // Tokens/LLVM worker threads. However, since codegen is usually faster
1174 // than LLVM processing, the queue of LLVM WorkItems would fill up and each
1175 // WorkItem potentially holds on to a substantial amount of memory.
1176 //
1177 // So the actual goal is to always produce just enough LLVM WorkItems as
1178 // not to starve our LLVM worker threads. That means, once we have enough
1179 // WorkItems in our queue, we can block the main thread, so it does not
1180 // produce more until we need them.
1181 //
1182 // Doing LLVM Work on the Main Thread
1183 // ----------------------------------
1184 // Since the main thread owns the compiler process's implicit `Token`, it is
1185 // wasteful to keep it blocked without doing any work. Therefore, what we do
1186 // in this case is: We spawn off an additional LLVM worker thread that helps
1187 // reduce the queue. The work it is doing corresponds to the implicit
1188 // `Token`. The coordinator will mark the main thread as being busy with
1189 // LLVM work. (The actual work happens on another OS thread but we just care
1190 // about `Tokens`, not actual threads).
1191 //
1192 // When any LLVM worker thread finishes while the main thread is marked as
1193 // "busy with LLVM work", we can do a little switcheroo: We give the Token
1194 // of the just finished thread to the LLVM worker thread that is working on
1195 // behalf of the main thread's implicit Token, thus freeing up the main
1196 // thread again. The coordinator can then again decide what the main thread
1197 // should do. This allows the coordinator to make decisions at more points
1198 // in time.
1199 //
1200 // Striking a Balance between Throughput and Memory Consumption
1201 // ------------------------------------------------------------
1202 // Since our two goals, (1) use as many Tokens as possible and (2) keep
1203 // memory consumption as low as possible, are in conflict with each other,
1204 // we have to find a trade off between them. Right now, the goal is to keep
1205 // all workers busy, which means that no worker should find the queue empty
1206 // when it is ready to start.
1207 // How do we do achieve this? Good question :) We actually never know how
1208 // many `Tokens` are potentially available so it's hard to say how much to
1209 // fill up the queue before switching the main thread to LLVM work. Also we
1210 // currently don't have a means to estimate how long a running LLVM worker
1211 // will still be busy with it's current WorkItem. However, we know the
1212 // maximal count of available Tokens that makes sense (=the number of CPU
1213 // cores), so we can take a conservative guess. The heuristic we use here
1214 // is implemented in the `queue_full_enough()` function.
1215 //
1216 // Some Background on Jobservers
1217 // -----------------------------
1218 // It's worth also touching on the management of parallelism here. We don't
1219 // want to just spawn a thread per work item because while that's optimal
1220 // parallelism it may overload a system with too many threads or violate our
1221 // configuration for the maximum amount of cpu to use for this process. To
1222 // manage this we use the `jobserver` crate.
1223 //
1224 // Job servers are an artifact of GNU make and are used to manage
1225 // parallelism between processes. A jobserver is a glorified IPC semaphore
1226 // basically. Whenever we want to run some work we acquire the semaphore,
1227 // and whenever we're done with that work we release the semaphore. In this
1228 // manner we can ensure that the maximum number of parallel workers is
1229 // capped at any one point in time.
1230 //
1231 // LTO and the coordinator thread
1232 // ------------------------------
1233 //
1234 // The final job the coordinator thread is responsible for is managing LTO
1235 // and how that works. When LTO is requested what we'll do is collect all
1236 // optimized LLVM modules into a local vector on the coordinator. Once all
1237 // modules have been codegened and optimized we hand this to the `lto`
1238 // module for further optimization. The `lto` module will return back a list
1239 // of more modules to work on, which the coordinator will continue to spawn
1240 // work for.
1241 //
1242 // Each LLVM module is automatically sent back to the coordinator for LTO if
1243 // necessary. There's already optimizations in place to avoid sending work
1244 // back to the coordinator if LTO isn't requested.
1245 return B::spawn_thread(cgcx.time_trace, move || {
1246 let mut worker_id_counter = 0;
1247 let mut free_worker_ids = Vec::new();
1248 let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| {
1249 if let Some(id) = free_worker_ids.pop() {
1250 id
1251 } else {
1252 let id = worker_id_counter;
1253 worker_id_counter += 1;
1254 id
1255 }
1256 };
1257
1258 // This is where we collect codegen units that have gone all the way
1259 // through codegen and LLVM.
1260 let mut compiled_modules = vec![];
1261 let mut compiled_allocator_module = None;
1262 let mut needs_link = Vec::new();
1263 let mut needs_fat_lto = Vec::new();
1264 let mut needs_thin_lto = Vec::new();
1265 let mut lto_import_only_modules = Vec::new();
1266 let mut started_lto = false;
1267
1268 /// Possible state transitions:
1269 /// - Ongoing -> Completed
1270 /// - Ongoing -> Aborted
1271 /// - Completed -> Aborted
1272 #[derive(Debug, PartialEq)]
1273 enum CodegenState {
1274 Ongoing,
1275 Completed,
1276 Aborted,
1277 }
1278 use CodegenState::*;
1279 let mut codegen_state = Ongoing;
1280
1281 // This is the queue of LLVM work items that still need processing.
1282 let mut work_items = Vec::<(WorkItem<B>, u64)>::new();
1283
1284 // This are the Jobserver Tokens we currently hold. Does not include
1285 // the implicit Token the compiler process owns no matter what.
1286 let mut tokens = Vec::new();
1287
1288 let mut main_thread_worker_state = MainThreadWorkerState::Idle;
1289 let mut running = 0;
1290
1291 let prof = &cgcx.prof;
1292 let mut llvm_start_time: Option<VerboseTimingGuard<'_>> = None;
1293
1294 // Run the message loop while there's still anything that needs message
1295 // processing. Note that as soon as codegen is aborted we simply want to
1296 // wait for all existing work to finish, so many of the conditions here
1297 // only apply if codegen hasn't been aborted as they represent pending
1298 // work to be done.
1299 while codegen_state == Ongoing
1300 || running > 0
1301 || main_thread_worker_state == MainThreadWorkerState::LLVMing
1302 || (codegen_state == Completed
1303 && !(work_items.is_empty()
1304 && needs_fat_lto.is_empty()
1305 && needs_thin_lto.is_empty()
1306 && lto_import_only_modules.is_empty()
1307 && main_thread_worker_state == MainThreadWorkerState::Idle))
1308 {
1309 // While there are still CGUs to be codegened, the coordinator has
1310 // to decide how to utilize the compiler processes implicit Token:
1311 // For codegenning more CGU or for running them through LLVM.
1312 if codegen_state == Ongoing {
1313 if main_thread_worker_state == MainThreadWorkerState::Idle {
1314 // Compute the number of workers that will be running once we've taken as many
1315 // items from the work queue as we can, plus one for the main thread. It's not
1316 // critically important that we use this instead of just `running`, but it
1317 // prevents the `queue_full_enough` heuristic from fluctuating just because a
1318 // worker finished up and we decreased the `running` count, even though we're
1319 // just going to increase it right after this when we put a new worker to work.
1320 let extra_tokens = tokens.len().checked_sub(running).unwrap();
1321 let additional_running = std::cmp::min(extra_tokens, work_items.len());
1322 let anticipated_running = running + additional_running + 1;
1323
1324 if !queue_full_enough(work_items.len(), anticipated_running) {
1325 // The queue is not full enough, process more codegen units:
1326 if codegen_worker_send.send(CguMessage).is_err() {
1327 panic!("Could not send CguMessage to main thread")
1328 }
1329 main_thread_worker_state = MainThreadWorkerState::Codegenning;
1330 } else {
1331 // The queue is full enough to not let the worker
1332 // threads starve. Use the implicit Token to do some
1333 // LLVM work too.
1334 let (item, _) =
1335 work_items.pop().expect("queue empty - queue_full_enough() broken?");
1336 let cgcx = CodegenContext {
1337 worker: get_worker_id(&mut free_worker_ids),
1338 ..cgcx.clone()
1339 };
1340 maybe_start_llvm_timer(
1341 prof,
1342 cgcx.config(item.module_kind()),
1343 &mut llvm_start_time,
1344 );
1345 main_thread_worker_state = MainThreadWorkerState::LLVMing;
1346 spawn_work(cgcx, item);
1347 }
1348 }
1349 } else if codegen_state == Completed {
1350 // If we've finished everything related to normal codegen
1351 // then it must be the case that we've got some LTO work to do.
1352 // Perform the serial work here of figuring out what we're
1353 // going to LTO and then push a bunch of work items onto our
1354 // queue to do LTO
1355 if work_items.is_empty()
1356 && running == 0
1357 && main_thread_worker_state == MainThreadWorkerState::Idle
1358 {
1359 assert!(!started_lto);
1360 started_lto = true;
1361
1362 let needs_fat_lto = mem::take(&mut needs_fat_lto);
1363 let needs_thin_lto = mem::take(&mut needs_thin_lto);
1364 let import_only_modules = mem::take(&mut lto_import_only_modules);
1365
1366 for (work, cost) in
1367 generate_lto_work(&cgcx, needs_fat_lto, needs_thin_lto, import_only_modules)
1368 {
1369 let insertion_index = work_items
1370 .binary_search_by_key(&cost, |&(_, cost)| cost)
1371 .unwrap_or_else(|e| e);
1372 work_items.insert(insertion_index, (work, cost));
1373 if !cgcx.opts.unstable_opts.no_parallel_llvm {
1374 helper.request_token();
1375 }
1376 }
1377 }
1378
1379 // In this branch, we know that everything has been codegened,
1380 // so it's just a matter of determining whether the implicit
1381 // Token is free to use for LLVM work.
1382 match main_thread_worker_state {
1383 MainThreadWorkerState::Idle => {
1384 if let Some((item, _)) = work_items.pop() {
1385 let cgcx = CodegenContext {
1386 worker: get_worker_id(&mut free_worker_ids),
1387 ..cgcx.clone()
1388 };
1389 maybe_start_llvm_timer(
1390 prof,
1391 cgcx.config(item.module_kind()),
1392 &mut llvm_start_time,
1393 );
1394 main_thread_worker_state = MainThreadWorkerState::LLVMing;
1395 spawn_work(cgcx, item);
1396 } else {
1397 // There is no unstarted work, so let the main thread
1398 // take over for a running worker. Otherwise the
1399 // implicit token would just go to waste.
1400 // We reduce the `running` counter by one. The
1401 // `tokens.truncate()` below will take care of
1402 // giving the Token back.
1403 debug_assert!(running > 0);
1404 running -= 1;
1405 main_thread_worker_state = MainThreadWorkerState::LLVMing;
1406 }
1407 }
1408 MainThreadWorkerState::Codegenning => bug!(
1409 "codegen worker should not be codegenning after \
1410 codegen was already completed"
1411 ),
1412 MainThreadWorkerState::LLVMing => {
1413 // Already making good use of that token
1414 }
1415 }
1416 } else {
1417 // Don't queue up any more work if codegen was aborted, we're
1418 // just waiting for our existing children to finish.
1419 assert!(codegen_state == Aborted);
1420 }
1421
1422 // Spin up what work we can, only doing this while we've got available
1423 // parallelism slots and work left to spawn.
1424 while codegen_state != Aborted && !work_items.is_empty() && running < tokens.len() {
1425 let (item, _) = work_items.pop().unwrap();
1426
1427 maybe_start_llvm_timer(prof, cgcx.config(item.module_kind()), &mut llvm_start_time);
1428
1429 let cgcx =
1430 CodegenContext { worker: get_worker_id(&mut free_worker_ids), ..cgcx.clone() };
1431
1432 spawn_work(cgcx, item);
1433 running += 1;
1434 }
1435
1436 // Relinquish accidentally acquired extra tokens
1437 tokens.truncate(running);
1438
1439 // If a thread exits successfully then we drop a token associated
1440 // with that worker and update our `running` count. We may later
1441 // re-acquire a token to continue running more work. We may also not
1442 // actually drop a token here if the worker was running with an
1443 // "ephemeral token"
1444 let mut free_worker = |worker_id| {
1445 if main_thread_worker_state == MainThreadWorkerState::LLVMing {
1446 main_thread_worker_state = MainThreadWorkerState::Idle;
1447 } else {
1448 running -= 1;
1449 }
1450
1451 free_worker_ids.push(worker_id);
1452 };
1453
1454 let msg = coordinator_receive.recv().unwrap();
1455 match *msg.downcast::<Message<B>>().ok().unwrap() {
1456 // Save the token locally and the next turn of the loop will use
1457 // this to spawn a new unit of work, or it may get dropped
1458 // immediately if we have no more work to spawn.
1459 Message::Token(token) => {
1460 match token {
1461 Ok(token) => {
1462 tokens.push(token);
1463
1464 if main_thread_worker_state == MainThreadWorkerState::LLVMing {
1465 // If the main thread token is used for LLVM work
1466 // at the moment, we turn that thread into a regular
1467 // LLVM worker thread, so the main thread is free
1468 // to react to codegen demand.
1469 main_thread_worker_state = MainThreadWorkerState::Idle;
1470 running += 1;
1471 }
1472 }
1473 Err(e) => {
1474 let msg = &format!("failed to acquire jobserver token: {}", e);
1475 shared_emitter.fatal(msg);
1476 codegen_state = Aborted;
1477 }
1478 }
1479 }
1480
1481 Message::CodegenDone { llvm_work_item, cost } => {
1482 // We keep the queue sorted by estimated processing cost,
1483 // so that more expensive items are processed earlier. This
1484 // is good for throughput as it gives the main thread more
1485 // time to fill up the queue and it avoids scheduling
1486 // expensive items to the end.
1487 // Note, however, that this is not ideal for memory
1488 // consumption, as LLVM module sizes are not evenly
1489 // distributed.
1490 let insertion_index = work_items.binary_search_by_key(&cost, |&(_, cost)| cost);
1491 let insertion_index = match insertion_index {
1492 Ok(idx) | Err(idx) => idx,
1493 };
1494 work_items.insert(insertion_index, (llvm_work_item, cost));
1495
1496 if !cgcx.opts.unstable_opts.no_parallel_llvm {
1497 helper.request_token();
1498 }
1499 assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning);
1500 main_thread_worker_state = MainThreadWorkerState::Idle;
1501 }
1502
1503 Message::CodegenComplete => {
1504 if codegen_state != Aborted {
1505 codegen_state = Completed;
1506 }
1507 assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning);
1508 main_thread_worker_state = MainThreadWorkerState::Idle;
1509 }
1510
1511 // If codegen is aborted that means translation was aborted due
1512 // to some normal-ish compiler error. In this situation we want
1513 // to exit as soon as possible, but we want to make sure all
1514 // existing work has finished. Flag codegen as being done, and
1515 // then conditions above will ensure no more work is spawned but
1516 // we'll keep executing this loop until `running` hits 0.
1517 Message::CodegenAborted => {
1518 codegen_state = Aborted;
1519 }
1520
1521 Message::WorkItem { result, worker_id } => {
1522 free_worker(worker_id);
1523
1524 match result {
1525 Ok(WorkItemResult::Compiled(compiled_module)) => {
1526 match compiled_module.kind {
1527 ModuleKind::Regular => {
1528 compiled_modules.push(compiled_module);
1529 }
1530 ModuleKind::Allocator => {
1531 assert!(compiled_allocator_module.is_none());
1532 compiled_allocator_module = Some(compiled_module);
1533 }
1534 ModuleKind::Metadata => bug!("Should be handled separately"),
1535 }
1536 }
1537 Ok(WorkItemResult::NeedsLink(module)) => {
1538 needs_link.push(module);
1539 }
1540 Ok(WorkItemResult::NeedsFatLTO(fat_lto_input)) => {
1541 assert!(!started_lto);
1542 needs_fat_lto.push(fat_lto_input);
1543 }
1544 Ok(WorkItemResult::NeedsThinLTO(name, thin_buffer)) => {
1545 assert!(!started_lto);
1546 needs_thin_lto.push((name, thin_buffer));
1547 }
1548 Err(Some(WorkerFatalError)) => {
1549 // Like `CodegenAborted`, wait for remaining work to finish.
1550 codegen_state = Aborted;
1551 }
1552 Err(None) => {
1553 // If the thread failed that means it panicked, so
1554 // we abort immediately.
1555 bug!("worker thread panicked");
1556 }
1557 }
1558 }
1559
1560 Message::AddImportOnlyModule { module_data, work_product } => {
1561 assert!(!started_lto);
1562 assert_eq!(codegen_state, Ongoing);
1563 assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning);
1564 lto_import_only_modules.push((module_data, work_product));
1565 main_thread_worker_state = MainThreadWorkerState::Idle;
1566 }
1567 }
1568 }
1569
1570 if codegen_state == Aborted {
1571 return Err(());
1572 }
1573
1574 let needs_link = mem::take(&mut needs_link);
1575 if !needs_link.is_empty() {
1576 assert!(compiled_modules.is_empty());
1577 let diag_handler = cgcx.create_diag_handler();
1578 let module = B::run_link(&cgcx, &diag_handler, needs_link).map_err(|_| ())?;
1579 let module = unsafe {
1580 B::codegen(&cgcx, &diag_handler, module, cgcx.config(ModuleKind::Regular))
1581 .map_err(|_| ())?
1582 };
1583 compiled_modules.push(module);
1584 }
1585
1586 // Drop to print timings
1587 drop(llvm_start_time);
1588
1589 // Regardless of what order these modules completed in, report them to
1590 // the backend in the same order every time to ensure that we're handing
1591 // out deterministic results.
1592 compiled_modules.sort_by(|a, b| a.name.cmp(&b.name));
1593
1594 Ok(CompiledModules {
1595 modules: compiled_modules,
1596 allocator_module: compiled_allocator_module,
1597 })
1598 });
1599
1600 // A heuristic that determines if we have enough LLVM WorkItems in the
1601 // queue so that the main thread can do LLVM work instead of codegen
1602 fn queue_full_enough(items_in_queue: usize, workers_running: usize) -> bool {
1603 // This heuristic scales ahead-of-time codegen according to available
1604 // concurrency, as measured by `workers_running`. The idea is that the
1605 // more concurrency we have available, the more demand there will be for
1606 // work items, and the fuller the queue should be kept to meet demand.
1607 // An important property of this approach is that we codegen ahead of
1608 // time only as much as necessary, so as to keep fewer LLVM modules in
1609 // memory at once, thereby reducing memory consumption.
1610 //
1611 // When the number of workers running is less than the max concurrency
1612 // available to us, this heuristic can cause us to instruct the main
1613 // thread to work on an LLVM item (that is, tell it to "LLVM") instead
1614 // of codegen, even though it seems like it *should* be codegenning so
1615 // that we can create more work items and spawn more LLVM workers.
1616 //
1617 // But this is not a problem. When the main thread is told to LLVM,
1618 // according to this heuristic and how work is scheduled, there is
1619 // always at least one item in the queue, and therefore at least one
1620 // pending jobserver token request. If there *is* more concurrency
1621 // available, we will immediately receive a token, which will upgrade
1622 // the main thread's LLVM worker to a real one (conceptually), and free
1623 // up the main thread to codegen if necessary. On the other hand, if
1624 // there isn't more concurrency, then the main thread working on an LLVM
1625 // item is appropriate, as long as the queue is full enough for demand.
1626 //
1627 // Speaking of which, how full should we keep the queue? Probably less
1628 // full than you'd think. A lot has to go wrong for the queue not to be
1629 // full enough and for that to have a negative effect on compile times.
1630 //
1631 // Workers are unlikely to finish at exactly the same time, so when one
1632 // finishes and takes another work item off the queue, we often have
1633 // ample time to codegen at that point before the next worker finishes.
1634 // But suppose that codegen takes so long that the workers exhaust the
1635 // queue, and we have one or more workers that have nothing to work on.
1636 // Well, it might not be so bad. Of all the LLVM modules we create and
1637 // optimize, one has to finish last. It's not necessarily the case that
1638 // by losing some concurrency for a moment, we delay the point at which
1639 // that last LLVM module is finished and the rest of compilation can
1640 // proceed. Also, when we can't take advantage of some concurrency, we
1641 // give tokens back to the job server. That enables some other rustc to
1642 // potentially make use of the available concurrency. That could even
1643 // *decrease* overall compile time if we're lucky. But yes, if no other
1644 // rustc can make use of the concurrency, then we've squandered it.
1645 //
1646 // However, keeping the queue full is also beneficial when we have a
1647 // surge in available concurrency. Then items can be taken from the
1648 // queue immediately, without having to wait for codegen.
1649 //
1650 // So, the heuristic below tries to keep one item in the queue for every
1651 // four running workers. Based on limited benchmarking, this appears to
1652 // be more than sufficient to avoid increasing compilation times.
1653 let quarter_of_workers = workers_running - 3 * workers_running / 4;
1654 items_in_queue > 0 && items_in_queue >= quarter_of_workers
1655 }
1656
1657 fn maybe_start_llvm_timer<'a>(
1658 prof: &'a SelfProfilerRef,
1659 config: &ModuleConfig,
1660 llvm_start_time: &mut Option<VerboseTimingGuard<'a>>,
1661 ) {
1662 if config.time_module && llvm_start_time.is_none() {
1663 *llvm_start_time = Some(prof.verbose_generic_activity("LLVM_passes"));
1664 }
1665 }
1666 }
1667
1668 /// `FatalError` is explicitly not `Send`.
1669 #[must_use]
1670 pub struct WorkerFatalError;
1671
spawn_work<B: ExtraBackendMethods>(cgcx: CodegenContext<B>, work: WorkItem<B>)1672 fn spawn_work<B: ExtraBackendMethods>(cgcx: CodegenContext<B>, work: WorkItem<B>) {
1673 B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || {
1674 // Set up a destructor which will fire off a message that we're done as
1675 // we exit.
1676 struct Bomb<B: ExtraBackendMethods> {
1677 coordinator_send: Sender<Box<dyn Any + Send>>,
1678 result: Option<Result<WorkItemResult<B>, FatalError>>,
1679 worker_id: usize,
1680 }
1681 impl<B: ExtraBackendMethods> Drop for Bomb<B> {
1682 fn drop(&mut self) {
1683 let worker_id = self.worker_id;
1684 let msg = match self.result.take() {
1685 Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id },
1686 Some(Err(FatalError)) => {
1687 Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id }
1688 }
1689 None => Message::WorkItem::<B> { result: Err(None), worker_id },
1690 };
1691 drop(self.coordinator_send.send(Box::new(msg)));
1692 }
1693 }
1694
1695 let mut bomb = Bomb::<B> {
1696 coordinator_send: cgcx.coordinator_send.clone(),
1697 result: None,
1698 worker_id: cgcx.worker,
1699 };
1700
1701 // Execute the work itself, and if it finishes successfully then flag
1702 // ourselves as a success as well.
1703 //
1704 // Note that we ignore any `FatalError` coming out of `execute_work_item`,
1705 // as a diagnostic was already sent off to the main thread - just
1706 // surface that there was an error in this worker.
1707 bomb.result = {
1708 let module_config = cgcx.config(work.module_kind());
1709
1710 Some(match work {
1711 WorkItem::Optimize(m) => {
1712 let _timer =
1713 cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name);
1714 execute_optimize_work_item(&cgcx, m, module_config)
1715 }
1716 WorkItem::CopyPostLtoArtifacts(m) => {
1717 let _timer = cgcx.prof.generic_activity_with_arg(
1718 "codegen_copy_artifacts_from_incr_cache",
1719 &*m.name,
1720 );
1721 Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config))
1722 }
1723 WorkItem::LTO(m) => {
1724 let _timer =
1725 cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name());
1726 execute_lto_work_item(&cgcx, m, module_config)
1727 }
1728 })
1729 };
1730 })
1731 .expect("failed to spawn thread");
1732 }
1733
1734 enum SharedEmitterMessage {
1735 Diagnostic(Diagnostic),
1736 InlineAsmError(u32, String, Level, Option<(String, Vec<InnerSpan>)>),
1737 AbortIfErrors,
1738 Fatal(String),
1739 }
1740
1741 #[derive(Clone)]
1742 pub struct SharedEmitter {
1743 sender: Sender<SharedEmitterMessage>,
1744 }
1745
1746 pub struct SharedEmitterMain {
1747 receiver: Receiver<SharedEmitterMessage>,
1748 }
1749
1750 impl SharedEmitter {
new() -> (SharedEmitter, SharedEmitterMain)1751 pub fn new() -> (SharedEmitter, SharedEmitterMain) {
1752 let (sender, receiver) = channel();
1753
1754 (SharedEmitter { sender }, SharedEmitterMain { receiver })
1755 }
1756
inline_asm_error( &self, cookie: u32, msg: String, level: Level, source: Option<(String, Vec<InnerSpan>)>, )1757 pub fn inline_asm_error(
1758 &self,
1759 cookie: u32,
1760 msg: String,
1761 level: Level,
1762 source: Option<(String, Vec<InnerSpan>)>,
1763 ) {
1764 drop(self.sender.send(SharedEmitterMessage::InlineAsmError(cookie, msg, level, source)));
1765 }
1766
fatal(&self, msg: &str)1767 pub fn fatal(&self, msg: &str) {
1768 drop(self.sender.send(SharedEmitterMessage::Fatal(msg.to_string())));
1769 }
1770 }
1771
1772 impl Translate for SharedEmitter {
fluent_bundle(&self) -> Option<&Lrc<rustc_errors::FluentBundle>>1773 fn fluent_bundle(&self) -> Option<&Lrc<rustc_errors::FluentBundle>> {
1774 None
1775 }
1776
fallback_fluent_bundle(&self) -> &rustc_errors::FluentBundle1777 fn fallback_fluent_bundle(&self) -> &rustc_errors::FluentBundle {
1778 panic!("shared emitter attempted to translate a diagnostic");
1779 }
1780 }
1781
1782 impl Emitter for SharedEmitter {
emit_diagnostic(&mut self, diag: &rustc_errors::Diagnostic)1783 fn emit_diagnostic(&mut self, diag: &rustc_errors::Diagnostic) {
1784 let args: FxHashMap<Cow<'_, str>, rustc_errors::DiagnosticArgValue<'_>> =
1785 diag.args().map(|(name, arg)| (name.clone(), arg.clone())).collect();
1786 drop(self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic {
1787 msg: diag.message.clone(),
1788 args: args.clone(),
1789 code: diag.code.clone(),
1790 lvl: diag.level(),
1791 })));
1792 for child in &diag.children {
1793 drop(self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic {
1794 msg: child.message.clone(),
1795 args: args.clone(),
1796 code: None,
1797 lvl: child.level,
1798 })));
1799 }
1800 drop(self.sender.send(SharedEmitterMessage::AbortIfErrors));
1801 }
1802
source_map(&self) -> Option<&Lrc<SourceMap>>1803 fn source_map(&self) -> Option<&Lrc<SourceMap>> {
1804 None
1805 }
1806 }
1807
1808 impl SharedEmitterMain {
check(&self, sess: &Session, blocking: bool)1809 pub fn check(&self, sess: &Session, blocking: bool) {
1810 loop {
1811 let message = if blocking {
1812 match self.receiver.recv() {
1813 Ok(message) => Ok(message),
1814 Err(_) => Err(()),
1815 }
1816 } else {
1817 match self.receiver.try_recv() {
1818 Ok(message) => Ok(message),
1819 Err(_) => Err(()),
1820 }
1821 };
1822
1823 match message {
1824 Ok(SharedEmitterMessage::Diagnostic(diag)) => {
1825 let handler = sess.diagnostic();
1826 let mut d = rustc_errors::Diagnostic::new_with_messages(diag.lvl, diag.msg);
1827 if let Some(code) = diag.code {
1828 d.code(code);
1829 }
1830 d.replace_args(diag.args);
1831 handler.emit_diagnostic(&mut d);
1832 }
1833 Ok(SharedEmitterMessage::InlineAsmError(cookie, msg, level, source)) => {
1834 let msg = msg.strip_prefix("error: ").unwrap_or(&msg).to_string();
1835
1836 let mut err = match level {
1837 Level::Error { lint: false } => sess.struct_err(msg).forget_guarantee(),
1838 Level::Warning(_) => sess.struct_warn(msg),
1839 Level::Note => sess.struct_note_without_error(msg),
1840 _ => bug!("Invalid inline asm diagnostic level"),
1841 };
1842
1843 // If the cookie is 0 then we don't have span information.
1844 if cookie != 0 {
1845 let pos = BytePos::from_u32(cookie);
1846 let span = Span::with_root_ctxt(pos, pos);
1847 err.set_span(span);
1848 };
1849
1850 // Point to the generated assembly if it is available.
1851 if let Some((buffer, spans)) = source {
1852 let source = sess
1853 .source_map()
1854 .new_source_file(FileName::inline_asm_source_code(&buffer), buffer);
1855 let spans: Vec<_> = spans
1856 .iter()
1857 .map(|sp| {
1858 Span::with_root_ctxt(
1859 source.normalized_byte_pos(sp.start as u32),
1860 source.normalized_byte_pos(sp.end as u32),
1861 )
1862 })
1863 .collect();
1864 err.span_note(spans, "instantiated into assembly here");
1865 }
1866
1867 err.emit();
1868 }
1869 Ok(SharedEmitterMessage::AbortIfErrors) => {
1870 sess.abort_if_errors();
1871 }
1872 Ok(SharedEmitterMessage::Fatal(msg)) => {
1873 sess.fatal(msg);
1874 }
1875 Err(_) => {
1876 break;
1877 }
1878 }
1879 }
1880 }
1881 }
1882
1883 pub struct Coordinator<B: ExtraBackendMethods> {
1884 pub sender: Sender<Box<dyn Any + Send>>,
1885 future: Option<thread::JoinHandle<Result<CompiledModules, ()>>>,
1886 // Only used for the Message type.
1887 phantom: PhantomData<B>,
1888 }
1889
1890 impl<B: ExtraBackendMethods> Coordinator<B> {
join(mut self) -> std::thread::Result<Result<CompiledModules, ()>>1891 fn join(mut self) -> std::thread::Result<Result<CompiledModules, ()>> {
1892 self.future.take().unwrap().join()
1893 }
1894 }
1895
1896 impl<B: ExtraBackendMethods> Drop for Coordinator<B> {
drop(&mut self)1897 fn drop(&mut self) {
1898 if let Some(future) = self.future.take() {
1899 // If we haven't joined yet, signal to the coordinator that it should spawn no more
1900 // work, and wait for worker threads to finish.
1901 drop(self.sender.send(Box::new(Message::CodegenAborted::<B>)));
1902 drop(future.join());
1903 }
1904 }
1905 }
1906
1907 pub struct OngoingCodegen<B: ExtraBackendMethods> {
1908 pub backend: B,
1909 pub metadata: EncodedMetadata,
1910 pub metadata_module: Option<CompiledModule>,
1911 pub crate_info: CrateInfo,
1912 pub codegen_worker_receive: Receiver<CguMessage>,
1913 pub shared_emitter_main: SharedEmitterMain,
1914 pub output_filenames: Arc<OutputFilenames>,
1915 pub coordinator: Coordinator<B>,
1916 }
1917
1918 impl<B: ExtraBackendMethods> OngoingCodegen<B> {
join(self, sess: &Session) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>)1919 pub fn join(self, sess: &Session) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
1920 let _timer = sess.timer("finish_ongoing_codegen");
1921
1922 self.shared_emitter_main.check(sess, true);
1923 let compiled_modules = sess.time("join_worker_thread", || match self.coordinator.join() {
1924 Ok(Ok(compiled_modules)) => compiled_modules,
1925 Ok(Err(())) => {
1926 sess.abort_if_errors();
1927 panic!("expected abort due to worker thread errors")
1928 }
1929 Err(_) => {
1930 bug!("panic during codegen/LLVM phase");
1931 }
1932 });
1933
1934 sess.cgu_reuse_tracker.check_expected_reuse(sess);
1935
1936 sess.abort_if_errors();
1937
1938 let work_products =
1939 copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules);
1940 produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames);
1941
1942 // FIXME: time_llvm_passes support - does this use a global context or
1943 // something?
1944 if sess.codegen_units().as_usize() == 1 && sess.opts.unstable_opts.time_llvm_passes {
1945 self.backend.print_pass_timings()
1946 }
1947
1948 (
1949 CodegenResults {
1950 metadata: self.metadata,
1951 crate_info: self.crate_info,
1952
1953 modules: compiled_modules.modules,
1954 allocator_module: compiled_modules.allocator_module,
1955 metadata_module: self.metadata_module,
1956 },
1957 work_products,
1958 )
1959 }
1960
submit_pre_codegened_module_to_llvm( &self, tcx: TyCtxt<'_>, module: ModuleCodegen<B::Module>, )1961 pub fn submit_pre_codegened_module_to_llvm(
1962 &self,
1963 tcx: TyCtxt<'_>,
1964 module: ModuleCodegen<B::Module>,
1965 ) {
1966 self.wait_for_signal_to_codegen_item();
1967 self.check_for_errors(tcx.sess);
1968
1969 // These are generally cheap and won't throw off scheduling.
1970 let cost = 0;
1971 submit_codegened_module_to_llvm(&self.backend, &self.coordinator.sender, module, cost);
1972 }
1973
codegen_finished(&self, tcx: TyCtxt<'_>)1974 pub fn codegen_finished(&self, tcx: TyCtxt<'_>) {
1975 self.wait_for_signal_to_codegen_item();
1976 self.check_for_errors(tcx.sess);
1977 drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::<B>)));
1978 }
1979
check_for_errors(&self, sess: &Session)1980 pub fn check_for_errors(&self, sess: &Session) {
1981 self.shared_emitter_main.check(sess, false);
1982 }
1983
wait_for_signal_to_codegen_item(&self)1984 pub fn wait_for_signal_to_codegen_item(&self) {
1985 match self.codegen_worker_receive.recv() {
1986 Ok(CguMessage) => {
1987 // Ok to proceed.
1988 }
1989 Err(_) => {
1990 // One of the LLVM threads must have panicked, fall through so
1991 // error handling can be reached.
1992 }
1993 }
1994 }
1995 }
1996
submit_codegened_module_to_llvm<B: ExtraBackendMethods>( _backend: &B, tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>, module: ModuleCodegen<B::Module>, cost: u64, )1997 pub fn submit_codegened_module_to_llvm<B: ExtraBackendMethods>(
1998 _backend: &B,
1999 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2000 module: ModuleCodegen<B::Module>,
2001 cost: u64,
2002 ) {
2003 let llvm_work_item = WorkItem::Optimize(module);
2004 drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost })));
2005 }
2006
submit_post_lto_module_to_llvm<B: ExtraBackendMethods>( _backend: &B, tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>, module: CachedModuleCodegen, )2007 pub fn submit_post_lto_module_to_llvm<B: ExtraBackendMethods>(
2008 _backend: &B,
2009 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2010 module: CachedModuleCodegen,
2011 ) {
2012 let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module);
2013 drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost: 0 })));
2014 }
2015
submit_pre_lto_module_to_llvm<B: ExtraBackendMethods>( _backend: &B, tcx: TyCtxt<'_>, tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>, module: CachedModuleCodegen, )2016 pub fn submit_pre_lto_module_to_llvm<B: ExtraBackendMethods>(
2017 _backend: &B,
2018 tcx: TyCtxt<'_>,
2019 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2020 module: CachedModuleCodegen,
2021 ) {
2022 let filename = pre_lto_bitcode_filename(&module.name);
2023 let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename);
2024 let file = fs::File::open(&bc_path)
2025 .unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e));
2026
2027 let mmap = unsafe {
2028 Mmap::map(file).unwrap_or_else(|e| {
2029 panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e)
2030 })
2031 };
2032 // Schedule the module to be loaded
2033 drop(tx_to_llvm_workers.send(Box::new(Message::AddImportOnlyModule::<B> {
2034 module_data: SerializedModule::FromUncompressedFile(mmap),
2035 work_product: module.source,
2036 })));
2037 }
2038
pre_lto_bitcode_filename(module_name: &str) -> String2039 pub fn pre_lto_bitcode_filename(module_name: &str) -> String {
2040 format!("{}.{}", module_name, PRE_LTO_BC_EXT)
2041 }
2042
msvc_imps_needed(tcx: TyCtxt<'_>) -> bool2043 fn msvc_imps_needed(tcx: TyCtxt<'_>) -> bool {
2044 // This should never be true (because it's not supported). If it is true,
2045 // something is wrong with commandline arg validation.
2046 assert!(
2047 !(tcx.sess.opts.cg.linker_plugin_lto.enabled()
2048 && tcx.sess.target.is_like_windows
2049 && tcx.sess.opts.cg.prefer_dynamic)
2050 );
2051
2052 tcx.sess.target.is_like_windows &&
2053 tcx.sess.crate_types().iter().any(|ct| *ct == CrateType::Rlib) &&
2054 // ThinLTO can't handle this workaround in all cases, so we don't
2055 // emit the `__imp_` symbols. Instead we make them unnecessary by disallowing
2056 // dynamic linking when linker plugin LTO is enabled.
2057 !tcx.sess.opts.cg.linker_plugin_lto.enabled()
2058 }
2059