1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4 */
5
6 #include <string.h>
7 #include <stdlib.h>
8 #include <inttypes.h>
9 #include <sys/mman.h>
10
11 #include <arch/elf.h>
12 #include <objtool/builtin.h>
13 #include <objtool/cfi.h>
14 #include <objtool/arch.h>
15 #include <objtool/check.h>
16 #include <objtool/special.h>
17 #include <objtool/warn.h>
18 #include <objtool/endianness.h>
19
20 #include <linux/objtool.h>
21 #include <linux/hashtable.h>
22 #include <linux/kernel.h>
23 #include <linux/static_call_types.h>
24
25 struct alternative {
26 struct list_head list;
27 struct instruction *insn;
28 bool skip_orig;
29 };
30
31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
32
33 static struct cfi_init_state initial_func_cfi;
34 static struct cfi_state init_cfi;
35 static struct cfi_state func_cfi;
36
find_insn(struct objtool_file * file,struct section * sec,unsigned long offset)37 struct instruction *find_insn(struct objtool_file *file,
38 struct section *sec, unsigned long offset)
39 {
40 struct instruction *insn;
41
42 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
43 if (insn->sec == sec && insn->offset == offset)
44 return insn;
45 }
46
47 return NULL;
48 }
49
next_insn_same_sec(struct objtool_file * file,struct instruction * insn)50 static struct instruction *next_insn_same_sec(struct objtool_file *file,
51 struct instruction *insn)
52 {
53 struct instruction *next = list_next_entry(insn, list);
54
55 if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
56 return NULL;
57
58 return next;
59 }
60
next_insn_same_func(struct objtool_file * file,struct instruction * insn)61 static struct instruction *next_insn_same_func(struct objtool_file *file,
62 struct instruction *insn)
63 {
64 struct instruction *next = list_next_entry(insn, list);
65 struct symbol *func = insn->func;
66
67 if (!func)
68 return NULL;
69
70 if (&next->list != &file->insn_list && next->func == func)
71 return next;
72
73 /* Check if we're already in the subfunction: */
74 if (func == func->cfunc)
75 return NULL;
76
77 /* Move to the subfunction: */
78 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
79 }
80
prev_insn_same_sym(struct objtool_file * file,struct instruction * insn)81 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
82 struct instruction *insn)
83 {
84 struct instruction *prev = list_prev_entry(insn, list);
85
86 if (&prev->list != &file->insn_list && prev->func == insn->func)
87 return prev;
88
89 return NULL;
90 }
91
92 #define func_for_each_insn(file, func, insn) \
93 for (insn = find_insn(file, func->sec, func->offset); \
94 insn; \
95 insn = next_insn_same_func(file, insn))
96
97 #define sym_for_each_insn(file, sym, insn) \
98 for (insn = find_insn(file, sym->sec, sym->offset); \
99 insn && &insn->list != &file->insn_list && \
100 insn->sec == sym->sec && \
101 insn->offset < sym->offset + sym->len; \
102 insn = list_next_entry(insn, list))
103
104 #define sym_for_each_insn_continue_reverse(file, sym, insn) \
105 for (insn = list_prev_entry(insn, list); \
106 &insn->list != &file->insn_list && \
107 insn->sec == sym->sec && insn->offset >= sym->offset; \
108 insn = list_prev_entry(insn, list))
109
110 #define sec_for_each_insn_from(file, insn) \
111 for (; insn; insn = next_insn_same_sec(file, insn))
112
113 #define sec_for_each_insn_continue(file, insn) \
114 for (insn = next_insn_same_sec(file, insn); insn; \
115 insn = next_insn_same_sec(file, insn))
116
is_jump_table_jump(struct instruction * insn)117 static bool is_jump_table_jump(struct instruction *insn)
118 {
119 struct alt_group *alt_group = insn->alt_group;
120
121 if (insn->jump_table)
122 return true;
123
124 /* Retpoline alternative for a jump table? */
125 return alt_group && alt_group->orig_group &&
126 alt_group->orig_group->first_insn->jump_table;
127 }
128
is_sibling_call(struct instruction * insn)129 static bool is_sibling_call(struct instruction *insn)
130 {
131 /*
132 * Assume only ELF functions can make sibling calls. This ensures
133 * sibling call detection consistency between vmlinux.o and individual
134 * objects.
135 */
136 if (!insn->func)
137 return false;
138
139 /* An indirect jump is either a sibling call or a jump to a table. */
140 if (insn->type == INSN_JUMP_DYNAMIC)
141 return !is_jump_table_jump(insn);
142
143 /* add_jump_destinations() sets insn->call_dest for sibling calls. */
144 return (is_static_jump(insn) && insn->call_dest);
145 }
146
147 /*
148 * This checks to see if the given function is a "noreturn" function.
149 *
150 * For global functions which are outside the scope of this object file, we
151 * have to keep a manual list of them.
152 *
153 * For local functions, we have to detect them manually by simply looking for
154 * the lack of a return instruction.
155 */
__dead_end_function(struct objtool_file * file,struct symbol * func,int recursion)156 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
157 int recursion)
158 {
159 int i;
160 struct instruction *insn;
161 bool empty = true;
162
163 /*
164 * Unfortunately these have to be hard coded because the noreturn
165 * attribute isn't provided in ELF data. Keep 'em sorted.
166 */
167 static const char * const global_noreturns[] = {
168 "__invalid_creds",
169 "__module_put_and_kthread_exit",
170 "__reiserfs_panic",
171 "__stack_chk_fail",
172 "__ubsan_handle_builtin_unreachable",
173 "cpu_bringup_and_idle",
174 "cpu_startup_entry",
175 "do_exit",
176 "do_group_exit",
177 "do_task_dead",
178 "ex_handler_msr_mce",
179 "fortify_panic",
180 "kthread_complete_and_exit",
181 "kthread_exit",
182 "kunit_try_catch_throw",
183 "lbug_with_loc",
184 "machine_real_restart",
185 "make_task_dead",
186 "panic",
187 "rewind_stack_and_make_dead",
188 "sev_es_terminate",
189 "snp_abort",
190 "stop_this_cpu",
191 "usercopy_abort",
192 "xen_start_kernel",
193 };
194
195 if (!func)
196 return false;
197
198 if (func->bind == STB_WEAK)
199 return false;
200
201 if (func->bind == STB_GLOBAL)
202 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
203 if (!strcmp(func->name, global_noreturns[i]))
204 return true;
205
206 if (!func->len)
207 return false;
208
209 insn = find_insn(file, func->sec, func->offset);
210 if (!insn || !insn->func)
211 return false;
212
213 func_for_each_insn(file, func, insn) {
214 empty = false;
215
216 if (insn->type == INSN_RETURN)
217 return false;
218 }
219
220 if (empty)
221 return false;
222
223 /*
224 * A function can have a sibling call instead of a return. In that
225 * case, the function's dead-end status depends on whether the target
226 * of the sibling call returns.
227 */
228 func_for_each_insn(file, func, insn) {
229 if (is_sibling_call(insn)) {
230 struct instruction *dest = insn->jump_dest;
231
232 if (!dest)
233 /* sibling call to another file */
234 return false;
235
236 /* local sibling call */
237 if (recursion == 5) {
238 /*
239 * Infinite recursion: two functions have
240 * sibling calls to each other. This is a very
241 * rare case. It means they aren't dead ends.
242 */
243 return false;
244 }
245
246 return __dead_end_function(file, dest->func, recursion+1);
247 }
248 }
249
250 return true;
251 }
252
dead_end_function(struct objtool_file * file,struct symbol * func)253 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
254 {
255 return __dead_end_function(file, func, 0);
256 }
257
init_cfi_state(struct cfi_state * cfi)258 static void init_cfi_state(struct cfi_state *cfi)
259 {
260 int i;
261
262 for (i = 0; i < CFI_NUM_REGS; i++) {
263 cfi->regs[i].base = CFI_UNDEFINED;
264 cfi->vals[i].base = CFI_UNDEFINED;
265 }
266 cfi->cfa.base = CFI_UNDEFINED;
267 cfi->drap_reg = CFI_UNDEFINED;
268 cfi->drap_offset = -1;
269 }
270
init_insn_state(struct objtool_file * file,struct insn_state * state,struct section * sec)271 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
272 struct section *sec)
273 {
274 memset(state, 0, sizeof(*state));
275 init_cfi_state(&state->cfi);
276
277 /*
278 * We need the full vmlinux for noinstr validation, otherwise we can
279 * not correctly determine insn->call_dest->sec (external symbols do
280 * not have a section).
281 */
282 if (opts.link && opts.noinstr && sec)
283 state->noinstr = sec->noinstr;
284 }
285
cfi_alloc(void)286 static struct cfi_state *cfi_alloc(void)
287 {
288 struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
289 if (!cfi) {
290 WARN("calloc failed");
291 exit(1);
292 }
293 nr_cfi++;
294 return cfi;
295 }
296
297 static int cfi_bits;
298 static struct hlist_head *cfi_hash;
299
cficmp(struct cfi_state * cfi1,struct cfi_state * cfi2)300 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
301 {
302 return memcmp((void *)cfi1 + sizeof(cfi1->hash),
303 (void *)cfi2 + sizeof(cfi2->hash),
304 sizeof(struct cfi_state) - sizeof(struct hlist_node));
305 }
306
cfi_key(struct cfi_state * cfi)307 static inline u32 cfi_key(struct cfi_state *cfi)
308 {
309 return jhash((void *)cfi + sizeof(cfi->hash),
310 sizeof(*cfi) - sizeof(cfi->hash), 0);
311 }
312
cfi_hash_find_or_add(struct cfi_state * cfi)313 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
314 {
315 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
316 struct cfi_state *obj;
317
318 hlist_for_each_entry(obj, head, hash) {
319 if (!cficmp(cfi, obj)) {
320 nr_cfi_cache++;
321 return obj;
322 }
323 }
324
325 obj = cfi_alloc();
326 *obj = *cfi;
327 hlist_add_head(&obj->hash, head);
328
329 return obj;
330 }
331
cfi_hash_add(struct cfi_state * cfi)332 static void cfi_hash_add(struct cfi_state *cfi)
333 {
334 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
335
336 hlist_add_head(&cfi->hash, head);
337 }
338
cfi_hash_alloc(unsigned long size)339 static void *cfi_hash_alloc(unsigned long size)
340 {
341 cfi_bits = max(10, ilog2(size));
342 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
343 PROT_READ|PROT_WRITE,
344 MAP_PRIVATE|MAP_ANON, -1, 0);
345 if (cfi_hash == (void *)-1L) {
346 WARN("mmap fail cfi_hash");
347 cfi_hash = NULL;
348 } else if (opts.stats) {
349 printf("cfi_bits: %d\n", cfi_bits);
350 }
351
352 return cfi_hash;
353 }
354
355 static unsigned long nr_insns;
356 static unsigned long nr_insns_visited;
357
358 /*
359 * Call the arch-specific instruction decoder for all the instructions and add
360 * them to the global instruction list.
361 */
decode_instructions(struct objtool_file * file)362 static int decode_instructions(struct objtool_file *file)
363 {
364 struct section *sec;
365 struct symbol *func;
366 unsigned long offset;
367 struct instruction *insn;
368 int ret;
369
370 for_each_sec(file, sec) {
371
372 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
373 continue;
374
375 if (strcmp(sec->name, ".altinstr_replacement") &&
376 strcmp(sec->name, ".altinstr_aux") &&
377 strncmp(sec->name, ".discard.", 9))
378 sec->text = true;
379
380 if (!strcmp(sec->name, ".noinstr.text") ||
381 !strcmp(sec->name, ".entry.text") ||
382 !strncmp(sec->name, ".text..__x86.", 13))
383 sec->noinstr = true;
384
385 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
386 insn = malloc(sizeof(*insn));
387 if (!insn) {
388 WARN("malloc failed");
389 return -1;
390 }
391 memset(insn, 0, sizeof(*insn));
392 INIT_LIST_HEAD(&insn->alts);
393 INIT_LIST_HEAD(&insn->stack_ops);
394 INIT_LIST_HEAD(&insn->call_node);
395
396 insn->sec = sec;
397 insn->offset = offset;
398
399 ret = arch_decode_instruction(file, sec, offset,
400 sec->sh.sh_size - offset,
401 &insn->len, &insn->type,
402 &insn->immediate,
403 &insn->stack_ops);
404 if (ret)
405 goto err;
406
407 /*
408 * By default, "ud2" is a dead end unless otherwise
409 * annotated, because GCC 7 inserts it for certain
410 * divide-by-zero cases.
411 */
412 if (insn->type == INSN_BUG)
413 insn->dead_end = true;
414
415 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
416 list_add_tail(&insn->list, &file->insn_list);
417 nr_insns++;
418 }
419
420 list_for_each_entry(func, &sec->symbol_list, list) {
421 if (func->type != STT_FUNC || func->alias != func)
422 continue;
423
424 if (!find_insn(file, sec, func->offset)) {
425 WARN("%s(): can't find starting instruction",
426 func->name);
427 return -1;
428 }
429
430 sym_for_each_insn(file, func, insn) {
431 insn->func = func;
432 if (insn->type == INSN_ENDBR && list_empty(&insn->call_node)) {
433 if (insn->offset == insn->func->offset) {
434 list_add_tail(&insn->call_node, &file->endbr_list);
435 file->nr_endbr++;
436 } else {
437 file->nr_endbr_int++;
438 }
439 }
440 }
441 }
442 }
443
444 if (opts.stats)
445 printf("nr_insns: %lu\n", nr_insns);
446
447 return 0;
448
449 err:
450 free(insn);
451 return ret;
452 }
453
454 /*
455 * Read the pv_ops[] .data table to find the static initialized values.
456 */
add_pv_ops(struct objtool_file * file,const char * symname)457 static int add_pv_ops(struct objtool_file *file, const char *symname)
458 {
459 struct symbol *sym, *func;
460 unsigned long off, end;
461 struct reloc *rel;
462 int idx;
463
464 sym = find_symbol_by_name(file->elf, symname);
465 if (!sym)
466 return 0;
467
468 off = sym->offset;
469 end = off + sym->len;
470 for (;;) {
471 rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
472 if (!rel)
473 break;
474
475 func = rel->sym;
476 if (func->type == STT_SECTION)
477 func = find_symbol_by_offset(rel->sym->sec, rel->addend);
478
479 idx = (rel->offset - sym->offset) / sizeof(unsigned long);
480
481 objtool_pv_add(file, idx, func);
482
483 off = rel->offset + 1;
484 if (off > end)
485 break;
486 }
487
488 return 0;
489 }
490
491 /*
492 * Allocate and initialize file->pv_ops[].
493 */
init_pv_ops(struct objtool_file * file)494 static int init_pv_ops(struct objtool_file *file)
495 {
496 static const char *pv_ops_tables[] = {
497 "pv_ops",
498 "xen_cpu_ops",
499 "xen_irq_ops",
500 "xen_mmu_ops",
501 NULL,
502 };
503 const char *pv_ops;
504 struct symbol *sym;
505 int idx, nr;
506
507 if (!opts.noinstr)
508 return 0;
509
510 file->pv_ops = NULL;
511
512 sym = find_symbol_by_name(file->elf, "pv_ops");
513 if (!sym)
514 return 0;
515
516 nr = sym->len / sizeof(unsigned long);
517 file->pv_ops = calloc(sizeof(struct pv_state), nr);
518 if (!file->pv_ops)
519 return -1;
520
521 for (idx = 0; idx < nr; idx++)
522 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
523
524 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
525 add_pv_ops(file, pv_ops);
526
527 return 0;
528 }
529
find_last_insn(struct objtool_file * file,struct section * sec)530 static struct instruction *find_last_insn(struct objtool_file *file,
531 struct section *sec)
532 {
533 struct instruction *insn = NULL;
534 unsigned int offset;
535 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
536
537 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
538 insn = find_insn(file, sec, offset);
539
540 return insn;
541 }
542
543 /*
544 * Mark "ud2" instructions and manually annotated dead ends.
545 */
add_dead_ends(struct objtool_file * file)546 static int add_dead_ends(struct objtool_file *file)
547 {
548 struct section *sec;
549 struct reloc *reloc;
550 struct instruction *insn;
551
552 /*
553 * Check for manually annotated dead ends.
554 */
555 sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
556 if (!sec)
557 goto reachable;
558
559 list_for_each_entry(reloc, &sec->reloc_list, list) {
560 if (reloc->sym->type != STT_SECTION) {
561 WARN("unexpected relocation symbol type in %s", sec->name);
562 return -1;
563 }
564 insn = find_insn(file, reloc->sym->sec, reloc->addend);
565 if (insn)
566 insn = list_prev_entry(insn, list);
567 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
568 insn = find_last_insn(file, reloc->sym->sec);
569 if (!insn) {
570 WARN("can't find unreachable insn at %s+0x%" PRIx64,
571 reloc->sym->sec->name, reloc->addend);
572 return -1;
573 }
574 } else {
575 WARN("can't find unreachable insn at %s+0x%" PRIx64,
576 reloc->sym->sec->name, reloc->addend);
577 return -1;
578 }
579
580 insn->dead_end = true;
581 }
582
583 reachable:
584 /*
585 * These manually annotated reachable checks are needed for GCC 4.4,
586 * where the Linux unreachable() macro isn't supported. In that case
587 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
588 * not a dead end.
589 */
590 sec = find_section_by_name(file->elf, ".rela.discard.reachable");
591 if (!sec)
592 return 0;
593
594 list_for_each_entry(reloc, &sec->reloc_list, list) {
595 if (reloc->sym->type != STT_SECTION) {
596 WARN("unexpected relocation symbol type in %s", sec->name);
597 return -1;
598 }
599 insn = find_insn(file, reloc->sym->sec, reloc->addend);
600 if (insn)
601 insn = list_prev_entry(insn, list);
602 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
603 insn = find_last_insn(file, reloc->sym->sec);
604 if (!insn) {
605 WARN("can't find reachable insn at %s+0x%" PRIx64,
606 reloc->sym->sec->name, reloc->addend);
607 return -1;
608 }
609 } else {
610 WARN("can't find reachable insn at %s+0x%" PRIx64,
611 reloc->sym->sec->name, reloc->addend);
612 return -1;
613 }
614
615 insn->dead_end = false;
616 }
617
618 return 0;
619 }
620
create_static_call_sections(struct objtool_file * file)621 static int create_static_call_sections(struct objtool_file *file)
622 {
623 struct section *sec;
624 struct static_call_site *site;
625 struct instruction *insn;
626 struct symbol *key_sym;
627 char *key_name, *tmp;
628 int idx;
629
630 sec = find_section_by_name(file->elf, ".static_call_sites");
631 if (sec) {
632 INIT_LIST_HEAD(&file->static_call_list);
633 WARN("file already has .static_call_sites section, skipping");
634 return 0;
635 }
636
637 if (list_empty(&file->static_call_list))
638 return 0;
639
640 idx = 0;
641 list_for_each_entry(insn, &file->static_call_list, call_node)
642 idx++;
643
644 sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
645 sizeof(struct static_call_site), idx);
646 if (!sec)
647 return -1;
648
649 idx = 0;
650 list_for_each_entry(insn, &file->static_call_list, call_node) {
651
652 site = (struct static_call_site *)sec->data->d_buf + idx;
653 memset(site, 0, sizeof(struct static_call_site));
654
655 /* populate reloc for 'addr' */
656 if (elf_add_reloc_to_insn(file->elf, sec,
657 idx * sizeof(struct static_call_site),
658 R_X86_64_PC32,
659 insn->sec, insn->offset))
660 return -1;
661
662 /* find key symbol */
663 key_name = strdup(insn->call_dest->name);
664 if (!key_name) {
665 perror("strdup");
666 return -1;
667 }
668 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
669 STATIC_CALL_TRAMP_PREFIX_LEN)) {
670 WARN("static_call: trampoline name malformed: %s", key_name);
671 free(key_name);
672 return -1;
673 }
674 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
675 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
676
677 key_sym = find_symbol_by_name(file->elf, tmp);
678 if (!key_sym) {
679 if (!opts.module) {
680 WARN("static_call: can't find static_call_key symbol: %s", tmp);
681 free(key_name);
682 return -1;
683 }
684
685 /*
686 * For modules(), the key might not be exported, which
687 * means the module can make static calls but isn't
688 * allowed to change them.
689 *
690 * In that case we temporarily set the key to be the
691 * trampoline address. This is fixed up in
692 * static_call_add_module().
693 */
694 key_sym = insn->call_dest;
695 }
696 free(key_name);
697
698 /* populate reloc for 'key' */
699 if (elf_add_reloc(file->elf, sec,
700 idx * sizeof(struct static_call_site) + 4,
701 R_X86_64_PC32, key_sym,
702 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
703 return -1;
704
705 idx++;
706 }
707
708 return 0;
709 }
710
create_retpoline_sites_sections(struct objtool_file * file)711 static int create_retpoline_sites_sections(struct objtool_file *file)
712 {
713 struct instruction *insn;
714 struct section *sec;
715 int idx;
716
717 sec = find_section_by_name(file->elf, ".retpoline_sites");
718 if (sec) {
719 WARN("file already has .retpoline_sites, skipping");
720 return 0;
721 }
722
723 idx = 0;
724 list_for_each_entry(insn, &file->retpoline_call_list, call_node)
725 idx++;
726
727 if (!idx)
728 return 0;
729
730 sec = elf_create_section(file->elf, ".retpoline_sites", 0,
731 sizeof(int), idx);
732 if (!sec) {
733 WARN("elf_create_section: .retpoline_sites");
734 return -1;
735 }
736
737 idx = 0;
738 list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
739
740 int *site = (int *)sec->data->d_buf + idx;
741 *site = 0;
742
743 if (elf_add_reloc_to_insn(file->elf, sec,
744 idx * sizeof(int),
745 R_X86_64_PC32,
746 insn->sec, insn->offset)) {
747 WARN("elf_add_reloc_to_insn: .retpoline_sites");
748 return -1;
749 }
750
751 idx++;
752 }
753
754 return 0;
755 }
756
create_return_sites_sections(struct objtool_file * file)757 static int create_return_sites_sections(struct objtool_file *file)
758 {
759 struct instruction *insn;
760 struct section *sec;
761 int idx;
762
763 sec = find_section_by_name(file->elf, ".return_sites");
764 if (sec) {
765 WARN("file already has .return_sites, skipping");
766 return 0;
767 }
768
769 idx = 0;
770 list_for_each_entry(insn, &file->return_thunk_list, call_node)
771 idx++;
772
773 if (!idx)
774 return 0;
775
776 sec = elf_create_section(file->elf, ".return_sites", 0,
777 sizeof(int), idx);
778 if (!sec) {
779 WARN("elf_create_section: .return_sites");
780 return -1;
781 }
782
783 idx = 0;
784 list_for_each_entry(insn, &file->return_thunk_list, call_node) {
785
786 int *site = (int *)sec->data->d_buf + idx;
787 *site = 0;
788
789 if (elf_add_reloc_to_insn(file->elf, sec,
790 idx * sizeof(int),
791 R_X86_64_PC32,
792 insn->sec, insn->offset)) {
793 WARN("elf_add_reloc_to_insn: .return_sites");
794 return -1;
795 }
796
797 idx++;
798 }
799
800 return 0;
801 }
802
create_ibt_endbr_seal_sections(struct objtool_file * file)803 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
804 {
805 struct instruction *insn;
806 struct section *sec;
807 int idx;
808
809 sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
810 if (sec) {
811 WARN("file already has .ibt_endbr_seal, skipping");
812 return 0;
813 }
814
815 idx = 0;
816 list_for_each_entry(insn, &file->endbr_list, call_node)
817 idx++;
818
819 if (opts.stats) {
820 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
821 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
822 printf("ibt: superfluous ENDBR: %d\n", idx);
823 }
824
825 if (!idx)
826 return 0;
827
828 sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0,
829 sizeof(int), idx);
830 if (!sec) {
831 WARN("elf_create_section: .ibt_endbr_seal");
832 return -1;
833 }
834
835 idx = 0;
836 list_for_each_entry(insn, &file->endbr_list, call_node) {
837
838 int *site = (int *)sec->data->d_buf + idx;
839 *site = 0;
840
841 if (elf_add_reloc_to_insn(file->elf, sec,
842 idx * sizeof(int),
843 R_X86_64_PC32,
844 insn->sec, insn->offset)) {
845 WARN("elf_add_reloc_to_insn: .ibt_endbr_seal");
846 return -1;
847 }
848
849 idx++;
850 }
851
852 return 0;
853 }
854
create_mcount_loc_sections(struct objtool_file * file)855 static int create_mcount_loc_sections(struct objtool_file *file)
856 {
857 struct section *sec;
858 unsigned long *loc;
859 struct instruction *insn;
860 int idx;
861
862 sec = find_section_by_name(file->elf, "__mcount_loc");
863 if (sec) {
864 INIT_LIST_HEAD(&file->mcount_loc_list);
865 WARN("file already has __mcount_loc section, skipping");
866 return 0;
867 }
868
869 if (list_empty(&file->mcount_loc_list))
870 return 0;
871
872 idx = 0;
873 list_for_each_entry(insn, &file->mcount_loc_list, call_node)
874 idx++;
875
876 sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx);
877 if (!sec)
878 return -1;
879
880 idx = 0;
881 list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
882
883 loc = (unsigned long *)sec->data->d_buf + idx;
884 memset(loc, 0, sizeof(unsigned long));
885
886 if (elf_add_reloc_to_insn(file->elf, sec,
887 idx * sizeof(unsigned long),
888 R_X86_64_64,
889 insn->sec, insn->offset))
890 return -1;
891
892 idx++;
893 }
894
895 return 0;
896 }
897
898 /*
899 * Warnings shouldn't be reported for ignored functions.
900 */
add_ignores(struct objtool_file * file)901 static void add_ignores(struct objtool_file *file)
902 {
903 struct instruction *insn;
904 struct section *sec;
905 struct symbol *func;
906 struct reloc *reloc;
907
908 sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
909 if (!sec)
910 return;
911
912 list_for_each_entry(reloc, &sec->reloc_list, list) {
913 switch (reloc->sym->type) {
914 case STT_FUNC:
915 func = reloc->sym;
916 break;
917
918 case STT_SECTION:
919 func = find_func_by_offset(reloc->sym->sec, reloc->addend);
920 if (!func)
921 continue;
922 break;
923
924 default:
925 WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
926 continue;
927 }
928
929 func_for_each_insn(file, func, insn)
930 insn->ignore = true;
931 }
932 }
933
934 /*
935 * This is a whitelist of functions that is allowed to be called with AC set.
936 * The list is meant to be minimal and only contains compiler instrumentation
937 * ABI and a few functions used to implement *_{to,from}_user() functions.
938 *
939 * These functions must not directly change AC, but may PUSHF/POPF.
940 */
941 static const char *uaccess_safe_builtin[] = {
942 /* KASAN */
943 "kasan_report",
944 "kasan_check_range",
945 /* KASAN out-of-line */
946 "__asan_loadN_noabort",
947 "__asan_load1_noabort",
948 "__asan_load2_noabort",
949 "__asan_load4_noabort",
950 "__asan_load8_noabort",
951 "__asan_load16_noabort",
952 "__asan_storeN_noabort",
953 "__asan_store1_noabort",
954 "__asan_store2_noabort",
955 "__asan_store4_noabort",
956 "__asan_store8_noabort",
957 "__asan_store16_noabort",
958 "__kasan_check_read",
959 "__kasan_check_write",
960 /* KASAN in-line */
961 "__asan_report_load_n_noabort",
962 "__asan_report_load1_noabort",
963 "__asan_report_load2_noabort",
964 "__asan_report_load4_noabort",
965 "__asan_report_load8_noabort",
966 "__asan_report_load16_noabort",
967 "__asan_report_store_n_noabort",
968 "__asan_report_store1_noabort",
969 "__asan_report_store2_noabort",
970 "__asan_report_store4_noabort",
971 "__asan_report_store8_noabort",
972 "__asan_report_store16_noabort",
973 /* KCSAN */
974 "__kcsan_check_access",
975 "__kcsan_mb",
976 "__kcsan_wmb",
977 "__kcsan_rmb",
978 "__kcsan_release",
979 "kcsan_found_watchpoint",
980 "kcsan_setup_watchpoint",
981 "kcsan_check_scoped_accesses",
982 "kcsan_disable_current",
983 "kcsan_enable_current_nowarn",
984 /* KCSAN/TSAN */
985 "__tsan_func_entry",
986 "__tsan_func_exit",
987 "__tsan_read_range",
988 "__tsan_write_range",
989 "__tsan_read1",
990 "__tsan_read2",
991 "__tsan_read4",
992 "__tsan_read8",
993 "__tsan_read16",
994 "__tsan_write1",
995 "__tsan_write2",
996 "__tsan_write4",
997 "__tsan_write8",
998 "__tsan_write16",
999 "__tsan_read_write1",
1000 "__tsan_read_write2",
1001 "__tsan_read_write4",
1002 "__tsan_read_write8",
1003 "__tsan_read_write16",
1004 "__tsan_volatile_read1",
1005 "__tsan_volatile_read2",
1006 "__tsan_volatile_read4",
1007 "__tsan_volatile_read8",
1008 "__tsan_volatile_read16",
1009 "__tsan_volatile_write1",
1010 "__tsan_volatile_write2",
1011 "__tsan_volatile_write4",
1012 "__tsan_volatile_write8",
1013 "__tsan_volatile_write16",
1014 "__tsan_atomic8_load",
1015 "__tsan_atomic16_load",
1016 "__tsan_atomic32_load",
1017 "__tsan_atomic64_load",
1018 "__tsan_atomic8_store",
1019 "__tsan_atomic16_store",
1020 "__tsan_atomic32_store",
1021 "__tsan_atomic64_store",
1022 "__tsan_atomic8_exchange",
1023 "__tsan_atomic16_exchange",
1024 "__tsan_atomic32_exchange",
1025 "__tsan_atomic64_exchange",
1026 "__tsan_atomic8_fetch_add",
1027 "__tsan_atomic16_fetch_add",
1028 "__tsan_atomic32_fetch_add",
1029 "__tsan_atomic64_fetch_add",
1030 "__tsan_atomic8_fetch_sub",
1031 "__tsan_atomic16_fetch_sub",
1032 "__tsan_atomic32_fetch_sub",
1033 "__tsan_atomic64_fetch_sub",
1034 "__tsan_atomic8_fetch_and",
1035 "__tsan_atomic16_fetch_and",
1036 "__tsan_atomic32_fetch_and",
1037 "__tsan_atomic64_fetch_and",
1038 "__tsan_atomic8_fetch_or",
1039 "__tsan_atomic16_fetch_or",
1040 "__tsan_atomic32_fetch_or",
1041 "__tsan_atomic64_fetch_or",
1042 "__tsan_atomic8_fetch_xor",
1043 "__tsan_atomic16_fetch_xor",
1044 "__tsan_atomic32_fetch_xor",
1045 "__tsan_atomic64_fetch_xor",
1046 "__tsan_atomic8_fetch_nand",
1047 "__tsan_atomic16_fetch_nand",
1048 "__tsan_atomic32_fetch_nand",
1049 "__tsan_atomic64_fetch_nand",
1050 "__tsan_atomic8_compare_exchange_strong",
1051 "__tsan_atomic16_compare_exchange_strong",
1052 "__tsan_atomic32_compare_exchange_strong",
1053 "__tsan_atomic64_compare_exchange_strong",
1054 "__tsan_atomic8_compare_exchange_weak",
1055 "__tsan_atomic16_compare_exchange_weak",
1056 "__tsan_atomic32_compare_exchange_weak",
1057 "__tsan_atomic64_compare_exchange_weak",
1058 "__tsan_atomic8_compare_exchange_val",
1059 "__tsan_atomic16_compare_exchange_val",
1060 "__tsan_atomic32_compare_exchange_val",
1061 "__tsan_atomic64_compare_exchange_val",
1062 "__tsan_atomic_thread_fence",
1063 "__tsan_atomic_signal_fence",
1064 "__tsan_unaligned_read16",
1065 "__tsan_unaligned_write16",
1066 /* KCOV */
1067 "write_comp_data",
1068 "check_kcov_mode",
1069 "__sanitizer_cov_trace_pc",
1070 "__sanitizer_cov_trace_const_cmp1",
1071 "__sanitizer_cov_trace_const_cmp2",
1072 "__sanitizer_cov_trace_const_cmp4",
1073 "__sanitizer_cov_trace_const_cmp8",
1074 "__sanitizer_cov_trace_cmp1",
1075 "__sanitizer_cov_trace_cmp2",
1076 "__sanitizer_cov_trace_cmp4",
1077 "__sanitizer_cov_trace_cmp8",
1078 "__sanitizer_cov_trace_switch",
1079 /* KMSAN */
1080 "kmsan_copy_to_user",
1081 "kmsan_report",
1082 "kmsan_unpoison_entry_regs",
1083 "kmsan_unpoison_memory",
1084 "__msan_chain_origin",
1085 "__msan_get_context_state",
1086 "__msan_instrument_asm_store",
1087 "__msan_metadata_ptr_for_load_1",
1088 "__msan_metadata_ptr_for_load_2",
1089 "__msan_metadata_ptr_for_load_4",
1090 "__msan_metadata_ptr_for_load_8",
1091 "__msan_metadata_ptr_for_load_n",
1092 "__msan_metadata_ptr_for_store_1",
1093 "__msan_metadata_ptr_for_store_2",
1094 "__msan_metadata_ptr_for_store_4",
1095 "__msan_metadata_ptr_for_store_8",
1096 "__msan_metadata_ptr_for_store_n",
1097 "__msan_poison_alloca",
1098 "__msan_warning",
1099 /* UBSAN */
1100 "ubsan_type_mismatch_common",
1101 "__ubsan_handle_type_mismatch",
1102 "__ubsan_handle_type_mismatch_v1",
1103 "__ubsan_handle_shift_out_of_bounds",
1104 /* misc */
1105 "csum_partial_copy_generic",
1106 "copy_mc_fragile",
1107 "copy_mc_fragile_handle_tail",
1108 "copy_mc_enhanced_fast_string",
1109 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1110 "clear_user_erms",
1111 "clear_user_rep_good",
1112 "clear_user_original",
1113 NULL
1114 };
1115
add_uaccess_safe(struct objtool_file * file)1116 static void add_uaccess_safe(struct objtool_file *file)
1117 {
1118 struct symbol *func;
1119 const char **name;
1120
1121 if (!opts.uaccess)
1122 return;
1123
1124 for (name = uaccess_safe_builtin; *name; name++) {
1125 func = find_symbol_by_name(file->elf, *name);
1126 if (!func)
1127 continue;
1128
1129 func->uaccess_safe = true;
1130 }
1131 }
1132
1133 /*
1134 * FIXME: For now, just ignore any alternatives which add retpolines. This is
1135 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1136 * But it at least allows objtool to understand the control flow *around* the
1137 * retpoline.
1138 */
add_ignore_alternatives(struct objtool_file * file)1139 static int add_ignore_alternatives(struct objtool_file *file)
1140 {
1141 struct section *sec;
1142 struct reloc *reloc;
1143 struct instruction *insn;
1144
1145 sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1146 if (!sec)
1147 return 0;
1148
1149 list_for_each_entry(reloc, &sec->reloc_list, list) {
1150 if (reloc->sym->type != STT_SECTION) {
1151 WARN("unexpected relocation symbol type in %s", sec->name);
1152 return -1;
1153 }
1154
1155 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1156 if (!insn) {
1157 WARN("bad .discard.ignore_alts entry");
1158 return -1;
1159 }
1160
1161 insn->ignore_alts = true;
1162 }
1163
1164 return 0;
1165 }
1166
1167 /*
1168 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1169 * will be added to the .retpoline_sites section.
1170 */
arch_is_retpoline(struct symbol * sym)1171 __weak bool arch_is_retpoline(struct symbol *sym)
1172 {
1173 return false;
1174 }
1175
1176 /*
1177 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1178 * will be added to the .return_sites section.
1179 */
arch_is_rethunk(struct symbol * sym)1180 __weak bool arch_is_rethunk(struct symbol *sym)
1181 {
1182 return false;
1183 }
1184
1185 /*
1186 * Symbols that are embedded inside other instructions, because sometimes crazy
1187 * code exists. These are mostly ignored for validation purposes.
1188 */
arch_is_embedded_insn(struct symbol * sym)1189 __weak bool arch_is_embedded_insn(struct symbol *sym)
1190 {
1191 return false;
1192 }
1193
1194 #define NEGATIVE_RELOC ((void *)-1L)
1195
insn_reloc(struct objtool_file * file,struct instruction * insn)1196 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1197 {
1198 if (insn->reloc == NEGATIVE_RELOC)
1199 return NULL;
1200
1201 if (!insn->reloc) {
1202 if (!file)
1203 return NULL;
1204
1205 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1206 insn->offset, insn->len);
1207 if (!insn->reloc) {
1208 insn->reloc = NEGATIVE_RELOC;
1209 return NULL;
1210 }
1211 }
1212
1213 return insn->reloc;
1214 }
1215
remove_insn_ops(struct instruction * insn)1216 static void remove_insn_ops(struct instruction *insn)
1217 {
1218 struct stack_op *op, *tmp;
1219
1220 list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
1221 list_del(&op->list);
1222 free(op);
1223 }
1224 }
1225
annotate_call_site(struct objtool_file * file,struct instruction * insn,bool sibling)1226 static void annotate_call_site(struct objtool_file *file,
1227 struct instruction *insn, bool sibling)
1228 {
1229 struct reloc *reloc = insn_reloc(file, insn);
1230 struct symbol *sym = insn->call_dest;
1231
1232 if (!sym)
1233 sym = reloc->sym;
1234
1235 /*
1236 * Alternative replacement code is just template code which is
1237 * sometimes copied to the original instruction. For now, don't
1238 * annotate it. (In the future we might consider annotating the
1239 * original instruction if/when it ever makes sense to do so.)
1240 */
1241 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1242 return;
1243
1244 if (sym->static_call_tramp) {
1245 list_add_tail(&insn->call_node, &file->static_call_list);
1246 return;
1247 }
1248
1249 if (sym->retpoline_thunk) {
1250 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1251 return;
1252 }
1253
1254 /*
1255 * Many compilers cannot disable KCOV or sanitizer calls with a function
1256 * attribute so they need a little help, NOP out any such calls from
1257 * noinstr text.
1258 */
1259 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1260 if (reloc) {
1261 reloc->type = R_NONE;
1262 elf_write_reloc(file->elf, reloc);
1263 }
1264
1265 elf_write_insn(file->elf, insn->sec,
1266 insn->offset, insn->len,
1267 sibling ? arch_ret_insn(insn->len)
1268 : arch_nop_insn(insn->len));
1269
1270 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1271
1272 if (sibling) {
1273 /*
1274 * We've replaced the tail-call JMP insn by two new
1275 * insn: RET; INT3, except we only have a single struct
1276 * insn here. Mark it retpoline_safe to avoid the SLS
1277 * warning, instead of adding another insn.
1278 */
1279 insn->retpoline_safe = true;
1280 }
1281
1282 return;
1283 }
1284
1285 if (opts.mcount && sym->fentry) {
1286 if (sibling)
1287 WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
1288
1289 if (reloc) {
1290 reloc->type = R_NONE;
1291 elf_write_reloc(file->elf, reloc);
1292 }
1293
1294 elf_write_insn(file->elf, insn->sec,
1295 insn->offset, insn->len,
1296 arch_nop_insn(insn->len));
1297
1298 insn->type = INSN_NOP;
1299
1300 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1301 return;
1302 }
1303
1304 if (!sibling && dead_end_function(file, sym))
1305 insn->dead_end = true;
1306 }
1307
add_call_dest(struct objtool_file * file,struct instruction * insn,struct symbol * dest,bool sibling)1308 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1309 struct symbol *dest, bool sibling)
1310 {
1311 insn->call_dest = dest;
1312 if (!dest)
1313 return;
1314
1315 /*
1316 * Whatever stack impact regular CALLs have, should be undone
1317 * by the RETURN of the called function.
1318 *
1319 * Annotated intra-function calls retain the stack_ops but
1320 * are converted to JUMP, see read_intra_function_calls().
1321 */
1322 remove_insn_ops(insn);
1323
1324 annotate_call_site(file, insn, sibling);
1325 }
1326
add_retpoline_call(struct objtool_file * file,struct instruction * insn)1327 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1328 {
1329 /*
1330 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1331 * so convert them accordingly.
1332 */
1333 switch (insn->type) {
1334 case INSN_CALL:
1335 insn->type = INSN_CALL_DYNAMIC;
1336 break;
1337 case INSN_JUMP_UNCONDITIONAL:
1338 insn->type = INSN_JUMP_DYNAMIC;
1339 break;
1340 case INSN_JUMP_CONDITIONAL:
1341 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1342 break;
1343 default:
1344 return;
1345 }
1346
1347 insn->retpoline_safe = true;
1348
1349 /*
1350 * Whatever stack impact regular CALLs have, should be undone
1351 * by the RETURN of the called function.
1352 *
1353 * Annotated intra-function calls retain the stack_ops but
1354 * are converted to JUMP, see read_intra_function_calls().
1355 */
1356 remove_insn_ops(insn);
1357
1358 annotate_call_site(file, insn, false);
1359 }
1360
add_return_call(struct objtool_file * file,struct instruction * insn,bool add)1361 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1362 {
1363 /*
1364 * Return thunk tail calls are really just returns in disguise,
1365 * so convert them accordingly.
1366 */
1367 insn->type = INSN_RETURN;
1368 insn->retpoline_safe = true;
1369
1370 if (add)
1371 list_add_tail(&insn->call_node, &file->return_thunk_list);
1372 }
1373
same_function(struct instruction * insn1,struct instruction * insn2)1374 static bool same_function(struct instruction *insn1, struct instruction *insn2)
1375 {
1376 return insn1->func->pfunc == insn2->func->pfunc;
1377 }
1378
is_first_func_insn(struct objtool_file * file,struct instruction * insn)1379 static bool is_first_func_insn(struct objtool_file *file, struct instruction *insn)
1380 {
1381 if (insn->offset == insn->func->offset)
1382 return true;
1383
1384 if (opts.ibt) {
1385 struct instruction *prev = prev_insn_same_sym(file, insn);
1386
1387 if (prev && prev->type == INSN_ENDBR &&
1388 insn->offset == insn->func->offset + prev->len)
1389 return true;
1390 }
1391
1392 return false;
1393 }
1394
1395 /*
1396 * Find the destination instructions for all jumps.
1397 */
add_jump_destinations(struct objtool_file * file)1398 static int add_jump_destinations(struct objtool_file *file)
1399 {
1400 struct instruction *insn, *jump_dest;
1401 struct reloc *reloc;
1402 struct section *dest_sec;
1403 unsigned long dest_off;
1404
1405 for_each_insn(file, insn) {
1406 if (insn->jump_dest) {
1407 /*
1408 * handle_group_alt() may have previously set
1409 * 'jump_dest' for some alternatives.
1410 */
1411 continue;
1412 }
1413 if (!is_static_jump(insn))
1414 continue;
1415
1416 reloc = insn_reloc(file, insn);
1417 if (!reloc) {
1418 dest_sec = insn->sec;
1419 dest_off = arch_jump_destination(insn);
1420 } else if (reloc->sym->type == STT_SECTION) {
1421 dest_sec = reloc->sym->sec;
1422 dest_off = arch_dest_reloc_offset(reloc->addend);
1423 } else if (reloc->sym->retpoline_thunk) {
1424 add_retpoline_call(file, insn);
1425 continue;
1426 } else if (reloc->sym->return_thunk) {
1427 add_return_call(file, insn, true);
1428 continue;
1429 } else if (insn->func) {
1430 /*
1431 * External sibling call or internal sibling call with
1432 * STT_FUNC reloc.
1433 */
1434 add_call_dest(file, insn, reloc->sym, true);
1435 continue;
1436 } else if (reloc->sym->sec->idx) {
1437 dest_sec = reloc->sym->sec;
1438 dest_off = reloc->sym->sym.st_value +
1439 arch_dest_reloc_offset(reloc->addend);
1440 } else {
1441 /* non-func asm code jumping to another file */
1442 continue;
1443 }
1444
1445 jump_dest = find_insn(file, dest_sec, dest_off);
1446 if (!jump_dest) {
1447 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1448
1449 /*
1450 * This is a special case for retbleed_untrain_ret().
1451 * It jumps to __x86_return_thunk(), but objtool
1452 * can't find the thunk's starting RET
1453 * instruction, because the RET is also in the
1454 * middle of another instruction. Objtool only
1455 * knows about the outer instruction.
1456 */
1457 if (sym && sym->embedded_insn) {
1458 add_return_call(file, insn, false);
1459 continue;
1460 }
1461
1462 WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
1463 insn->sec, insn->offset, dest_sec->name,
1464 dest_off);
1465 return -1;
1466 }
1467
1468 /*
1469 * Cross-function jump.
1470 */
1471 if (insn->func && jump_dest->func &&
1472 insn->func != jump_dest->func) {
1473
1474 /*
1475 * For GCC 8+, create parent/child links for any cold
1476 * subfunctions. This is _mostly_ redundant with a
1477 * similar initialization in read_symbols().
1478 *
1479 * If a function has aliases, we want the *first* such
1480 * function in the symbol table to be the subfunction's
1481 * parent. In that case we overwrite the
1482 * initialization done in read_symbols().
1483 *
1484 * However this code can't completely replace the
1485 * read_symbols() code because this doesn't detect the
1486 * case where the parent function's only reference to a
1487 * subfunction is through a jump table.
1488 */
1489 if (!strstr(insn->func->name, ".cold") &&
1490 strstr(jump_dest->func->name, ".cold")) {
1491 insn->func->cfunc = jump_dest->func;
1492 jump_dest->func->pfunc = insn->func;
1493
1494 } else if (!same_function(insn, jump_dest) &&
1495 is_first_func_insn(file, jump_dest)) {
1496 /*
1497 * Internal sibling call without reloc or with
1498 * STT_SECTION reloc.
1499 */
1500 add_call_dest(file, insn, jump_dest->func, true);
1501 continue;
1502 }
1503 }
1504
1505 insn->jump_dest = jump_dest;
1506 }
1507
1508 return 0;
1509 }
1510
find_call_destination(struct section * sec,unsigned long offset)1511 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1512 {
1513 struct symbol *call_dest;
1514
1515 call_dest = find_func_by_offset(sec, offset);
1516 if (!call_dest)
1517 call_dest = find_symbol_by_offset(sec, offset);
1518
1519 return call_dest;
1520 }
1521
1522 /*
1523 * Find the destination instructions for all calls.
1524 */
add_call_destinations(struct objtool_file * file)1525 static int add_call_destinations(struct objtool_file *file)
1526 {
1527 struct instruction *insn;
1528 unsigned long dest_off;
1529 struct symbol *dest;
1530 struct reloc *reloc;
1531
1532 for_each_insn(file, insn) {
1533 if (insn->type != INSN_CALL)
1534 continue;
1535
1536 reloc = insn_reloc(file, insn);
1537 if (!reloc) {
1538 dest_off = arch_jump_destination(insn);
1539 dest = find_call_destination(insn->sec, dest_off);
1540
1541 add_call_dest(file, insn, dest, false);
1542
1543 if (insn->ignore)
1544 continue;
1545
1546 if (!insn->call_dest) {
1547 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
1548 return -1;
1549 }
1550
1551 if (insn->func && insn->call_dest->type != STT_FUNC) {
1552 WARN_FUNC("unsupported call to non-function",
1553 insn->sec, insn->offset);
1554 return -1;
1555 }
1556
1557 } else if (reloc->sym->type == STT_SECTION) {
1558 dest_off = arch_dest_reloc_offset(reloc->addend);
1559 dest = find_call_destination(reloc->sym->sec, dest_off);
1560 if (!dest) {
1561 WARN_FUNC("can't find call dest symbol at %s+0x%lx",
1562 insn->sec, insn->offset,
1563 reloc->sym->sec->name,
1564 dest_off);
1565 return -1;
1566 }
1567
1568 add_call_dest(file, insn, dest, false);
1569
1570 } else if (reloc->sym->retpoline_thunk) {
1571 add_retpoline_call(file, insn);
1572
1573 } else
1574 add_call_dest(file, insn, reloc->sym, false);
1575 }
1576
1577 return 0;
1578 }
1579
1580 /*
1581 * The .alternatives section requires some extra special care over and above
1582 * other special sections because alternatives are patched in place.
1583 */
handle_group_alt(struct objtool_file * file,struct special_alt * special_alt,struct instruction * orig_insn,struct instruction ** new_insn)1584 static int handle_group_alt(struct objtool_file *file,
1585 struct special_alt *special_alt,
1586 struct instruction *orig_insn,
1587 struct instruction **new_insn)
1588 {
1589 struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
1590 struct alt_group *orig_alt_group, *new_alt_group;
1591 unsigned long dest_off;
1592
1593
1594 orig_alt_group = malloc(sizeof(*orig_alt_group));
1595 if (!orig_alt_group) {
1596 WARN("malloc failed");
1597 return -1;
1598 }
1599 orig_alt_group->cfi = calloc(special_alt->orig_len,
1600 sizeof(struct cfi_state *));
1601 if (!orig_alt_group->cfi) {
1602 WARN("calloc failed");
1603 return -1;
1604 }
1605
1606 last_orig_insn = NULL;
1607 insn = orig_insn;
1608 sec_for_each_insn_from(file, insn) {
1609 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1610 break;
1611
1612 insn->alt_group = orig_alt_group;
1613 last_orig_insn = insn;
1614 }
1615 orig_alt_group->orig_group = NULL;
1616 orig_alt_group->first_insn = orig_insn;
1617 orig_alt_group->last_insn = last_orig_insn;
1618
1619
1620 new_alt_group = malloc(sizeof(*new_alt_group));
1621 if (!new_alt_group) {
1622 WARN("malloc failed");
1623 return -1;
1624 }
1625
1626 if (special_alt->new_len < special_alt->orig_len) {
1627 /*
1628 * Insert a fake nop at the end to make the replacement
1629 * alt_group the same size as the original. This is needed to
1630 * allow propagate_alt_cfi() to do its magic. When the last
1631 * instruction affects the stack, the instruction after it (the
1632 * nop) will propagate the new state to the shared CFI array.
1633 */
1634 nop = malloc(sizeof(*nop));
1635 if (!nop) {
1636 WARN("malloc failed");
1637 return -1;
1638 }
1639 memset(nop, 0, sizeof(*nop));
1640 INIT_LIST_HEAD(&nop->alts);
1641 INIT_LIST_HEAD(&nop->stack_ops);
1642
1643 nop->sec = special_alt->new_sec;
1644 nop->offset = special_alt->new_off + special_alt->new_len;
1645 nop->len = special_alt->orig_len - special_alt->new_len;
1646 nop->type = INSN_NOP;
1647 nop->func = orig_insn->func;
1648 nop->alt_group = new_alt_group;
1649 nop->ignore = orig_insn->ignore_alts;
1650 }
1651
1652 if (!special_alt->new_len) {
1653 *new_insn = nop;
1654 goto end;
1655 }
1656
1657 insn = *new_insn;
1658 sec_for_each_insn_from(file, insn) {
1659 struct reloc *alt_reloc;
1660
1661 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1662 break;
1663
1664 last_new_insn = insn;
1665
1666 insn->ignore = orig_insn->ignore_alts;
1667 insn->func = orig_insn->func;
1668 insn->alt_group = new_alt_group;
1669
1670 /*
1671 * Since alternative replacement code is copy/pasted by the
1672 * kernel after applying relocations, generally such code can't
1673 * have relative-address relocation references to outside the
1674 * .altinstr_replacement section, unless the arch's
1675 * alternatives code can adjust the relative offsets
1676 * accordingly.
1677 */
1678 alt_reloc = insn_reloc(file, insn);
1679 if (alt_reloc &&
1680 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1681
1682 WARN_FUNC("unsupported relocation in alternatives section",
1683 insn->sec, insn->offset);
1684 return -1;
1685 }
1686
1687 if (!is_static_jump(insn))
1688 continue;
1689
1690 if (!insn->immediate)
1691 continue;
1692
1693 dest_off = arch_jump_destination(insn);
1694 if (dest_off == special_alt->new_off + special_alt->new_len) {
1695 insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
1696 if (!insn->jump_dest) {
1697 WARN_FUNC("can't find alternative jump destination",
1698 insn->sec, insn->offset);
1699 return -1;
1700 }
1701 }
1702 }
1703
1704 if (!last_new_insn) {
1705 WARN_FUNC("can't find last new alternative instruction",
1706 special_alt->new_sec, special_alt->new_off);
1707 return -1;
1708 }
1709
1710 if (nop)
1711 list_add(&nop->list, &last_new_insn->list);
1712 end:
1713 new_alt_group->orig_group = orig_alt_group;
1714 new_alt_group->first_insn = *new_insn;
1715 new_alt_group->last_insn = nop ? : last_new_insn;
1716 new_alt_group->cfi = orig_alt_group->cfi;
1717 return 0;
1718 }
1719
1720 /*
1721 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1722 * If the original instruction is a jump, make the alt entry an effective nop
1723 * by just skipping the original instruction.
1724 */
handle_jump_alt(struct objtool_file * file,struct special_alt * special_alt,struct instruction * orig_insn,struct instruction ** new_insn)1725 static int handle_jump_alt(struct objtool_file *file,
1726 struct special_alt *special_alt,
1727 struct instruction *orig_insn,
1728 struct instruction **new_insn)
1729 {
1730 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1731 orig_insn->type != INSN_NOP) {
1732
1733 WARN_FUNC("unsupported instruction at jump label",
1734 orig_insn->sec, orig_insn->offset);
1735 return -1;
1736 }
1737
1738 if (opts.hack_jump_label && special_alt->key_addend & 2) {
1739 struct reloc *reloc = insn_reloc(file, orig_insn);
1740
1741 if (reloc) {
1742 reloc->type = R_NONE;
1743 elf_write_reloc(file->elf, reloc);
1744 }
1745 elf_write_insn(file->elf, orig_insn->sec,
1746 orig_insn->offset, orig_insn->len,
1747 arch_nop_insn(orig_insn->len));
1748 orig_insn->type = INSN_NOP;
1749 }
1750
1751 if (orig_insn->type == INSN_NOP) {
1752 if (orig_insn->len == 2)
1753 file->jl_nop_short++;
1754 else
1755 file->jl_nop_long++;
1756
1757 return 0;
1758 }
1759
1760 if (orig_insn->len == 2)
1761 file->jl_short++;
1762 else
1763 file->jl_long++;
1764
1765 *new_insn = list_next_entry(orig_insn, list);
1766 return 0;
1767 }
1768
1769 /*
1770 * Read all the special sections which have alternate instructions which can be
1771 * patched in or redirected to at runtime. Each instruction having alternate
1772 * instruction(s) has them added to its insn->alts list, which will be
1773 * traversed in validate_branch().
1774 */
add_special_section_alts(struct objtool_file * file)1775 static int add_special_section_alts(struct objtool_file *file)
1776 {
1777 struct list_head special_alts;
1778 struct instruction *orig_insn, *new_insn;
1779 struct special_alt *special_alt, *tmp;
1780 struct alternative *alt;
1781 int ret;
1782
1783 ret = special_get_alts(file->elf, &special_alts);
1784 if (ret)
1785 return ret;
1786
1787 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1788
1789 orig_insn = find_insn(file, special_alt->orig_sec,
1790 special_alt->orig_off);
1791 if (!orig_insn) {
1792 WARN_FUNC("special: can't find orig instruction",
1793 special_alt->orig_sec, special_alt->orig_off);
1794 ret = -1;
1795 goto out;
1796 }
1797
1798 new_insn = NULL;
1799 if (!special_alt->group || special_alt->new_len) {
1800 new_insn = find_insn(file, special_alt->new_sec,
1801 special_alt->new_off);
1802 if (!new_insn) {
1803 WARN_FUNC("special: can't find new instruction",
1804 special_alt->new_sec,
1805 special_alt->new_off);
1806 ret = -1;
1807 goto out;
1808 }
1809 }
1810
1811 if (special_alt->group) {
1812 if (!special_alt->orig_len) {
1813 WARN_FUNC("empty alternative entry",
1814 orig_insn->sec, orig_insn->offset);
1815 continue;
1816 }
1817
1818 ret = handle_group_alt(file, special_alt, orig_insn,
1819 &new_insn);
1820 if (ret)
1821 goto out;
1822 } else if (special_alt->jump_or_nop) {
1823 ret = handle_jump_alt(file, special_alt, orig_insn,
1824 &new_insn);
1825 if (ret)
1826 goto out;
1827 }
1828
1829 alt = malloc(sizeof(*alt));
1830 if (!alt) {
1831 WARN("malloc failed");
1832 ret = -1;
1833 goto out;
1834 }
1835
1836 alt->insn = new_insn;
1837 alt->skip_orig = special_alt->skip_orig;
1838 orig_insn->ignore_alts |= special_alt->skip_alt;
1839 list_add_tail(&alt->list, &orig_insn->alts);
1840
1841 list_del(&special_alt->list);
1842 free(special_alt);
1843 }
1844
1845 if (opts.stats) {
1846 printf("jl\\\tNOP\tJMP\n");
1847 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1848 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1849 }
1850
1851 out:
1852 return ret;
1853 }
1854
add_jump_table(struct objtool_file * file,struct instruction * insn,struct reloc * table)1855 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
1856 struct reloc *table)
1857 {
1858 struct reloc *reloc = table;
1859 struct instruction *dest_insn;
1860 struct alternative *alt;
1861 struct symbol *pfunc = insn->func->pfunc;
1862 unsigned int prev_offset = 0;
1863
1864 /*
1865 * Each @reloc is a switch table relocation which points to the target
1866 * instruction.
1867 */
1868 list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
1869
1870 /* Check for the end of the table: */
1871 if (reloc != table && reloc->jump_table_start)
1872 break;
1873
1874 /* Make sure the table entries are consecutive: */
1875 if (prev_offset && reloc->offset != prev_offset + 8)
1876 break;
1877
1878 /* Detect function pointers from contiguous objects: */
1879 if (reloc->sym->sec == pfunc->sec &&
1880 reloc->addend == pfunc->offset)
1881 break;
1882
1883 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
1884 if (!dest_insn)
1885 break;
1886
1887 /* Make sure the destination is in the same function: */
1888 if (!dest_insn->func || dest_insn->func->pfunc != pfunc)
1889 break;
1890
1891 alt = malloc(sizeof(*alt));
1892 if (!alt) {
1893 WARN("malloc failed");
1894 return -1;
1895 }
1896
1897 alt->insn = dest_insn;
1898 list_add_tail(&alt->list, &insn->alts);
1899 prev_offset = reloc->offset;
1900 }
1901
1902 if (!prev_offset) {
1903 WARN_FUNC("can't find switch jump table",
1904 insn->sec, insn->offset);
1905 return -1;
1906 }
1907
1908 return 0;
1909 }
1910
1911 /*
1912 * find_jump_table() - Given a dynamic jump, find the switch jump table
1913 * associated with it.
1914 */
find_jump_table(struct objtool_file * file,struct symbol * func,struct instruction * insn)1915 static struct reloc *find_jump_table(struct objtool_file *file,
1916 struct symbol *func,
1917 struct instruction *insn)
1918 {
1919 struct reloc *table_reloc;
1920 struct instruction *dest_insn, *orig_insn = insn;
1921
1922 /*
1923 * Backward search using the @first_jump_src links, these help avoid
1924 * much of the 'in between' code. Which avoids us getting confused by
1925 * it.
1926 */
1927 for (;
1928 insn && insn->func && insn->func->pfunc == func;
1929 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
1930
1931 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
1932 break;
1933
1934 /* allow small jumps within the range */
1935 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
1936 insn->jump_dest &&
1937 (insn->jump_dest->offset <= insn->offset ||
1938 insn->jump_dest->offset > orig_insn->offset))
1939 break;
1940
1941 table_reloc = arch_find_switch_table(file, insn);
1942 if (!table_reloc)
1943 continue;
1944 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
1945 if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func)
1946 continue;
1947
1948 return table_reloc;
1949 }
1950
1951 return NULL;
1952 }
1953
1954 /*
1955 * First pass: Mark the head of each jump table so that in the next pass,
1956 * we know when a given jump table ends and the next one starts.
1957 */
mark_func_jump_tables(struct objtool_file * file,struct symbol * func)1958 static void mark_func_jump_tables(struct objtool_file *file,
1959 struct symbol *func)
1960 {
1961 struct instruction *insn, *last = NULL;
1962 struct reloc *reloc;
1963
1964 func_for_each_insn(file, func, insn) {
1965 if (!last)
1966 last = insn;
1967
1968 /*
1969 * Store back-pointers for unconditional forward jumps such
1970 * that find_jump_table() can back-track using those and
1971 * avoid some potentially confusing code.
1972 */
1973 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
1974 insn->offset > last->offset &&
1975 insn->jump_dest->offset > insn->offset &&
1976 !insn->jump_dest->first_jump_src) {
1977
1978 insn->jump_dest->first_jump_src = insn;
1979 last = insn->jump_dest;
1980 }
1981
1982 if (insn->type != INSN_JUMP_DYNAMIC)
1983 continue;
1984
1985 reloc = find_jump_table(file, func, insn);
1986 if (reloc) {
1987 reloc->jump_table_start = true;
1988 insn->jump_table = reloc;
1989 }
1990 }
1991 }
1992
add_func_jump_tables(struct objtool_file * file,struct symbol * func)1993 static int add_func_jump_tables(struct objtool_file *file,
1994 struct symbol *func)
1995 {
1996 struct instruction *insn;
1997 int ret;
1998
1999 func_for_each_insn(file, func, insn) {
2000 if (!insn->jump_table)
2001 continue;
2002
2003 ret = add_jump_table(file, insn, insn->jump_table);
2004 if (ret)
2005 return ret;
2006 }
2007
2008 return 0;
2009 }
2010
2011 /*
2012 * For some switch statements, gcc generates a jump table in the .rodata
2013 * section which contains a list of addresses within the function to jump to.
2014 * This finds these jump tables and adds them to the insn->alts lists.
2015 */
add_jump_table_alts(struct objtool_file * file)2016 static int add_jump_table_alts(struct objtool_file *file)
2017 {
2018 struct section *sec;
2019 struct symbol *func;
2020 int ret;
2021
2022 if (!file->rodata)
2023 return 0;
2024
2025 for_each_sec(file, sec) {
2026 list_for_each_entry(func, &sec->symbol_list, list) {
2027 if (func->type != STT_FUNC)
2028 continue;
2029
2030 mark_func_jump_tables(file, func);
2031 ret = add_func_jump_tables(file, func);
2032 if (ret)
2033 return ret;
2034 }
2035 }
2036
2037 return 0;
2038 }
2039
set_func_state(struct cfi_state * state)2040 static void set_func_state(struct cfi_state *state)
2041 {
2042 state->cfa = initial_func_cfi.cfa;
2043 memcpy(&state->regs, &initial_func_cfi.regs,
2044 CFI_NUM_REGS * sizeof(struct cfi_reg));
2045 state->stack_size = initial_func_cfi.cfa.offset;
2046 }
2047
read_unwind_hints(struct objtool_file * file)2048 static int read_unwind_hints(struct objtool_file *file)
2049 {
2050 struct cfi_state cfi = init_cfi;
2051 struct section *sec, *relocsec;
2052 struct unwind_hint *hint;
2053 struct instruction *insn;
2054 struct reloc *reloc;
2055 int i;
2056
2057 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2058 if (!sec)
2059 return 0;
2060
2061 relocsec = sec->reloc;
2062 if (!relocsec) {
2063 WARN("missing .rela.discard.unwind_hints section");
2064 return -1;
2065 }
2066
2067 if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2068 WARN("struct unwind_hint size mismatch");
2069 return -1;
2070 }
2071
2072 file->hints = true;
2073
2074 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2075 hint = (struct unwind_hint *)sec->data->d_buf + i;
2076
2077 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2078 if (!reloc) {
2079 WARN("can't find reloc for unwind_hints[%d]", i);
2080 return -1;
2081 }
2082
2083 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2084 if (!insn) {
2085 WARN("can't find insn for unwind_hints[%d]", i);
2086 return -1;
2087 }
2088
2089 insn->hint = true;
2090
2091 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2092 insn->hint = false;
2093 insn->save = true;
2094 continue;
2095 }
2096
2097 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2098 insn->restore = true;
2099 continue;
2100 }
2101
2102 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2103 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2104
2105 if (sym && sym->bind == STB_GLOBAL) {
2106 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2107 WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR",
2108 insn->sec, insn->offset);
2109 }
2110
2111 insn->entry = 1;
2112 }
2113 }
2114
2115 if (hint->type == UNWIND_HINT_TYPE_ENTRY) {
2116 hint->type = UNWIND_HINT_TYPE_CALL;
2117 insn->entry = 1;
2118 }
2119
2120 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2121 insn->cfi = &func_cfi;
2122 continue;
2123 }
2124
2125 if (insn->cfi)
2126 cfi = *(insn->cfi);
2127
2128 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2129 WARN_FUNC("unsupported unwind_hint sp base reg %d",
2130 insn->sec, insn->offset, hint->sp_reg);
2131 return -1;
2132 }
2133
2134 cfi.cfa.offset = bswap_if_needed(hint->sp_offset);
2135 cfi.type = hint->type;
2136 cfi.end = hint->end;
2137
2138 insn->cfi = cfi_hash_find_or_add(&cfi);
2139 }
2140
2141 return 0;
2142 }
2143
read_noendbr_hints(struct objtool_file * file)2144 static int read_noendbr_hints(struct objtool_file *file)
2145 {
2146 struct section *sec;
2147 struct instruction *insn;
2148 struct reloc *reloc;
2149
2150 sec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2151 if (!sec)
2152 return 0;
2153
2154 list_for_each_entry(reloc, &sec->reloc_list, list) {
2155 insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend);
2156 if (!insn) {
2157 WARN("bad .discard.noendbr entry");
2158 return -1;
2159 }
2160
2161 insn->noendbr = 1;
2162 }
2163
2164 return 0;
2165 }
2166
read_retpoline_hints(struct objtool_file * file)2167 static int read_retpoline_hints(struct objtool_file *file)
2168 {
2169 struct section *sec;
2170 struct instruction *insn;
2171 struct reloc *reloc;
2172
2173 sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2174 if (!sec)
2175 return 0;
2176
2177 list_for_each_entry(reloc, &sec->reloc_list, list) {
2178 if (reloc->sym->type != STT_SECTION) {
2179 WARN("unexpected relocation symbol type in %s", sec->name);
2180 return -1;
2181 }
2182
2183 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2184 if (!insn) {
2185 WARN("bad .discard.retpoline_safe entry");
2186 return -1;
2187 }
2188
2189 if (insn->type != INSN_JUMP_DYNAMIC &&
2190 insn->type != INSN_CALL_DYNAMIC &&
2191 insn->type != INSN_RETURN &&
2192 insn->type != INSN_NOP) {
2193 WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop",
2194 insn->sec, insn->offset);
2195 return -1;
2196 }
2197
2198 insn->retpoline_safe = true;
2199 }
2200
2201 return 0;
2202 }
2203
read_instr_hints(struct objtool_file * file)2204 static int read_instr_hints(struct objtool_file *file)
2205 {
2206 struct section *sec;
2207 struct instruction *insn;
2208 struct reloc *reloc;
2209
2210 sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2211 if (!sec)
2212 return 0;
2213
2214 list_for_each_entry(reloc, &sec->reloc_list, list) {
2215 if (reloc->sym->type != STT_SECTION) {
2216 WARN("unexpected relocation symbol type in %s", sec->name);
2217 return -1;
2218 }
2219
2220 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2221 if (!insn) {
2222 WARN("bad .discard.instr_end entry");
2223 return -1;
2224 }
2225
2226 insn->instr--;
2227 }
2228
2229 sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2230 if (!sec)
2231 return 0;
2232
2233 list_for_each_entry(reloc, &sec->reloc_list, list) {
2234 if (reloc->sym->type != STT_SECTION) {
2235 WARN("unexpected relocation symbol type in %s", sec->name);
2236 return -1;
2237 }
2238
2239 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2240 if (!insn) {
2241 WARN("bad .discard.instr_begin entry");
2242 return -1;
2243 }
2244
2245 insn->instr++;
2246 }
2247
2248 return 0;
2249 }
2250
read_intra_function_calls(struct objtool_file * file)2251 static int read_intra_function_calls(struct objtool_file *file)
2252 {
2253 struct instruction *insn;
2254 struct section *sec;
2255 struct reloc *reloc;
2256
2257 sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2258 if (!sec)
2259 return 0;
2260
2261 list_for_each_entry(reloc, &sec->reloc_list, list) {
2262 unsigned long dest_off;
2263
2264 if (reloc->sym->type != STT_SECTION) {
2265 WARN("unexpected relocation symbol type in %s",
2266 sec->name);
2267 return -1;
2268 }
2269
2270 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2271 if (!insn) {
2272 WARN("bad .discard.intra_function_call entry");
2273 return -1;
2274 }
2275
2276 if (insn->type != INSN_CALL) {
2277 WARN_FUNC("intra_function_call not a direct call",
2278 insn->sec, insn->offset);
2279 return -1;
2280 }
2281
2282 /*
2283 * Treat intra-function CALLs as JMPs, but with a stack_op.
2284 * See add_call_destinations(), which strips stack_ops from
2285 * normal CALLs.
2286 */
2287 insn->type = INSN_JUMP_UNCONDITIONAL;
2288
2289 dest_off = arch_jump_destination(insn);
2290 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2291 if (!insn->jump_dest) {
2292 WARN_FUNC("can't find call dest at %s+0x%lx",
2293 insn->sec, insn->offset,
2294 insn->sec->name, dest_off);
2295 return -1;
2296 }
2297 }
2298
2299 return 0;
2300 }
2301
2302 /*
2303 * Return true if name matches an instrumentation function, where calls to that
2304 * function from noinstr code can safely be removed, but compilers won't do so.
2305 */
is_profiling_func(const char * name)2306 static bool is_profiling_func(const char *name)
2307 {
2308 /*
2309 * Many compilers cannot disable KCOV with a function attribute.
2310 */
2311 if (!strncmp(name, "__sanitizer_cov_", 16))
2312 return true;
2313
2314 /*
2315 * Some compilers currently do not remove __tsan_func_entry/exit nor
2316 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2317 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2318 * minimum Clang version is 14.0, this can be removed.
2319 */
2320 if (!strncmp(name, "__tsan_func_", 12) ||
2321 !strcmp(name, "__tsan_atomic_signal_fence"))
2322 return true;
2323
2324 return false;
2325 }
2326
classify_symbols(struct objtool_file * file)2327 static int classify_symbols(struct objtool_file *file)
2328 {
2329 struct section *sec;
2330 struct symbol *func;
2331
2332 for_each_sec(file, sec) {
2333 list_for_each_entry(func, &sec->symbol_list, list) {
2334 if (func->bind != STB_GLOBAL)
2335 continue;
2336
2337 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2338 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2339 func->static_call_tramp = true;
2340
2341 if (arch_is_retpoline(func))
2342 func->retpoline_thunk = true;
2343
2344 if (arch_is_rethunk(func))
2345 func->return_thunk = true;
2346
2347 if (arch_is_embedded_insn(func))
2348 func->embedded_insn = true;
2349
2350 if (!strcmp(func->name, "__fentry__"))
2351 func->fentry = true;
2352
2353 if (is_profiling_func(func->name))
2354 func->profiling_func = true;
2355 }
2356 }
2357
2358 return 0;
2359 }
2360
mark_rodata(struct objtool_file * file)2361 static void mark_rodata(struct objtool_file *file)
2362 {
2363 struct section *sec;
2364 bool found = false;
2365
2366 /*
2367 * Search for the following rodata sections, each of which can
2368 * potentially contain jump tables:
2369 *
2370 * - .rodata: can contain GCC switch tables
2371 * - .rodata.<func>: same, if -fdata-sections is being used
2372 * - .rodata..c_jump_table: contains C annotated jump tables
2373 *
2374 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2375 */
2376 for_each_sec(file, sec) {
2377 if (!strncmp(sec->name, ".rodata", 7) &&
2378 !strstr(sec->name, ".str1.")) {
2379 sec->rodata = true;
2380 found = true;
2381 }
2382 }
2383
2384 file->rodata = found;
2385 }
2386
decode_sections(struct objtool_file * file)2387 static int decode_sections(struct objtool_file *file)
2388 {
2389 int ret;
2390
2391 mark_rodata(file);
2392
2393 ret = init_pv_ops(file);
2394 if (ret)
2395 return ret;
2396
2397 ret = decode_instructions(file);
2398 if (ret)
2399 return ret;
2400
2401 add_ignores(file);
2402 add_uaccess_safe(file);
2403
2404 ret = add_ignore_alternatives(file);
2405 if (ret)
2406 return ret;
2407
2408 /*
2409 * Must be before read_unwind_hints() since that needs insn->noendbr.
2410 */
2411 ret = read_noendbr_hints(file);
2412 if (ret)
2413 return ret;
2414
2415 /*
2416 * Must be before add_{jump_call}_destination.
2417 */
2418 ret = classify_symbols(file);
2419 if (ret)
2420 return ret;
2421
2422 /*
2423 * Must be before add_jump_destinations(), which depends on 'func'
2424 * being set for alternatives, to enable proper sibling call detection.
2425 */
2426 ret = add_special_section_alts(file);
2427 if (ret)
2428 return ret;
2429
2430 ret = add_jump_destinations(file);
2431 if (ret)
2432 return ret;
2433
2434 /*
2435 * Must be before add_call_destination(); it changes INSN_CALL to
2436 * INSN_JUMP.
2437 */
2438 ret = read_intra_function_calls(file);
2439 if (ret)
2440 return ret;
2441
2442 ret = add_call_destinations(file);
2443 if (ret)
2444 return ret;
2445
2446 /*
2447 * Must be after add_call_destinations() such that it can override
2448 * dead_end_function() marks.
2449 */
2450 ret = add_dead_ends(file);
2451 if (ret)
2452 return ret;
2453
2454 ret = add_jump_table_alts(file);
2455 if (ret)
2456 return ret;
2457
2458 ret = read_unwind_hints(file);
2459 if (ret)
2460 return ret;
2461
2462 ret = read_retpoline_hints(file);
2463 if (ret)
2464 return ret;
2465
2466 ret = read_instr_hints(file);
2467 if (ret)
2468 return ret;
2469
2470 return 0;
2471 }
2472
is_special_call(struct instruction * insn)2473 static bool is_special_call(struct instruction *insn)
2474 {
2475 if (insn->type == INSN_CALL) {
2476 struct symbol *dest = insn->call_dest;
2477
2478 if (!dest)
2479 return false;
2480
2481 if (dest->fentry || dest->embedded_insn)
2482 return true;
2483 }
2484
2485 return false;
2486 }
2487
has_modified_stack_frame(struct instruction * insn,struct insn_state * state)2488 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2489 {
2490 struct cfi_state *cfi = &state->cfi;
2491 int i;
2492
2493 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2494 return true;
2495
2496 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2497 return true;
2498
2499 if (cfi->stack_size != initial_func_cfi.cfa.offset)
2500 return true;
2501
2502 for (i = 0; i < CFI_NUM_REGS; i++) {
2503 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2504 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2505 return true;
2506 }
2507
2508 return false;
2509 }
2510
check_reg_frame_pos(const struct cfi_reg * reg,int expected_offset)2511 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2512 int expected_offset)
2513 {
2514 return reg->base == CFI_CFA &&
2515 reg->offset == expected_offset;
2516 }
2517
has_valid_stack_frame(struct insn_state * state)2518 static bool has_valid_stack_frame(struct insn_state *state)
2519 {
2520 struct cfi_state *cfi = &state->cfi;
2521
2522 if (cfi->cfa.base == CFI_BP &&
2523 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2524 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2525 return true;
2526
2527 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2528 return true;
2529
2530 return false;
2531 }
2532
update_cfi_state_regs(struct instruction * insn,struct cfi_state * cfi,struct stack_op * op)2533 static int update_cfi_state_regs(struct instruction *insn,
2534 struct cfi_state *cfi,
2535 struct stack_op *op)
2536 {
2537 struct cfi_reg *cfa = &cfi->cfa;
2538
2539 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2540 return 0;
2541
2542 /* push */
2543 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2544 cfa->offset += 8;
2545
2546 /* pop */
2547 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2548 cfa->offset -= 8;
2549
2550 /* add immediate to sp */
2551 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2552 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2553 cfa->offset -= op->src.offset;
2554
2555 return 0;
2556 }
2557
save_reg(struct cfi_state * cfi,unsigned char reg,int base,int offset)2558 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2559 {
2560 if (arch_callee_saved_reg(reg) &&
2561 cfi->regs[reg].base == CFI_UNDEFINED) {
2562 cfi->regs[reg].base = base;
2563 cfi->regs[reg].offset = offset;
2564 }
2565 }
2566
restore_reg(struct cfi_state * cfi,unsigned char reg)2567 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2568 {
2569 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2570 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2571 }
2572
2573 /*
2574 * A note about DRAP stack alignment:
2575 *
2576 * GCC has the concept of a DRAP register, which is used to help keep track of
2577 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2578 * register. The typical DRAP pattern is:
2579 *
2580 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2581 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2582 * 41 ff 72 f8 pushq -0x8(%r10)
2583 * 55 push %rbp
2584 * 48 89 e5 mov %rsp,%rbp
2585 * (more pushes)
2586 * 41 52 push %r10
2587 * ...
2588 * 41 5a pop %r10
2589 * (more pops)
2590 * 5d pop %rbp
2591 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2592 * c3 retq
2593 *
2594 * There are some variations in the epilogues, like:
2595 *
2596 * 5b pop %rbx
2597 * 41 5a pop %r10
2598 * 41 5c pop %r12
2599 * 41 5d pop %r13
2600 * 41 5e pop %r14
2601 * c9 leaveq
2602 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2603 * c3 retq
2604 *
2605 * and:
2606 *
2607 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2608 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2609 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2610 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2611 * c9 leaveq
2612 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2613 * c3 retq
2614 *
2615 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2616 * restored beforehand:
2617 *
2618 * 41 55 push %r13
2619 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2620 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2621 * ...
2622 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2623 * 41 5d pop %r13
2624 * c3 retq
2625 */
update_cfi_state(struct instruction * insn,struct instruction * next_insn,struct cfi_state * cfi,struct stack_op * op)2626 static int update_cfi_state(struct instruction *insn,
2627 struct instruction *next_insn,
2628 struct cfi_state *cfi, struct stack_op *op)
2629 {
2630 struct cfi_reg *cfa = &cfi->cfa;
2631 struct cfi_reg *regs = cfi->regs;
2632
2633 /* stack operations don't make sense with an undefined CFA */
2634 if (cfa->base == CFI_UNDEFINED) {
2635 if (insn->func) {
2636 WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2637 return -1;
2638 }
2639 return 0;
2640 }
2641
2642 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2643 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2644 return update_cfi_state_regs(insn, cfi, op);
2645
2646 switch (op->dest.type) {
2647
2648 case OP_DEST_REG:
2649 switch (op->src.type) {
2650
2651 case OP_SRC_REG:
2652 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2653 cfa->base == CFI_SP &&
2654 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) {
2655
2656 /* mov %rsp, %rbp */
2657 cfa->base = op->dest.reg;
2658 cfi->bp_scratch = false;
2659 }
2660
2661 else if (op->src.reg == CFI_SP &&
2662 op->dest.reg == CFI_BP && cfi->drap) {
2663
2664 /* drap: mov %rsp, %rbp */
2665 regs[CFI_BP].base = CFI_BP;
2666 regs[CFI_BP].offset = -cfi->stack_size;
2667 cfi->bp_scratch = false;
2668 }
2669
2670 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2671
2672 /*
2673 * mov %rsp, %reg
2674 *
2675 * This is needed for the rare case where GCC
2676 * does:
2677 *
2678 * mov %rsp, %rax
2679 * ...
2680 * mov %rax, %rsp
2681 */
2682 cfi->vals[op->dest.reg].base = CFI_CFA;
2683 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2684 }
2685
2686 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2687 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2688
2689 /*
2690 * mov %rbp, %rsp
2691 *
2692 * Restore the original stack pointer (Clang).
2693 */
2694 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2695 }
2696
2697 else if (op->dest.reg == cfa->base) {
2698
2699 /* mov %reg, %rsp */
2700 if (cfa->base == CFI_SP &&
2701 cfi->vals[op->src.reg].base == CFI_CFA) {
2702
2703 /*
2704 * This is needed for the rare case
2705 * where GCC does something dumb like:
2706 *
2707 * lea 0x8(%rsp), %rcx
2708 * ...
2709 * mov %rcx, %rsp
2710 */
2711 cfa->offset = -cfi->vals[op->src.reg].offset;
2712 cfi->stack_size = cfa->offset;
2713
2714 } else if (cfa->base == CFI_SP &&
2715 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2716 cfi->vals[op->src.reg].offset == cfa->offset) {
2717
2718 /*
2719 * Stack swizzle:
2720 *
2721 * 1: mov %rsp, (%[tos])
2722 * 2: mov %[tos], %rsp
2723 * ...
2724 * 3: pop %rsp
2725 *
2726 * Where:
2727 *
2728 * 1 - places a pointer to the previous
2729 * stack at the Top-of-Stack of the
2730 * new stack.
2731 *
2732 * 2 - switches to the new stack.
2733 *
2734 * 3 - pops the Top-of-Stack to restore
2735 * the original stack.
2736 *
2737 * Note: we set base to SP_INDIRECT
2738 * here and preserve offset. Therefore
2739 * when the unwinder reaches ToS it
2740 * will dereference SP and then add the
2741 * offset to find the next frame, IOW:
2742 * (%rsp) + offset.
2743 */
2744 cfa->base = CFI_SP_INDIRECT;
2745
2746 } else {
2747 cfa->base = CFI_UNDEFINED;
2748 cfa->offset = 0;
2749 }
2750 }
2751
2752 else if (op->dest.reg == CFI_SP &&
2753 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2754 cfi->vals[op->src.reg].offset == cfa->offset) {
2755
2756 /*
2757 * The same stack swizzle case 2) as above. But
2758 * because we can't change cfa->base, case 3)
2759 * will become a regular POP. Pretend we're a
2760 * PUSH so things don't go unbalanced.
2761 */
2762 cfi->stack_size += 8;
2763 }
2764
2765
2766 break;
2767
2768 case OP_SRC_ADD:
2769 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2770
2771 /* add imm, %rsp */
2772 cfi->stack_size -= op->src.offset;
2773 if (cfa->base == CFI_SP)
2774 cfa->offset -= op->src.offset;
2775 break;
2776 }
2777
2778 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2779
2780 /* lea disp(%rbp), %rsp */
2781 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2782 break;
2783 }
2784
2785 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2786
2787 /* drap: lea disp(%rsp), %drap */
2788 cfi->drap_reg = op->dest.reg;
2789
2790 /*
2791 * lea disp(%rsp), %reg
2792 *
2793 * This is needed for the rare case where GCC
2794 * does something dumb like:
2795 *
2796 * lea 0x8(%rsp), %rcx
2797 * ...
2798 * mov %rcx, %rsp
2799 */
2800 cfi->vals[op->dest.reg].base = CFI_CFA;
2801 cfi->vals[op->dest.reg].offset = \
2802 -cfi->stack_size + op->src.offset;
2803
2804 break;
2805 }
2806
2807 if (cfi->drap && op->dest.reg == CFI_SP &&
2808 op->src.reg == cfi->drap_reg) {
2809
2810 /* drap: lea disp(%drap), %rsp */
2811 cfa->base = CFI_SP;
2812 cfa->offset = cfi->stack_size = -op->src.offset;
2813 cfi->drap_reg = CFI_UNDEFINED;
2814 cfi->drap = false;
2815 break;
2816 }
2817
2818 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
2819 WARN_FUNC("unsupported stack register modification",
2820 insn->sec, insn->offset);
2821 return -1;
2822 }
2823
2824 break;
2825
2826 case OP_SRC_AND:
2827 if (op->dest.reg != CFI_SP ||
2828 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2829 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
2830 WARN_FUNC("unsupported stack pointer realignment",
2831 insn->sec, insn->offset);
2832 return -1;
2833 }
2834
2835 if (cfi->drap_reg != CFI_UNDEFINED) {
2836 /* drap: and imm, %rsp */
2837 cfa->base = cfi->drap_reg;
2838 cfa->offset = cfi->stack_size = 0;
2839 cfi->drap = true;
2840 }
2841
2842 /*
2843 * Older versions of GCC (4.8ish) realign the stack
2844 * without DRAP, with a frame pointer.
2845 */
2846
2847 break;
2848
2849 case OP_SRC_POP:
2850 case OP_SRC_POPF:
2851 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2852
2853 /* pop %rsp; # restore from a stack swizzle */
2854 cfa->base = CFI_SP;
2855 break;
2856 }
2857
2858 if (!cfi->drap && op->dest.reg == cfa->base) {
2859
2860 /* pop %rbp */
2861 cfa->base = CFI_SP;
2862 }
2863
2864 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
2865 op->dest.reg == cfi->drap_reg &&
2866 cfi->drap_offset == -cfi->stack_size) {
2867
2868 /* drap: pop %drap */
2869 cfa->base = cfi->drap_reg;
2870 cfa->offset = 0;
2871 cfi->drap_offset = -1;
2872
2873 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
2874
2875 /* pop %reg */
2876 restore_reg(cfi, op->dest.reg);
2877 }
2878
2879 cfi->stack_size -= 8;
2880 if (cfa->base == CFI_SP)
2881 cfa->offset -= 8;
2882
2883 break;
2884
2885 case OP_SRC_REG_INDIRECT:
2886 if (!cfi->drap && op->dest.reg == cfa->base &&
2887 op->dest.reg == CFI_BP) {
2888
2889 /* mov disp(%rsp), %rbp */
2890 cfa->base = CFI_SP;
2891 cfa->offset = cfi->stack_size;
2892 }
2893
2894 if (cfi->drap && op->src.reg == CFI_BP &&
2895 op->src.offset == cfi->drap_offset) {
2896
2897 /* drap: mov disp(%rbp), %drap */
2898 cfa->base = cfi->drap_reg;
2899 cfa->offset = 0;
2900 cfi->drap_offset = -1;
2901 }
2902
2903 if (cfi->drap && op->src.reg == CFI_BP &&
2904 op->src.offset == regs[op->dest.reg].offset) {
2905
2906 /* drap: mov disp(%rbp), %reg */
2907 restore_reg(cfi, op->dest.reg);
2908
2909 } else if (op->src.reg == cfa->base &&
2910 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
2911
2912 /* mov disp(%rbp), %reg */
2913 /* mov disp(%rsp), %reg */
2914 restore_reg(cfi, op->dest.reg);
2915
2916 } else if (op->src.reg == CFI_SP &&
2917 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
2918
2919 /* mov disp(%rsp), %reg */
2920 restore_reg(cfi, op->dest.reg);
2921 }
2922
2923 break;
2924
2925 default:
2926 WARN_FUNC("unknown stack-related instruction",
2927 insn->sec, insn->offset);
2928 return -1;
2929 }
2930
2931 break;
2932
2933 case OP_DEST_PUSH:
2934 case OP_DEST_PUSHF:
2935 cfi->stack_size += 8;
2936 if (cfa->base == CFI_SP)
2937 cfa->offset += 8;
2938
2939 if (op->src.type != OP_SRC_REG)
2940 break;
2941
2942 if (cfi->drap) {
2943 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2944
2945 /* drap: push %drap */
2946 cfa->base = CFI_BP_INDIRECT;
2947 cfa->offset = -cfi->stack_size;
2948
2949 /* save drap so we know when to restore it */
2950 cfi->drap_offset = -cfi->stack_size;
2951
2952 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
2953
2954 /* drap: push %rbp */
2955 cfi->stack_size = 0;
2956
2957 } else {
2958
2959 /* drap: push %reg */
2960 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
2961 }
2962
2963 } else {
2964
2965 /* push %reg */
2966 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
2967 }
2968
2969 /* detect when asm code uses rbp as a scratch register */
2970 if (opts.stackval && insn->func && op->src.reg == CFI_BP &&
2971 cfa->base != CFI_BP)
2972 cfi->bp_scratch = true;
2973 break;
2974
2975 case OP_DEST_REG_INDIRECT:
2976
2977 if (cfi->drap) {
2978 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2979
2980 /* drap: mov %drap, disp(%rbp) */
2981 cfa->base = CFI_BP_INDIRECT;
2982 cfa->offset = op->dest.offset;
2983
2984 /* save drap offset so we know when to restore it */
2985 cfi->drap_offset = op->dest.offset;
2986 } else {
2987
2988 /* drap: mov reg, disp(%rbp) */
2989 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
2990 }
2991
2992 } else if (op->dest.reg == cfa->base) {
2993
2994 /* mov reg, disp(%rbp) */
2995 /* mov reg, disp(%rsp) */
2996 save_reg(cfi, op->src.reg, CFI_CFA,
2997 op->dest.offset - cfi->cfa.offset);
2998
2999 } else if (op->dest.reg == CFI_SP) {
3000
3001 /* mov reg, disp(%rsp) */
3002 save_reg(cfi, op->src.reg, CFI_CFA,
3003 op->dest.offset - cfi->stack_size);
3004
3005 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3006
3007 /* mov %rsp, (%reg); # setup a stack swizzle. */
3008 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3009 cfi->vals[op->dest.reg].offset = cfa->offset;
3010 }
3011
3012 break;
3013
3014 case OP_DEST_MEM:
3015 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3016 WARN_FUNC("unknown stack-related memory operation",
3017 insn->sec, insn->offset);
3018 return -1;
3019 }
3020
3021 /* pop mem */
3022 cfi->stack_size -= 8;
3023 if (cfa->base == CFI_SP)
3024 cfa->offset -= 8;
3025
3026 break;
3027
3028 default:
3029 WARN_FUNC("unknown stack-related instruction",
3030 insn->sec, insn->offset);
3031 return -1;
3032 }
3033
3034 return 0;
3035 }
3036
3037 /*
3038 * The stack layouts of alternatives instructions can sometimes diverge when
3039 * they have stack modifications. That's fine as long as the potential stack
3040 * layouts don't conflict at any given potential instruction boundary.
3041 *
3042 * Flatten the CFIs of the different alternative code streams (both original
3043 * and replacement) into a single shared CFI array which can be used to detect
3044 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3045 */
propagate_alt_cfi(struct objtool_file * file,struct instruction * insn)3046 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3047 {
3048 struct cfi_state **alt_cfi;
3049 int group_off;
3050
3051 if (!insn->alt_group)
3052 return 0;
3053
3054 if (!insn->cfi) {
3055 WARN("CFI missing");
3056 return -1;
3057 }
3058
3059 alt_cfi = insn->alt_group->cfi;
3060 group_off = insn->offset - insn->alt_group->first_insn->offset;
3061
3062 if (!alt_cfi[group_off]) {
3063 alt_cfi[group_off] = insn->cfi;
3064 } else {
3065 if (cficmp(alt_cfi[group_off], insn->cfi)) {
3066 WARN_FUNC("stack layout conflict in alternatives",
3067 insn->sec, insn->offset);
3068 return -1;
3069 }
3070 }
3071
3072 return 0;
3073 }
3074
handle_insn_ops(struct instruction * insn,struct instruction * next_insn,struct insn_state * state)3075 static int handle_insn_ops(struct instruction *insn,
3076 struct instruction *next_insn,
3077 struct insn_state *state)
3078 {
3079 struct stack_op *op;
3080
3081 list_for_each_entry(op, &insn->stack_ops, list) {
3082
3083 if (update_cfi_state(insn, next_insn, &state->cfi, op))
3084 return 1;
3085
3086 if (!insn->alt_group)
3087 continue;
3088
3089 if (op->dest.type == OP_DEST_PUSHF) {
3090 if (!state->uaccess_stack) {
3091 state->uaccess_stack = 1;
3092 } else if (state->uaccess_stack >> 31) {
3093 WARN_FUNC("PUSHF stack exhausted",
3094 insn->sec, insn->offset);
3095 return 1;
3096 }
3097 state->uaccess_stack <<= 1;
3098 state->uaccess_stack |= state->uaccess;
3099 }
3100
3101 if (op->src.type == OP_SRC_POPF) {
3102 if (state->uaccess_stack) {
3103 state->uaccess = state->uaccess_stack & 1;
3104 state->uaccess_stack >>= 1;
3105 if (state->uaccess_stack == 1)
3106 state->uaccess_stack = 0;
3107 }
3108 }
3109 }
3110
3111 return 0;
3112 }
3113
insn_cfi_match(struct instruction * insn,struct cfi_state * cfi2)3114 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3115 {
3116 struct cfi_state *cfi1 = insn->cfi;
3117 int i;
3118
3119 if (!cfi1) {
3120 WARN("CFI missing");
3121 return false;
3122 }
3123
3124 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3125
3126 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3127 insn->sec, insn->offset,
3128 cfi1->cfa.base, cfi1->cfa.offset,
3129 cfi2->cfa.base, cfi2->cfa.offset);
3130
3131 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3132 for (i = 0; i < CFI_NUM_REGS; i++) {
3133 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3134 sizeof(struct cfi_reg)))
3135 continue;
3136
3137 WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3138 insn->sec, insn->offset,
3139 i, cfi1->regs[i].base, cfi1->regs[i].offset,
3140 i, cfi2->regs[i].base, cfi2->regs[i].offset);
3141 break;
3142 }
3143
3144 } else if (cfi1->type != cfi2->type) {
3145
3146 WARN_FUNC("stack state mismatch: type1=%d type2=%d",
3147 insn->sec, insn->offset, cfi1->type, cfi2->type);
3148
3149 } else if (cfi1->drap != cfi2->drap ||
3150 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3151 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3152
3153 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3154 insn->sec, insn->offset,
3155 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3156 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3157
3158 } else
3159 return true;
3160
3161 return false;
3162 }
3163
func_uaccess_safe(struct symbol * func)3164 static inline bool func_uaccess_safe(struct symbol *func)
3165 {
3166 if (func)
3167 return func->uaccess_safe;
3168
3169 return false;
3170 }
3171
call_dest_name(struct instruction * insn)3172 static inline const char *call_dest_name(struct instruction *insn)
3173 {
3174 static char pvname[19];
3175 struct reloc *rel;
3176 int idx;
3177
3178 if (insn->call_dest)
3179 return insn->call_dest->name;
3180
3181 rel = insn_reloc(NULL, insn);
3182 if (rel && !strcmp(rel->sym->name, "pv_ops")) {
3183 idx = (rel->addend / sizeof(void *));
3184 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3185 return pvname;
3186 }
3187
3188 return "{dynamic}";
3189 }
3190
pv_call_dest(struct objtool_file * file,struct instruction * insn)3191 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3192 {
3193 struct symbol *target;
3194 struct reloc *rel;
3195 int idx;
3196
3197 rel = insn_reloc(file, insn);
3198 if (!rel || strcmp(rel->sym->name, "pv_ops"))
3199 return false;
3200
3201 idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *));
3202
3203 if (file->pv_ops[idx].clean)
3204 return true;
3205
3206 file->pv_ops[idx].clean = true;
3207
3208 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3209 if (!target->sec->noinstr) {
3210 WARN("pv_ops[%d]: %s", idx, target->name);
3211 file->pv_ops[idx].clean = false;
3212 }
3213 }
3214
3215 return file->pv_ops[idx].clean;
3216 }
3217
noinstr_call_dest(struct objtool_file * file,struct instruction * insn,struct symbol * func)3218 static inline bool noinstr_call_dest(struct objtool_file *file,
3219 struct instruction *insn,
3220 struct symbol *func)
3221 {
3222 /*
3223 * We can't deal with indirect function calls at present;
3224 * assume they're instrumented.
3225 */
3226 if (!func) {
3227 if (file->pv_ops)
3228 return pv_call_dest(file, insn);
3229
3230 return false;
3231 }
3232
3233 /*
3234 * If the symbol is from a noinstr section; we good.
3235 */
3236 if (func->sec->noinstr)
3237 return true;
3238
3239 /*
3240 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3241 * something 'BAD' happened. At the risk of taking the machine down,
3242 * let them proceed to get the message out.
3243 */
3244 if (!strncmp(func->name, "__ubsan_handle_", 15))
3245 return true;
3246
3247 return false;
3248 }
3249
validate_call(struct objtool_file * file,struct instruction * insn,struct insn_state * state)3250 static int validate_call(struct objtool_file *file,
3251 struct instruction *insn,
3252 struct insn_state *state)
3253 {
3254 if (state->noinstr && state->instr <= 0 &&
3255 !noinstr_call_dest(file, insn, insn->call_dest)) {
3256 WARN_FUNC("call to %s() leaves .noinstr.text section",
3257 insn->sec, insn->offset, call_dest_name(insn));
3258 return 1;
3259 }
3260
3261 if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
3262 WARN_FUNC("call to %s() with UACCESS enabled",
3263 insn->sec, insn->offset, call_dest_name(insn));
3264 return 1;
3265 }
3266
3267 if (state->df) {
3268 WARN_FUNC("call to %s() with DF set",
3269 insn->sec, insn->offset, call_dest_name(insn));
3270 return 1;
3271 }
3272
3273 return 0;
3274 }
3275
validate_sibling_call(struct objtool_file * file,struct instruction * insn,struct insn_state * state)3276 static int validate_sibling_call(struct objtool_file *file,
3277 struct instruction *insn,
3278 struct insn_state *state)
3279 {
3280 if (has_modified_stack_frame(insn, state)) {
3281 WARN_FUNC("sibling call from callable instruction with modified stack frame",
3282 insn->sec, insn->offset);
3283 return 1;
3284 }
3285
3286 return validate_call(file, insn, state);
3287 }
3288
validate_return(struct symbol * func,struct instruction * insn,struct insn_state * state)3289 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3290 {
3291 if (state->noinstr && state->instr > 0) {
3292 WARN_FUNC("return with instrumentation enabled",
3293 insn->sec, insn->offset);
3294 return 1;
3295 }
3296
3297 if (state->uaccess && !func_uaccess_safe(func)) {
3298 WARN_FUNC("return with UACCESS enabled",
3299 insn->sec, insn->offset);
3300 return 1;
3301 }
3302
3303 if (!state->uaccess && func_uaccess_safe(func)) {
3304 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
3305 insn->sec, insn->offset);
3306 return 1;
3307 }
3308
3309 if (state->df) {
3310 WARN_FUNC("return with DF set",
3311 insn->sec, insn->offset);
3312 return 1;
3313 }
3314
3315 if (func && has_modified_stack_frame(insn, state)) {
3316 WARN_FUNC("return with modified stack frame",
3317 insn->sec, insn->offset);
3318 return 1;
3319 }
3320
3321 if (state->cfi.bp_scratch) {
3322 WARN_FUNC("BP used as a scratch register",
3323 insn->sec, insn->offset);
3324 return 1;
3325 }
3326
3327 return 0;
3328 }
3329
next_insn_to_validate(struct objtool_file * file,struct instruction * insn)3330 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3331 struct instruction *insn)
3332 {
3333 struct alt_group *alt_group = insn->alt_group;
3334
3335 /*
3336 * Simulate the fact that alternatives are patched in-place. When the
3337 * end of a replacement alt_group is reached, redirect objtool flow to
3338 * the end of the original alt_group.
3339 */
3340 if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
3341 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3342
3343 return next_insn_same_sec(file, insn);
3344 }
3345
3346 /*
3347 * Follow the branch starting at the given instruction, and recursively follow
3348 * any other branches (jumps). Meanwhile, track the frame pointer state at
3349 * each instruction and validate all the rules described in
3350 * tools/objtool/Documentation/objtool.txt.
3351 */
validate_branch(struct objtool_file * file,struct symbol * func,struct instruction * insn,struct insn_state state)3352 static int validate_branch(struct objtool_file *file, struct symbol *func,
3353 struct instruction *insn, struct insn_state state)
3354 {
3355 struct alternative *alt;
3356 struct instruction *next_insn, *prev_insn = NULL;
3357 struct section *sec;
3358 u8 visited;
3359 int ret;
3360
3361 sec = insn->sec;
3362
3363 while (1) {
3364 next_insn = next_insn_to_validate(file, insn);
3365
3366 if (func && insn->func && func != insn->func->pfunc) {
3367 /* Ignore KCFI type preambles, which always fall through */
3368 if (!strncmp(func->name, "__cfi_", 6))
3369 return 0;
3370
3371 WARN("%s() falls through to next function %s()",
3372 func->name, insn->func->name);
3373 return 1;
3374 }
3375
3376 if (func && insn->ignore) {
3377 WARN_FUNC("BUG: why am I validating an ignored function?",
3378 sec, insn->offset);
3379 return 1;
3380 }
3381
3382 visited = VISITED_BRANCH << state.uaccess;
3383 if (insn->visited & VISITED_BRANCH_MASK) {
3384 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3385 return 1;
3386
3387 if (insn->visited & visited)
3388 return 0;
3389 } else {
3390 nr_insns_visited++;
3391 }
3392
3393 if (state.noinstr)
3394 state.instr += insn->instr;
3395
3396 if (insn->hint) {
3397 if (insn->restore) {
3398 struct instruction *save_insn, *i;
3399
3400 i = insn;
3401 save_insn = NULL;
3402
3403 sym_for_each_insn_continue_reverse(file, func, i) {
3404 if (i->save) {
3405 save_insn = i;
3406 break;
3407 }
3408 }
3409
3410 if (!save_insn) {
3411 WARN_FUNC("no corresponding CFI save for CFI restore",
3412 sec, insn->offset);
3413 return 1;
3414 }
3415
3416 if (!save_insn->visited) {
3417 WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo",
3418 sec, insn->offset);
3419 return 1;
3420 }
3421
3422 insn->cfi = save_insn->cfi;
3423 nr_cfi_reused++;
3424 }
3425
3426 state.cfi = *insn->cfi;
3427 } else {
3428 /* XXX track if we actually changed state.cfi */
3429
3430 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3431 insn->cfi = prev_insn->cfi;
3432 nr_cfi_reused++;
3433 } else {
3434 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3435 }
3436 }
3437
3438 insn->visited |= visited;
3439
3440 if (propagate_alt_cfi(file, insn))
3441 return 1;
3442
3443 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
3444 bool skip_orig = false;
3445
3446 list_for_each_entry(alt, &insn->alts, list) {
3447 if (alt->skip_orig)
3448 skip_orig = true;
3449
3450 ret = validate_branch(file, func, alt->insn, state);
3451 if (ret) {
3452 if (opts.backtrace)
3453 BT_FUNC("(alt)", insn);
3454 return ret;
3455 }
3456 }
3457
3458 if (skip_orig)
3459 return 0;
3460 }
3461
3462 if (handle_insn_ops(insn, next_insn, &state))
3463 return 1;
3464
3465 switch (insn->type) {
3466
3467 case INSN_RETURN:
3468 return validate_return(func, insn, &state);
3469
3470 case INSN_CALL:
3471 case INSN_CALL_DYNAMIC:
3472 ret = validate_call(file, insn, &state);
3473 if (ret)
3474 return ret;
3475
3476 if (opts.stackval && func && !is_special_call(insn) &&
3477 !has_valid_stack_frame(&state)) {
3478 WARN_FUNC("call without frame pointer save/setup",
3479 sec, insn->offset);
3480 return 1;
3481 }
3482
3483 if (insn->dead_end)
3484 return 0;
3485
3486 break;
3487
3488 case INSN_JUMP_CONDITIONAL:
3489 case INSN_JUMP_UNCONDITIONAL:
3490 if (is_sibling_call(insn)) {
3491 ret = validate_sibling_call(file, insn, &state);
3492 if (ret)
3493 return ret;
3494
3495 } else if (insn->jump_dest) {
3496 ret = validate_branch(file, func,
3497 insn->jump_dest, state);
3498 if (ret) {
3499 if (opts.backtrace)
3500 BT_FUNC("(branch)", insn);
3501 return ret;
3502 }
3503 }
3504
3505 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3506 return 0;
3507
3508 break;
3509
3510 case INSN_JUMP_DYNAMIC:
3511 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3512 if (is_sibling_call(insn)) {
3513 ret = validate_sibling_call(file, insn, &state);
3514 if (ret)
3515 return ret;
3516 }
3517
3518 if (insn->type == INSN_JUMP_DYNAMIC)
3519 return 0;
3520
3521 break;
3522
3523 case INSN_CONTEXT_SWITCH:
3524 if (func && (!next_insn || !next_insn->hint)) {
3525 WARN_FUNC("unsupported instruction in callable function",
3526 sec, insn->offset);
3527 return 1;
3528 }
3529 return 0;
3530
3531 case INSN_STAC:
3532 if (state.uaccess) {
3533 WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
3534 return 1;
3535 }
3536
3537 state.uaccess = true;
3538 break;
3539
3540 case INSN_CLAC:
3541 if (!state.uaccess && func) {
3542 WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
3543 return 1;
3544 }
3545
3546 if (func_uaccess_safe(func) && !state.uaccess_stack) {
3547 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
3548 return 1;
3549 }
3550
3551 state.uaccess = false;
3552 break;
3553
3554 case INSN_STD:
3555 if (state.df) {
3556 WARN_FUNC("recursive STD", sec, insn->offset);
3557 return 1;
3558 }
3559
3560 state.df = true;
3561 break;
3562
3563 case INSN_CLD:
3564 if (!state.df && func) {
3565 WARN_FUNC("redundant CLD", sec, insn->offset);
3566 return 1;
3567 }
3568
3569 state.df = false;
3570 break;
3571
3572 default:
3573 break;
3574 }
3575
3576 if (insn->dead_end)
3577 return 0;
3578
3579 if (!next_insn) {
3580 if (state.cfi.cfa.base == CFI_UNDEFINED)
3581 return 0;
3582 WARN("%s: unexpected end of section", sec->name);
3583 return 1;
3584 }
3585
3586 prev_insn = insn;
3587 insn = next_insn;
3588 }
3589
3590 return 0;
3591 }
3592
validate_unwind_hints(struct objtool_file * file,struct section * sec)3593 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3594 {
3595 struct instruction *insn;
3596 struct insn_state state;
3597 int ret, warnings = 0;
3598
3599 if (!file->hints)
3600 return 0;
3601
3602 init_insn_state(file, &state, sec);
3603
3604 if (sec) {
3605 insn = find_insn(file, sec, 0);
3606 if (!insn)
3607 return 0;
3608 } else {
3609 insn = list_first_entry(&file->insn_list, typeof(*insn), list);
3610 }
3611
3612 while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
3613 if (insn->hint && !insn->visited && !insn->ignore) {
3614 ret = validate_branch(file, insn->func, insn, state);
3615 if (ret && opts.backtrace)
3616 BT_FUNC("<=== (hint)", insn);
3617 warnings += ret;
3618 }
3619
3620 insn = list_next_entry(insn, list);
3621 }
3622
3623 return warnings;
3624 }
3625
3626 /*
3627 * Validate rethunk entry constraint: must untrain RET before the first RET.
3628 *
3629 * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes
3630 * before an actual RET instruction.
3631 */
validate_entry(struct objtool_file * file,struct instruction * insn)3632 static int validate_entry(struct objtool_file *file, struct instruction *insn)
3633 {
3634 struct instruction *next, *dest;
3635 int ret, warnings = 0;
3636
3637 for (;;) {
3638 next = next_insn_to_validate(file, insn);
3639
3640 if (insn->visited & VISITED_ENTRY)
3641 return 0;
3642
3643 insn->visited |= VISITED_ENTRY;
3644
3645 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
3646 struct alternative *alt;
3647 bool skip_orig = false;
3648
3649 list_for_each_entry(alt, &insn->alts, list) {
3650 if (alt->skip_orig)
3651 skip_orig = true;
3652
3653 ret = validate_entry(file, alt->insn);
3654 if (ret) {
3655 if (opts.backtrace)
3656 BT_FUNC("(alt)", insn);
3657 return ret;
3658 }
3659 }
3660
3661 if (skip_orig)
3662 return 0;
3663 }
3664
3665 switch (insn->type) {
3666
3667 case INSN_CALL_DYNAMIC:
3668 case INSN_JUMP_DYNAMIC:
3669 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3670 WARN_FUNC("early indirect call", insn->sec, insn->offset);
3671 return 1;
3672
3673 case INSN_JUMP_UNCONDITIONAL:
3674 case INSN_JUMP_CONDITIONAL:
3675 if (!is_sibling_call(insn)) {
3676 if (!insn->jump_dest) {
3677 WARN_FUNC("unresolved jump target after linking?!?",
3678 insn->sec, insn->offset);
3679 return -1;
3680 }
3681 ret = validate_entry(file, insn->jump_dest);
3682 if (ret) {
3683 if (opts.backtrace) {
3684 BT_FUNC("(branch%s)", insn,
3685 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3686 }
3687 return ret;
3688 }
3689
3690 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3691 return 0;
3692
3693 break;
3694 }
3695
3696 /* fallthrough */
3697 case INSN_CALL:
3698 dest = find_insn(file, insn->call_dest->sec,
3699 insn->call_dest->offset);
3700 if (!dest) {
3701 WARN("Unresolved function after linking!?: %s",
3702 insn->call_dest->name);
3703 return -1;
3704 }
3705
3706 ret = validate_entry(file, dest);
3707 if (ret) {
3708 if (opts.backtrace)
3709 BT_FUNC("(call)", insn);
3710 return ret;
3711 }
3712 /*
3713 * If a call returns without error, it must have seen UNTRAIN_RET.
3714 * Therefore any non-error return is a success.
3715 */
3716 return 0;
3717
3718 case INSN_RETURN:
3719 WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset);
3720 return 1;
3721
3722 case INSN_NOP:
3723 if (insn->retpoline_safe)
3724 return 0;
3725 break;
3726
3727 default:
3728 break;
3729 }
3730
3731 if (!next) {
3732 WARN_FUNC("teh end!", insn->sec, insn->offset);
3733 return -1;
3734 }
3735 insn = next;
3736 }
3737
3738 return warnings;
3739 }
3740
3741 /*
3742 * Validate that all branches starting at 'insn->entry' encounter UNRET_END
3743 * before RET.
3744 */
validate_unret(struct objtool_file * file)3745 static int validate_unret(struct objtool_file *file)
3746 {
3747 struct instruction *insn;
3748 int ret, warnings = 0;
3749
3750 for_each_insn(file, insn) {
3751 if (!insn->entry)
3752 continue;
3753
3754 ret = validate_entry(file, insn);
3755 if (ret < 0) {
3756 WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset);
3757 return ret;
3758 }
3759 warnings += ret;
3760 }
3761
3762 return warnings;
3763 }
3764
validate_retpoline(struct objtool_file * file)3765 static int validate_retpoline(struct objtool_file *file)
3766 {
3767 struct instruction *insn;
3768 int warnings = 0;
3769
3770 for_each_insn(file, insn) {
3771 if (insn->type != INSN_JUMP_DYNAMIC &&
3772 insn->type != INSN_CALL_DYNAMIC &&
3773 insn->type != INSN_RETURN)
3774 continue;
3775
3776 if (insn->retpoline_safe)
3777 continue;
3778
3779 /*
3780 * .init.text code is ran before userspace and thus doesn't
3781 * strictly need retpolines, except for modules which are
3782 * loaded late, they very much do need retpoline in their
3783 * .init.text
3784 */
3785 if (!strcmp(insn->sec->name, ".init.text") && !opts.module)
3786 continue;
3787
3788 if (insn->type == INSN_RETURN) {
3789 if (opts.rethunk) {
3790 WARN_FUNC("'naked' return found in RETHUNK build",
3791 insn->sec, insn->offset);
3792 } else
3793 continue;
3794 } else {
3795 WARN_FUNC("indirect %s found in RETPOLINE build",
3796 insn->sec, insn->offset,
3797 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3798 }
3799
3800 warnings++;
3801 }
3802
3803 return warnings;
3804 }
3805
is_kasan_insn(struct instruction * insn)3806 static bool is_kasan_insn(struct instruction *insn)
3807 {
3808 return (insn->type == INSN_CALL &&
3809 !strcmp(insn->call_dest->name, "__asan_handle_no_return"));
3810 }
3811
is_ubsan_insn(struct instruction * insn)3812 static bool is_ubsan_insn(struct instruction *insn)
3813 {
3814 return (insn->type == INSN_CALL &&
3815 !strcmp(insn->call_dest->name,
3816 "__ubsan_handle_builtin_unreachable"));
3817 }
3818
ignore_unreachable_insn(struct objtool_file * file,struct instruction * insn)3819 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
3820 {
3821 int i;
3822 struct instruction *prev_insn;
3823
3824 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
3825 return true;
3826
3827 /*
3828 * Ignore alternative replacement instructions. This can happen
3829 * when a whitelisted function uses one of the ALTERNATIVE macros.
3830 */
3831 if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
3832 !strcmp(insn->sec->name, ".altinstr_aux"))
3833 return true;
3834
3835 /*
3836 * Whole archive runs might encounter dead code from weak symbols.
3837 * This is where the linker will have dropped the weak symbol in
3838 * favour of a regular symbol, but leaves the code in place.
3839 *
3840 * In this case we'll find a piece of code (whole function) that is not
3841 * covered by a !section symbol. Ignore them.
3842 */
3843 if (opts.link && !insn->func) {
3844 int size = find_symbol_hole_containing(insn->sec, insn->offset);
3845 unsigned long end = insn->offset + size;
3846
3847 if (!size) /* not a hole */
3848 return false;
3849
3850 if (size < 0) /* hole until the end */
3851 return true;
3852
3853 sec_for_each_insn_continue(file, insn) {
3854 /*
3855 * If we reach a visited instruction at or before the
3856 * end of the hole, ignore the unreachable.
3857 */
3858 if (insn->visited)
3859 return true;
3860
3861 if (insn->offset >= end)
3862 break;
3863
3864 /*
3865 * If this hole jumps to a .cold function, mark it ignore too.
3866 */
3867 if (insn->jump_dest && insn->jump_dest->func &&
3868 strstr(insn->jump_dest->func->name, ".cold")) {
3869 struct instruction *dest = insn->jump_dest;
3870 func_for_each_insn(file, dest->func, dest)
3871 dest->ignore = true;
3872 }
3873 }
3874
3875 return false;
3876 }
3877
3878 if (!insn->func)
3879 return false;
3880
3881 if (insn->func->static_call_tramp)
3882 return true;
3883
3884 /*
3885 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
3886 * __builtin_unreachable(). The BUG() macro has an unreachable() after
3887 * the UD2, which causes GCC's undefined trap logic to emit another UD2
3888 * (or occasionally a JMP to UD2).
3889 *
3890 * It may also insert a UD2 after calling a __noreturn function.
3891 */
3892 prev_insn = list_prev_entry(insn, list);
3893 if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
3894 (insn->type == INSN_BUG ||
3895 (insn->type == INSN_JUMP_UNCONDITIONAL &&
3896 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
3897 return true;
3898
3899 /*
3900 * Check if this (or a subsequent) instruction is related to
3901 * CONFIG_UBSAN or CONFIG_KASAN.
3902 *
3903 * End the search at 5 instructions to avoid going into the weeds.
3904 */
3905 for (i = 0; i < 5; i++) {
3906
3907 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
3908 return true;
3909
3910 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
3911 if (insn->jump_dest &&
3912 insn->jump_dest->func == insn->func) {
3913 insn = insn->jump_dest;
3914 continue;
3915 }
3916
3917 break;
3918 }
3919
3920 if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
3921 break;
3922
3923 insn = list_next_entry(insn, list);
3924 }
3925
3926 return false;
3927 }
3928
validate_symbol(struct objtool_file * file,struct section * sec,struct symbol * sym,struct insn_state * state)3929 static int validate_symbol(struct objtool_file *file, struct section *sec,
3930 struct symbol *sym, struct insn_state *state)
3931 {
3932 struct instruction *insn;
3933 int ret;
3934
3935 if (!sym->len) {
3936 WARN("%s() is missing an ELF size annotation", sym->name);
3937 return 1;
3938 }
3939
3940 if (sym->pfunc != sym || sym->alias != sym)
3941 return 0;
3942
3943 insn = find_insn(file, sec, sym->offset);
3944 if (!insn || insn->ignore || insn->visited)
3945 return 0;
3946
3947 state->uaccess = sym->uaccess_safe;
3948
3949 ret = validate_branch(file, insn->func, insn, *state);
3950 if (ret && opts.backtrace)
3951 BT_FUNC("<=== (sym)", insn);
3952 return ret;
3953 }
3954
validate_section(struct objtool_file * file,struct section * sec)3955 static int validate_section(struct objtool_file *file, struct section *sec)
3956 {
3957 struct insn_state state;
3958 struct symbol *func;
3959 int warnings = 0;
3960
3961 list_for_each_entry(func, &sec->symbol_list, list) {
3962 if (func->type != STT_FUNC)
3963 continue;
3964
3965 init_insn_state(file, &state, sec);
3966 set_func_state(&state.cfi);
3967
3968 warnings += validate_symbol(file, sec, func, &state);
3969 }
3970
3971 return warnings;
3972 }
3973
validate_noinstr_sections(struct objtool_file * file)3974 static int validate_noinstr_sections(struct objtool_file *file)
3975 {
3976 struct section *sec;
3977 int warnings = 0;
3978
3979 sec = find_section_by_name(file->elf, ".noinstr.text");
3980 if (sec) {
3981 warnings += validate_section(file, sec);
3982 warnings += validate_unwind_hints(file, sec);
3983 }
3984
3985 sec = find_section_by_name(file->elf, ".entry.text");
3986 if (sec) {
3987 warnings += validate_section(file, sec);
3988 warnings += validate_unwind_hints(file, sec);
3989 }
3990
3991 return warnings;
3992 }
3993
validate_functions(struct objtool_file * file)3994 static int validate_functions(struct objtool_file *file)
3995 {
3996 struct section *sec;
3997 int warnings = 0;
3998
3999 for_each_sec(file, sec) {
4000 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4001 continue;
4002
4003 warnings += validate_section(file, sec);
4004 }
4005
4006 return warnings;
4007 }
4008
mark_endbr_used(struct instruction * insn)4009 static void mark_endbr_used(struct instruction *insn)
4010 {
4011 if (!list_empty(&insn->call_node))
4012 list_del_init(&insn->call_node);
4013 }
4014
validate_ibt_insn(struct objtool_file * file,struct instruction * insn)4015 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4016 {
4017 struct instruction *dest;
4018 struct reloc *reloc;
4019 unsigned long off;
4020 int warnings = 0;
4021
4022 /*
4023 * Looking for function pointer load relocations. Ignore
4024 * direct/indirect branches:
4025 */
4026 switch (insn->type) {
4027 case INSN_CALL:
4028 case INSN_CALL_DYNAMIC:
4029 case INSN_JUMP_CONDITIONAL:
4030 case INSN_JUMP_UNCONDITIONAL:
4031 case INSN_JUMP_DYNAMIC:
4032 case INSN_JUMP_DYNAMIC_CONDITIONAL:
4033 case INSN_RETURN:
4034 case INSN_NOP:
4035 return 0;
4036 default:
4037 break;
4038 }
4039
4040 for (reloc = insn_reloc(file, insn);
4041 reloc;
4042 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4043 reloc->offset + 1,
4044 (insn->offset + insn->len) - (reloc->offset + 1))) {
4045
4046 /*
4047 * static_call_update() references the trampoline, which
4048 * doesn't have (or need) ENDBR. Skip warning in that case.
4049 */
4050 if (reloc->sym->static_call_tramp)
4051 continue;
4052
4053 off = reloc->sym->offset;
4054 if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)
4055 off += arch_dest_reloc_offset(reloc->addend);
4056 else
4057 off += reloc->addend;
4058
4059 dest = find_insn(file, reloc->sym->sec, off);
4060 if (!dest)
4061 continue;
4062
4063 if (dest->type == INSN_ENDBR) {
4064 mark_endbr_used(dest);
4065 continue;
4066 }
4067
4068 if (dest->func && dest->func == insn->func) {
4069 /*
4070 * Anything from->to self is either _THIS_IP_ or
4071 * IRET-to-self.
4072 *
4073 * There is no sane way to annotate _THIS_IP_ since the
4074 * compiler treats the relocation as a constant and is
4075 * happy to fold in offsets, skewing any annotation we
4076 * do, leading to vast amounts of false-positives.
4077 *
4078 * There's also compiler generated _THIS_IP_ through
4079 * KCOV and such which we have no hope of annotating.
4080 *
4081 * As such, blanket accept self-references without
4082 * issue.
4083 */
4084 continue;
4085 }
4086
4087 if (dest->noendbr)
4088 continue;
4089
4090 WARN_FUNC("relocation to !ENDBR: %s",
4091 insn->sec, insn->offset,
4092 offstr(dest->sec, dest->offset));
4093
4094 warnings++;
4095 }
4096
4097 return warnings;
4098 }
4099
validate_ibt_data_reloc(struct objtool_file * file,struct reloc * reloc)4100 static int validate_ibt_data_reloc(struct objtool_file *file,
4101 struct reloc *reloc)
4102 {
4103 struct instruction *dest;
4104
4105 dest = find_insn(file, reloc->sym->sec,
4106 reloc->sym->offset + reloc->addend);
4107 if (!dest)
4108 return 0;
4109
4110 if (dest->type == INSN_ENDBR) {
4111 mark_endbr_used(dest);
4112 return 0;
4113 }
4114
4115 if (dest->noendbr)
4116 return 0;
4117
4118 WARN_FUNC("data relocation to !ENDBR: %s",
4119 reloc->sec->base, reloc->offset,
4120 offstr(dest->sec, dest->offset));
4121
4122 return 1;
4123 }
4124
4125 /*
4126 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4127 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4128 * NOPs) later, in create_ibt_endbr_seal_sections().
4129 */
validate_ibt(struct objtool_file * file)4130 static int validate_ibt(struct objtool_file *file)
4131 {
4132 struct section *sec;
4133 struct reloc *reloc;
4134 struct instruction *insn;
4135 int warnings = 0;
4136
4137 for_each_insn(file, insn)
4138 warnings += validate_ibt_insn(file, insn);
4139
4140 for_each_sec(file, sec) {
4141
4142 /* Already done by validate_ibt_insn() */
4143 if (sec->sh.sh_flags & SHF_EXECINSTR)
4144 continue;
4145
4146 if (!sec->reloc)
4147 continue;
4148
4149 /*
4150 * These sections can reference text addresses, but not with
4151 * the intent to indirect branch to them.
4152 */
4153 if ((!strncmp(sec->name, ".discard", 8) &&
4154 strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
4155 !strncmp(sec->name, ".debug", 6) ||
4156 !strcmp(sec->name, ".altinstructions") ||
4157 !strcmp(sec->name, ".ibt_endbr_seal") ||
4158 !strcmp(sec->name, ".orc_unwind_ip") ||
4159 !strcmp(sec->name, ".parainstructions") ||
4160 !strcmp(sec->name, ".retpoline_sites") ||
4161 !strcmp(sec->name, ".smp_locks") ||
4162 !strcmp(sec->name, ".static_call_sites") ||
4163 !strcmp(sec->name, "_error_injection_whitelist") ||
4164 !strcmp(sec->name, "_kprobe_blacklist") ||
4165 !strcmp(sec->name, "__bug_table") ||
4166 !strcmp(sec->name, "__ex_table") ||
4167 !strcmp(sec->name, "__jump_table") ||
4168 !strcmp(sec->name, "__mcount_loc") ||
4169 !strcmp(sec->name, ".kcfi_traps") ||
4170 strstr(sec->name, "__patchable_function_entries"))
4171 continue;
4172
4173 list_for_each_entry(reloc, &sec->reloc->reloc_list, list)
4174 warnings += validate_ibt_data_reloc(file, reloc);
4175 }
4176
4177 return warnings;
4178 }
4179
validate_sls(struct objtool_file * file)4180 static int validate_sls(struct objtool_file *file)
4181 {
4182 struct instruction *insn, *next_insn;
4183 int warnings = 0;
4184
4185 for_each_insn(file, insn) {
4186 next_insn = next_insn_same_sec(file, insn);
4187
4188 if (insn->retpoline_safe)
4189 continue;
4190
4191 switch (insn->type) {
4192 case INSN_RETURN:
4193 if (!next_insn || next_insn->type != INSN_TRAP) {
4194 WARN_FUNC("missing int3 after ret",
4195 insn->sec, insn->offset);
4196 warnings++;
4197 }
4198
4199 break;
4200 case INSN_JUMP_DYNAMIC:
4201 if (!next_insn || next_insn->type != INSN_TRAP) {
4202 WARN_FUNC("missing int3 after indirect jump",
4203 insn->sec, insn->offset);
4204 warnings++;
4205 }
4206 break;
4207 default:
4208 break;
4209 }
4210 }
4211
4212 return warnings;
4213 }
4214
validate_reachable_instructions(struct objtool_file * file)4215 static int validate_reachable_instructions(struct objtool_file *file)
4216 {
4217 struct instruction *insn;
4218
4219 if (file->ignore_unreachables)
4220 return 0;
4221
4222 for_each_insn(file, insn) {
4223 if (insn->visited || ignore_unreachable_insn(file, insn))
4224 continue;
4225
4226 WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
4227 return 1;
4228 }
4229
4230 return 0;
4231 }
4232
check(struct objtool_file * file)4233 int check(struct objtool_file *file)
4234 {
4235 int ret, warnings = 0;
4236
4237 arch_initial_func_cfi_state(&initial_func_cfi);
4238 init_cfi_state(&init_cfi);
4239 init_cfi_state(&func_cfi);
4240 set_func_state(&func_cfi);
4241
4242 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4243 goto out;
4244
4245 cfi_hash_add(&init_cfi);
4246 cfi_hash_add(&func_cfi);
4247
4248 ret = decode_sections(file);
4249 if (ret < 0)
4250 goto out;
4251
4252 warnings += ret;
4253
4254 if (list_empty(&file->insn_list))
4255 goto out;
4256
4257 if (opts.retpoline) {
4258 ret = validate_retpoline(file);
4259 if (ret < 0)
4260 return ret;
4261 warnings += ret;
4262 }
4263
4264 if (opts.stackval || opts.orc || opts.uaccess) {
4265 ret = validate_functions(file);
4266 if (ret < 0)
4267 goto out;
4268 warnings += ret;
4269
4270 ret = validate_unwind_hints(file, NULL);
4271 if (ret < 0)
4272 goto out;
4273 warnings += ret;
4274
4275 if (!warnings) {
4276 ret = validate_reachable_instructions(file);
4277 if (ret < 0)
4278 goto out;
4279 warnings += ret;
4280 }
4281
4282 } else if (opts.noinstr) {
4283 ret = validate_noinstr_sections(file);
4284 if (ret < 0)
4285 goto out;
4286 warnings += ret;
4287 }
4288
4289 if (opts.unret) {
4290 /*
4291 * Must be after validate_branch() and friends, it plays
4292 * further games with insn->visited.
4293 */
4294 ret = validate_unret(file);
4295 if (ret < 0)
4296 return ret;
4297 warnings += ret;
4298 }
4299
4300 if (opts.ibt) {
4301 ret = validate_ibt(file);
4302 if (ret < 0)
4303 goto out;
4304 warnings += ret;
4305 }
4306
4307 if (opts.sls) {
4308 ret = validate_sls(file);
4309 if (ret < 0)
4310 goto out;
4311 warnings += ret;
4312 }
4313
4314 if (opts.static_call) {
4315 ret = create_static_call_sections(file);
4316 if (ret < 0)
4317 goto out;
4318 warnings += ret;
4319 }
4320
4321 if (opts.retpoline) {
4322 ret = create_retpoline_sites_sections(file);
4323 if (ret < 0)
4324 goto out;
4325 warnings += ret;
4326 }
4327
4328 if (opts.rethunk) {
4329 ret = create_return_sites_sections(file);
4330 if (ret < 0)
4331 goto out;
4332 warnings += ret;
4333 }
4334
4335 if (opts.mcount) {
4336 ret = create_mcount_loc_sections(file);
4337 if (ret < 0)
4338 goto out;
4339 warnings += ret;
4340 }
4341
4342 if (opts.ibt) {
4343 ret = create_ibt_endbr_seal_sections(file);
4344 if (ret < 0)
4345 goto out;
4346 warnings += ret;
4347 }
4348
4349 if (opts.orc && !list_empty(&file->insn_list)) {
4350 ret = orc_create(file);
4351 if (ret < 0)
4352 goto out;
4353 warnings += ret;
4354 }
4355
4356
4357 if (opts.stats) {
4358 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4359 printf("nr_cfi: %ld\n", nr_cfi);
4360 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4361 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4362 }
4363
4364 out:
4365 /*
4366 * For now, don't fail the kernel build on fatal warnings. These
4367 * errors are still fairly common due to the growing matrix of
4368 * supported toolchains and their recent pace of change.
4369 */
4370 return 0;
4371 }
4372