1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * sorttable.h
4 *
5 * Added ORC unwind tables sort support and other updates:
6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by:
7 * Shile Zhang <shile.zhang@linux.alibaba.com>
8 *
9 * Copyright 2011 - 2012 Cavium, Inc.
10 *
11 * Some of code was taken out of arch/x86/kernel/unwind_orc.c, written by:
12 * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
13 *
14 * Some of this code was taken out of recordmcount.h written by:
15 *
16 * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved.
17 * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
18 */
19
20 #undef extable_ent_size
21 #undef compare_extable
22 #undef do_sort
23 #undef Elf_Addr
24 #undef Elf_Ehdr
25 #undef Elf_Shdr
26 #undef Elf_Rel
27 #undef Elf_Rela
28 #undef Elf_Sym
29 #undef ELF_R_SYM
30 #undef Elf_r_sym
31 #undef ELF_R_INFO
32 #undef Elf_r_info
33 #undef ELF_ST_BIND
34 #undef ELF_ST_TYPE
35 #undef fn_ELF_R_SYM
36 #undef fn_ELF_R_INFO
37 #undef uint_t
38 #undef _r
39 #undef _w
40
41 #ifdef SORTTABLE_64
42 # define extable_ent_size 16
43 # define compare_extable compare_extable_64
44 # define do_sort do_sort_64
45 # define Elf_Addr Elf64_Addr
46 # define Elf_Ehdr Elf64_Ehdr
47 # define Elf_Shdr Elf64_Shdr
48 # define Elf_Rel Elf64_Rel
49 # define Elf_Rela Elf64_Rela
50 # define Elf_Sym Elf64_Sym
51 # define ELF_R_SYM ELF64_R_SYM
52 # define Elf_r_sym Elf64_r_sym
53 # define ELF_R_INFO ELF64_R_INFO
54 # define Elf_r_info Elf64_r_info
55 # define ELF_ST_BIND ELF64_ST_BIND
56 # define ELF_ST_TYPE ELF64_ST_TYPE
57 # define fn_ELF_R_SYM fn_ELF64_R_SYM
58 # define fn_ELF_R_INFO fn_ELF64_R_INFO
59 # define uint_t uint64_t
60 # define _r r8
61 # define _w w8
62 #else
63 # define extable_ent_size 8
64 # define compare_extable compare_extable_32
65 # define do_sort do_sort_32
66 # define Elf_Addr Elf32_Addr
67 # define Elf_Ehdr Elf32_Ehdr
68 # define Elf_Shdr Elf32_Shdr
69 # define Elf_Rel Elf32_Rel
70 # define Elf_Rela Elf32_Rela
71 # define Elf_Sym Elf32_Sym
72 # define ELF_R_SYM ELF32_R_SYM
73 # define Elf_r_sym Elf32_r_sym
74 # define ELF_R_INFO ELF32_R_INFO
75 # define Elf_r_info Elf32_r_info
76 # define ELF_ST_BIND ELF32_ST_BIND
77 # define ELF_ST_TYPE ELF32_ST_TYPE
78 # define fn_ELF_R_SYM fn_ELF32_R_SYM
79 # define fn_ELF_R_INFO fn_ELF32_R_INFO
80 # define uint_t uint32_t
81 # define _r r
82 # define _w w
83 #endif
84
85 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
86 /* ORC unwinder only support X86_64 */
87 #include <errno.h>
88 #include <pthread.h>
89 #include <asm/orc_types.h>
90
91 #define ERRSTR_MAXSZ 256
92
93 char g_err[ERRSTR_MAXSZ];
94 int *g_orc_ip_table;
95 struct orc_entry *g_orc_table;
96
97 pthread_t orc_sort_thread;
98
orc_ip(const int * ip)99 static inline unsigned long orc_ip(const int *ip)
100 {
101 return (unsigned long)ip + *ip;
102 }
103
orc_sort_cmp(const void * _a,const void * _b)104 static int orc_sort_cmp(const void *_a, const void *_b)
105 {
106 struct orc_entry *orc_a;
107 const int *a = g_orc_ip_table + *(int *)_a;
108 const int *b = g_orc_ip_table + *(int *)_b;
109 unsigned long a_val = orc_ip(a);
110 unsigned long b_val = orc_ip(b);
111
112 if (a_val > b_val)
113 return 1;
114 if (a_val < b_val)
115 return -1;
116
117 /*
118 * The "weak" section terminator entries need to always be on the left
119 * to ensure the lookup code skips them in favor of real entries.
120 * These terminator entries exist to handle any gaps created by
121 * whitelisted .o files which didn't get objtool generation.
122 */
123 orc_a = g_orc_table + (a - g_orc_ip_table);
124 return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1;
125 }
126
sort_orctable(void * arg)127 static void *sort_orctable(void *arg)
128 {
129 int i;
130 int *idxs = NULL;
131 int *tmp_orc_ip_table = NULL;
132 struct orc_entry *tmp_orc_table = NULL;
133 unsigned int *orc_ip_size = (unsigned int *)arg;
134 unsigned int num_entries = *orc_ip_size / sizeof(int);
135 unsigned int orc_size = num_entries * sizeof(struct orc_entry);
136
137 idxs = (int *)malloc(*orc_ip_size);
138 if (!idxs) {
139 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s",
140 strerror(errno));
141 pthread_exit(g_err);
142 }
143
144 tmp_orc_ip_table = (int *)malloc(*orc_ip_size);
145 if (!tmp_orc_ip_table) {
146 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s",
147 strerror(errno));
148 pthread_exit(g_err);
149 }
150
151 tmp_orc_table = (struct orc_entry *)malloc(orc_size);
152 if (!tmp_orc_table) {
153 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s",
154 strerror(errno));
155 pthread_exit(g_err);
156 }
157
158 /* initialize indices array, convert ip_table to absolute address */
159 for (i = 0; i < num_entries; i++) {
160 idxs[i] = i;
161 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int);
162 }
163 memcpy(tmp_orc_table, g_orc_table, orc_size);
164
165 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp);
166
167 for (i = 0; i < num_entries; i++) {
168 if (idxs[i] == i)
169 continue;
170
171 /* convert back to relative address */
172 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int);
173 g_orc_table[i] = tmp_orc_table[idxs[i]];
174 }
175
176 free(idxs);
177 free(tmp_orc_ip_table);
178 free(tmp_orc_table);
179 pthread_exit(NULL);
180 }
181 #endif
182
compare_extable(const void * a,const void * b)183 static int compare_extable(const void *a, const void *b)
184 {
185 Elf_Addr av = _r(a);
186 Elf_Addr bv = _r(b);
187
188 if (av < bv)
189 return -1;
190 if (av > bv)
191 return 1;
192 return 0;
193 }
194
do_sort(Elf_Ehdr * ehdr,char const * const fname,table_sort_t custom_sort)195 static int do_sort(Elf_Ehdr *ehdr,
196 char const *const fname,
197 table_sort_t custom_sort)
198 {
199 int rc = -1;
200 Elf_Shdr *s, *shdr = (Elf_Shdr *)((char *)ehdr + _r(&ehdr->e_shoff));
201 Elf_Shdr *strtab_sec = NULL;
202 Elf_Shdr *symtab_sec = NULL;
203 Elf_Shdr *extab_sec = NULL;
204 Elf_Sym *sym;
205 const Elf_Sym *symtab;
206 Elf32_Word *symtab_shndx = NULL;
207 Elf_Sym *sort_needed_sym = NULL;
208 Elf_Shdr *sort_needed_sec;
209 Elf_Rel *relocs = NULL;
210 int relocs_size = 0;
211 uint32_t *sort_needed_loc;
212 const char *secstrings;
213 const char *strtab;
214 char *extab_image;
215 int extab_index = 0;
216 int i;
217 int idx;
218 unsigned int shnum;
219 unsigned int shstrndx;
220 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
221 unsigned int orc_ip_size = 0;
222 unsigned int orc_size = 0;
223 unsigned int orc_num_entries = 0;
224 #endif
225
226 shstrndx = r2(&ehdr->e_shstrndx);
227 if (shstrndx == SHN_XINDEX)
228 shstrndx = r(&shdr[0].sh_link);
229 secstrings = (const char *)ehdr + _r(&shdr[shstrndx].sh_offset);
230
231 shnum = r2(&ehdr->e_shnum);
232 if (shnum == SHN_UNDEF)
233 shnum = _r(&shdr[0].sh_size);
234
235 for (i = 0, s = shdr; s < shdr + shnum; i++, s++) {
236 idx = r(&s->sh_name);
237 if (!strcmp(secstrings + idx, "__ex_table")) {
238 extab_sec = s;
239 extab_index = i;
240 }
241 if (!strcmp(secstrings + idx, ".symtab"))
242 symtab_sec = s;
243 if (!strcmp(secstrings + idx, ".strtab"))
244 strtab_sec = s;
245
246 if ((r(&s->sh_type) == SHT_REL ||
247 r(&s->sh_type) == SHT_RELA) &&
248 r(&s->sh_info) == extab_index) {
249 relocs = (void *)ehdr + _r(&s->sh_offset);
250 relocs_size = _r(&s->sh_size);
251 }
252 if (r(&s->sh_type) == SHT_SYMTAB_SHNDX)
253 symtab_shndx = (Elf32_Word *)((const char *)ehdr +
254 _r(&s->sh_offset));
255
256 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
257 /* locate the ORC unwind tables */
258 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) {
259 orc_ip_size = s->sh_size;
260 g_orc_ip_table = (int *)((void *)ehdr +
261 s->sh_offset);
262 }
263 if (!strcmp(secstrings + idx, ".orc_unwind")) {
264 orc_size = s->sh_size;
265 g_orc_table = (struct orc_entry *)((void *)ehdr +
266 s->sh_offset);
267 }
268 #endif
269 } /* for loop */
270
271 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
272 if (!g_orc_ip_table || !g_orc_table) {
273 fprintf(stderr,
274 "incomplete ORC unwind tables in file: %s\n", fname);
275 goto out;
276 }
277
278 orc_num_entries = orc_ip_size / sizeof(int);
279 if (orc_ip_size % sizeof(int) != 0 ||
280 orc_size % sizeof(struct orc_entry) != 0 ||
281 orc_num_entries != orc_size / sizeof(struct orc_entry)) {
282 fprintf(stderr,
283 "inconsistent ORC unwind table entries in file: %s\n",
284 fname);
285 goto out;
286 }
287
288 /* create thread to sort ORC unwind tables concurrently */
289 if (pthread_create(&orc_sort_thread, NULL,
290 sort_orctable, &orc_ip_size)) {
291 fprintf(stderr,
292 "pthread_create orc_sort_thread failed '%s': %s\n",
293 strerror(errno), fname);
294 goto out;
295 }
296 #endif
297 if (!extab_sec) {
298 fprintf(stderr, "no __ex_table in file: %s\n", fname);
299 goto out;
300 }
301
302 if (!symtab_sec) {
303 fprintf(stderr, "no .symtab in file: %s\n", fname);
304 goto out;
305 }
306
307 if (!strtab_sec) {
308 fprintf(stderr, "no .strtab in file: %s\n", fname);
309 goto out;
310 }
311
312 extab_image = (void *)ehdr + _r(&extab_sec->sh_offset);
313 strtab = (const char *)ehdr + _r(&strtab_sec->sh_offset);
314 symtab = (const Elf_Sym *)((const char *)ehdr +
315 _r(&symtab_sec->sh_offset));
316
317 if (custom_sort) {
318 custom_sort(extab_image, _r(&extab_sec->sh_size));
319 } else {
320 int num_entries = _r(&extab_sec->sh_size) / extable_ent_size;
321 qsort(extab_image, num_entries,
322 extable_ent_size, compare_extable);
323 }
324
325 /* If there were relocations, we no longer need them. */
326 if (relocs)
327 memset(relocs, 0, relocs_size);
328
329 /* find the flag main_extable_sort_needed */
330 for (sym = (void *)ehdr + _r(&symtab_sec->sh_offset);
331 sym < sym + _r(&symtab_sec->sh_size) / sizeof(Elf_Sym);
332 sym++) {
333 if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT)
334 continue;
335 if (!strcmp(strtab + r(&sym->st_name),
336 "main_extable_sort_needed")) {
337 sort_needed_sym = sym;
338 break;
339 }
340 }
341
342 if (!sort_needed_sym) {
343 fprintf(stderr,
344 "no main_extable_sort_needed symbol in file: %s\n",
345 fname);
346 goto out;
347 }
348
349 sort_needed_sec = &shdr[get_secindex(r2(&sym->st_shndx),
350 sort_needed_sym - symtab,
351 symtab_shndx)];
352 sort_needed_loc = (void *)ehdr +
353 _r(&sort_needed_sec->sh_offset) +
354 _r(&sort_needed_sym->st_value) -
355 _r(&sort_needed_sec->sh_addr);
356
357 /* extable has been sorted, clear the flag */
358 w(0, sort_needed_loc);
359 rc = 0;
360
361 out:
362 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
363 if (orc_sort_thread) {
364 void *retval = NULL;
365 /* wait for ORC tables sort done */
366 rc = pthread_join(orc_sort_thread, &retval);
367 if (rc)
368 fprintf(stderr,
369 "pthread_join failed '%s': %s\n",
370 strerror(errno), fname);
371 else if (retval) {
372 rc = -1;
373 fprintf(stderr,
374 "failed to sort ORC tables '%s': %s\n",
375 (char *)retval, fname);
376 }
377 }
378 #endif
379 return rc;
380 }
381