Lines Matching +full:three +full:- +full:conversion +full:- +full:cycles
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
34 /* raw BTF data in non-native endianness */
46 * +--------------------------------+
48 * +--------------------------------+
52 * types_data-+ |
53 * strs_data------------+
57 * representation is broken up into three independently allocated
64 * +----------+ +---------+ +-----------+
66 * +----------+ +---------+ +-----------+
70 * types_data----+ |
71 * strset__data(strs_set)-----+
73 * +----------+---------+-----------+
75 * raw_data----->+----------+---------+-----------+
80 size_t types_data_cap; /* used size stored in hdr->type_len */
83 * type_offs[0] corresponds to the first non-VOID type:
84 * - for base BTF it's type [1];
85 * - for split BTF it's the first non-base BTF type.
90 * - doesn't include special [0] void type;
91 * - for split BTF counts number of types added on top of base BTF.
99 * - for base BTF it's equal to 1;
100 * - for split BTF it's equal to biggest type ID of base BTF plus 1.
104 * - for base BTF it's equal to 0;
105 * - for split BTF it's equal to total size of base BTF's string section size.
109 /* only one of strs_data or strs_set can be non-NULL, depending on
168 memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz); in libbpf_add_mem()
185 p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); in libbpf_ensure_mem()
187 return -ENOMEM; in libbpf_ensure_mem()
194 return libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), in btf_add_type_offs_mem()
195 btf->nr_types, BTF_MAX_NR_TYPES, add_cnt); in btf_add_type_offs_mem()
204 return -ENOMEM; in btf_add_type_idx_entry()
212 h->magic = bswap_16(h->magic); in btf_bswap_hdr()
213 h->hdr_len = bswap_32(h->hdr_len); in btf_bswap_hdr()
214 h->type_off = bswap_32(h->type_off); in btf_bswap_hdr()
215 h->type_len = bswap_32(h->type_len); in btf_bswap_hdr()
216 h->str_off = bswap_32(h->str_off); in btf_bswap_hdr()
217 h->str_len = bswap_32(h->str_len); in btf_bswap_hdr()
222 struct btf_header *hdr = btf->hdr; in btf_parse_hdr()
225 if (btf->raw_size < sizeof(struct btf_header)) { in btf_parse_hdr()
227 return -EINVAL; in btf_parse_hdr()
230 if (hdr->magic == bswap_16(BTF_MAGIC)) { in btf_parse_hdr()
231 btf->swapped_endian = true; in btf_parse_hdr()
232 if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) { in btf_parse_hdr()
233 pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", in btf_parse_hdr()
234 bswap_32(hdr->hdr_len)); in btf_parse_hdr()
235 return -ENOTSUP; in btf_parse_hdr()
238 } else if (hdr->magic != BTF_MAGIC) { in btf_parse_hdr()
239 pr_debug("Invalid BTF magic: %x\n", hdr->magic); in btf_parse_hdr()
240 return -EINVAL; in btf_parse_hdr()
243 if (btf->raw_size < hdr->hdr_len) { in btf_parse_hdr()
245 hdr->hdr_len, btf->raw_size); in btf_parse_hdr()
246 return -EINVAL; in btf_parse_hdr()
249 meta_left = btf->raw_size - hdr->hdr_len; in btf_parse_hdr()
250 if (meta_left < (long long)hdr->str_off + hdr->str_len) { in btf_parse_hdr()
251 pr_debug("Invalid BTF total size: %u\n", btf->raw_size); in btf_parse_hdr()
252 return -EINVAL; in btf_parse_hdr()
255 if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) { in btf_parse_hdr()
257 hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len); in btf_parse_hdr()
258 return -EINVAL; in btf_parse_hdr()
261 if (hdr->type_off % 4) { in btf_parse_hdr()
263 return -EINVAL; in btf_parse_hdr()
271 const struct btf_header *hdr = btf->hdr; in btf_parse_str_sec()
272 const char *start = btf->strs_data; in btf_parse_str_sec()
273 const char *end = start + btf->hdr->str_len; in btf_parse_str_sec()
275 if (btf->base_btf && hdr->str_len == 0) in btf_parse_str_sec()
277 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) { in btf_parse_str_sec()
279 return -EINVAL; in btf_parse_str_sec()
281 if (!btf->base_btf && start[0]) { in btf_parse_str_sec()
283 return -EINVAL; in btf_parse_str_sec()
325 return -EINVAL; in btf_type_size()
331 t->name_off = bswap_32(t->name_off); in btf_bswap_type_base()
332 t->info = bswap_32(t->info); in btf_bswap_type_base()
333 t->type = bswap_32(t->type); in btf_bswap_type_base()
363 e->name_off = bswap_32(e->name_off); in btf_bswap_type_rest()
364 e->val = bswap_32(e->val); in btf_bswap_type_rest()
369 e64->name_off = bswap_32(e64->name_off); in btf_bswap_type_rest()
370 e64->val_lo32 = bswap_32(e64->val_lo32); in btf_bswap_type_rest()
371 e64->val_hi32 = bswap_32(e64->val_hi32); in btf_bswap_type_rest()
376 a->type = bswap_32(a->type); in btf_bswap_type_rest()
377 a->index_type = bswap_32(a->index_type); in btf_bswap_type_rest()
378 a->nelems = bswap_32(a->nelems); in btf_bswap_type_rest()
383 m->name_off = bswap_32(m->name_off); in btf_bswap_type_rest()
384 m->type = bswap_32(m->type); in btf_bswap_type_rest()
385 m->offset = bswap_32(m->offset); in btf_bswap_type_rest()
390 p->name_off = bswap_32(p->name_off); in btf_bswap_type_rest()
391 p->type = bswap_32(p->type); in btf_bswap_type_rest()
395 btf_var(t)->linkage = bswap_32(btf_var(t)->linkage); in btf_bswap_type_rest()
399 v->type = bswap_32(v->type); in btf_bswap_type_rest()
400 v->offset = bswap_32(v->offset); in btf_bswap_type_rest()
401 v->size = bswap_32(v->size); in btf_bswap_type_rest()
405 btf_decl_tag(t)->component_idx = bswap_32(btf_decl_tag(t)->component_idx); in btf_bswap_type_rest()
409 return -EINVAL; in btf_bswap_type_rest()
415 struct btf_header *hdr = btf->hdr; in btf_parse_type_sec()
416 void *next_type = btf->types_data; in btf_parse_type_sec()
417 void *end_type = next_type + hdr->type_len; in btf_parse_type_sec()
421 if (btf->swapped_endian) in btf_parse_type_sec()
428 pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types); in btf_parse_type_sec()
429 return -EINVAL; in btf_parse_type_sec()
432 if (btf->swapped_endian && btf_bswap_type_rest(next_type)) in btf_parse_type_sec()
433 return -EINVAL; in btf_parse_type_sec()
435 err = btf_add_type_idx_entry(btf, next_type - btf->types_data); in btf_parse_type_sec()
440 btf->nr_types++; in btf_parse_type_sec()
445 return -EINVAL; in btf_parse_type_sec()
453 return btf->start_id + btf->nr_types; in btf__type_cnt()
458 return btf->base_btf; in btf__base_btf()
461 /* internal helper returning non-const pointer to a type */
466 if (type_id < btf->start_id) in btf_type_by_id()
467 return btf_type_by_id(btf->base_btf, type_id); in btf_type_by_id()
468 return btf->types_data + btf->type_offs[type_id - btf->start_id]; in btf_type_by_id()
473 if (type_id >= btf->start_id + btf->nr_types) in btf__type_by_id()
497 if (btf->base_btf && btf->base_btf->ptr_sz > 0) in determine_ptr_size()
498 return btf->base_btf->ptr_sz; in determine_ptr_size()
506 if (t->size != 4 && t->size != 8) in determine_ptr_size()
509 name = btf__name_by_offset(btf, t->name_off); in determine_ptr_size()
515 return t->size; in determine_ptr_size()
519 return -1; in determine_ptr_size()
524 if (!btf->ptr_sz) in btf_ptr_sz()
525 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); in btf_ptr_sz()
526 return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz; in btf_ptr_sz()
539 if (!btf->ptr_sz) in btf__pointer_size()
540 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); in btf__pointer_size()
542 if (btf->ptr_sz < 0) in btf__pointer_size()
546 return btf->ptr_sz; in btf__pointer_size()
555 return libbpf_err(-EINVAL); in btf__set_pointer_size()
556 btf->ptr_sz = ptr_sz; in btf__set_pointer_size()
574 return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; in btf__endianness()
576 return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; in btf__endianness()
582 return libbpf_err(-EINVAL); in btf__set_endianness()
584 btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); in btf__set_endianness()
585 if (!btf->swapped_endian) { in btf__set_endianness()
586 free(btf->raw_data_swapped); in btf__set_endianness()
587 btf->raw_data_swapped = NULL; in btf__set_endianness()
609 __s64 size = -1; in btf__resolve_size()
622 size = t->size; in btf__resolve_size()
634 type_id = t->type; in btf__resolve_size()
638 if (nelems && array->nelems > UINT32_MAX / nelems) in btf__resolve_size()
639 return libbpf_err(-E2BIG); in btf__resolve_size()
640 nelems *= array->nelems; in btf__resolve_size()
641 type_id = array->type; in btf__resolve_size()
644 return libbpf_err(-EINVAL); in btf__resolve_size()
652 return libbpf_err(-EINVAL); in btf__resolve_size()
654 return libbpf_err(-E2BIG); in btf__resolve_size()
669 return min(btf_ptr_sz(btf), (size_t)t->size); in btf__align_of()
677 return btf__align_of(btf, t->type); in btf__align_of()
679 return btf__align_of(btf, btf_array(t)->type); in btf__align_of()
687 align = btf__align_of(btf, m->type); in btf__align_of()
696 (m->offset % (8 * align)) != 0) in btf__align_of()
703 if ((t->size % max_align) != 0) in btf__align_of()
723 type_id = t->type; in btf__resolve_type()
729 return libbpf_err(-EINVAL); in btf__resolve_type()
743 const char *name = btf__name_by_offset(btf, t->name_off); in btf__find_by_name()
749 return libbpf_err(-ENOENT); in btf__find_by_name()
766 name = btf__name_by_offset(btf, t->name_off); in btf_find_by_name_kind()
771 return libbpf_err(-ENOENT); in btf_find_by_name_kind()
777 return btf_find_by_name_kind(btf, btf->start_id, type_name, kind); in btf__find_by_name_kind_own()
788 return (void *)btf->hdr != btf->raw_data; in btf_is_modifiable()
796 if (btf->fd >= 0) in btf__free()
797 close(btf->fd); in btf__free()
801 * in-memory representation for header, types, and strings in btf__free()
806 free(btf->hdr); in btf__free()
807 free(btf->types_data); in btf__free()
808 strset__free(btf->strs_set); in btf__free()
810 free(btf->raw_data); in btf__free()
811 free(btf->raw_data_swapped); in btf__free()
812 free(btf->type_offs); in btf__free()
822 return ERR_PTR(-ENOMEM); in btf_new_empty()
824 btf->nr_types = 0; in btf_new_empty()
825 btf->start_id = 1; in btf_new_empty()
826 btf->start_str_off = 0; in btf_new_empty()
827 btf->fd = -1; in btf_new_empty()
828 btf->ptr_sz = sizeof(void *); in btf_new_empty()
829 btf->swapped_endian = false; in btf_new_empty()
832 btf->base_btf = base_btf; in btf_new_empty()
833 btf->start_id = btf__type_cnt(base_btf); in btf_new_empty()
834 btf->start_str_off = base_btf->hdr->str_len; in btf_new_empty()
838 btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1); in btf_new_empty()
839 btf->raw_data = calloc(1, btf->raw_size); in btf_new_empty()
840 if (!btf->raw_data) { in btf_new_empty()
842 return ERR_PTR(-ENOMEM); in btf_new_empty()
845 btf->hdr = btf->raw_data; in btf_new_empty()
846 btf->hdr->hdr_len = sizeof(struct btf_header); in btf_new_empty()
847 btf->hdr->magic = BTF_MAGIC; in btf_new_empty()
848 btf->hdr->version = BTF_VERSION; in btf_new_empty()
850 btf->types_data = btf->raw_data + btf->hdr->hdr_len; in btf_new_empty()
851 btf->strs_data = btf->raw_data + btf->hdr->hdr_len; in btf_new_empty()
852 btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */ in btf_new_empty()
874 return ERR_PTR(-ENOMEM); in btf_new()
876 btf->nr_types = 0; in btf_new()
877 btf->start_id = 1; in btf_new()
878 btf->start_str_off = 0; in btf_new()
879 btf->fd = -1; in btf_new()
882 btf->base_btf = base_btf; in btf_new()
883 btf->start_id = btf__type_cnt(base_btf); in btf_new()
884 btf->start_str_off = base_btf->hdr->str_len; in btf_new()
887 btf->raw_data = malloc(size); in btf_new()
888 if (!btf->raw_data) { in btf_new()
889 err = -ENOMEM; in btf_new()
892 memcpy(btf->raw_data, data, size); in btf_new()
893 btf->raw_size = size; in btf_new()
895 btf->hdr = btf->raw_data; in btf_new()
900 btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off; in btf_new()
901 btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off; in btf_new()
926 int err = 0, fd = -1, idx = 0; in btf_parse_elf()
935 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); in btf_parse_elf()
940 err = -errno; in btf_parse_elf()
945 err = -LIBBPF_ERRNO__FORMAT; in btf_parse_elf()
1005 err = -ENODATA; in btf_parse_elf()
1008 btf = btf_new(btf_data->d_buf, btf_data->d_size, base_btf); in btf_parse_elf()
1026 *btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); in btf_parse_elf()
1069 err = -errno; in btf_parse_raw()
1075 err = -EIO; in btf_parse_raw()
1080 err = -EPROTO; in btf_parse_raw()
1086 err = -errno; in btf_parse_raw()
1091 err = -errno; in btf_parse_raw()
1096 err = -errno; in btf_parse_raw()
1100 /* pre-alloc memory and read all of BTF data */ in btf_parse_raw()
1103 err = -ENOMEM; in btf_parse_raw()
1107 err = -EIO; in btf_parse_raw()
1143 if (err != -EPROTO) in btf_parse()
1168 if (btf->fd >= 0) in btf_load_into_kernel()
1169 return libbpf_err(-EEXIST); in btf_load_into_kernel()
1171 return libbpf_err(-EINVAL); in btf_load_into_kernel()
1176 err = -ENOMEM; in btf_load_into_kernel()
1179 btf->raw_size = raw_size; in btf_load_into_kernel()
1180 btf->raw_data = raw_data; in btf_load_into_kernel()
1185 * retry, using either auto-allocated or custom log_buf. This way in btf_load_into_kernel()
1186 * non-NULL custom log_buf provides a buffer just in case, but hopes in btf_load_into_kernel()
1198 err = -ENOMEM; in btf_load_into_kernel()
1210 btf->fd = bpf_btf_load(raw_data, raw_size, &opts); in btf_load_into_kernel()
1211 if (btf->fd < 0) { in btf_load_into_kernel()
1223 err = -errno; in btf_load_into_kernel()
1227 pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf); in btf_load_into_kernel()
1242 return btf->fd; in btf__fd()
1247 btf->fd = fd; in btf__set_fd()
1252 return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set); in btf_strs_data()
1257 struct btf_header *hdr = btf->hdr; in btf_get_raw_data()
1263 data = swap_endian ? btf->raw_data_swapped : btf->raw_data; in btf_get_raw_data()
1265 *size = btf->raw_size; in btf_get_raw_data()
1269 data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len; in btf_get_raw_data()
1275 memcpy(p, hdr, hdr->hdr_len); in btf_get_raw_data()
1278 p += hdr->hdr_len; in btf_get_raw_data()
1280 memcpy(p, btf->types_data, hdr->type_len); in btf_get_raw_data()
1282 for (i = 0; i < btf->nr_types; i++) { in btf_get_raw_data()
1283 t = p + btf->type_offs[i]; in btf_get_raw_data()
1284 /* btf_bswap_type_rest() relies on native t->info, so in btf_get_raw_data()
1293 p += hdr->type_len; in btf_get_raw_data()
1295 memcpy(p, btf_strs_data(btf), hdr->str_len); in btf_get_raw_data()
1296 p += hdr->str_len; in btf_get_raw_data()
1311 data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian); in btf__raw_data()
1315 btf->raw_size = data_sz; in btf__raw_data()
1316 if (btf->swapped_endian) in btf__raw_data()
1317 btf->raw_data_swapped = data; in btf__raw_data()
1319 btf->raw_data = data; in btf__raw_data()
1329 if (offset < btf->start_str_off) in btf__str_by_offset()
1330 return btf__str_by_offset(btf->base_btf, offset); in btf__str_by_offset()
1331 else if (offset - btf->start_str_off < btf->hdr->str_len) in btf__str_by_offset()
1332 return btf_strs_data(btf) + (offset - btf->start_str_off); in btf__str_by_offset()
1352 * let's start with a sane default - 4KiB here - and resize it only if in btf_get_from_fd()
1358 return ERR_PTR(-ENOMEM); in btf_get_from_fd()
1371 btf = ERR_PTR(-ENOMEM); in btf_get_from_fd()
1385 btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG); in btf_get_from_fd()
1403 return libbpf_err_ptr(-errno); in btf__load_from_kernel_by_id_split()
1418 if (btf->raw_data) { in btf_invalidate_raw_data()
1419 free(btf->raw_data); in btf_invalidate_raw_data()
1420 btf->raw_data = NULL; in btf_invalidate_raw_data()
1422 if (btf->raw_data_swapped) { in btf_invalidate_raw_data()
1423 free(btf->raw_data_swapped); in btf_invalidate_raw_data()
1424 btf->raw_data_swapped = NULL; in btf_invalidate_raw_data()
1428 /* Ensure BTF is ready to be modified (by splitting into a three memory
1436 int err = -ENOMEM; in btf_ensure_modifiable()
1444 /* split raw data into three memory regions */ in btf_ensure_modifiable()
1445 hdr = malloc(btf->hdr->hdr_len); in btf_ensure_modifiable()
1446 types = malloc(btf->hdr->type_len); in btf_ensure_modifiable()
1450 memcpy(hdr, btf->hdr, btf->hdr->hdr_len); in btf_ensure_modifiable()
1451 memcpy(types, btf->types_data, btf->hdr->type_len); in btf_ensure_modifiable()
1454 set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len); in btf_ensure_modifiable()
1461 btf->hdr = hdr; in btf_ensure_modifiable()
1462 btf->types_data = types; in btf_ensure_modifiable()
1463 btf->types_data_cap = btf->hdr->type_len; in btf_ensure_modifiable()
1464 btf->strs_data = NULL; in btf_ensure_modifiable()
1465 btf->strs_set = set; in btf_ensure_modifiable()
1469 if (btf->hdr->str_len == 0) in btf_ensure_modifiable()
1470 btf->strs_deduped = true; in btf_ensure_modifiable()
1471 if (!btf->base_btf && btf->hdr->str_len == 1) in btf_ensure_modifiable()
1472 btf->strs_deduped = true; in btf_ensure_modifiable()
1488 * - >0 offset into string section, if string is found;
1489 * - -ENOENT, if string is not in the string section;
1490 * - <0, on any other error.
1496 if (btf->base_btf) { in btf__find_str()
1497 off = btf__find_str(btf->base_btf, s); in btf__find_str()
1498 if (off != -ENOENT) in btf__find_str()
1504 return libbpf_err(-ENOMEM); in btf__find_str()
1506 off = strset__find_str(btf->strs_set, s); in btf__find_str()
1510 return btf->start_str_off + off; in btf__find_str()
1515 * - > 0 offset into string section, on success;
1516 * - < 0, on error.
1522 if (btf->base_btf) { in btf__add_str()
1523 off = btf__find_str(btf->base_btf, s); in btf__add_str()
1524 if (off != -ENOENT) in btf__add_str()
1529 return libbpf_err(-ENOMEM); in btf__add_str()
1531 off = strset__add_str(btf->strs_set, s); in btf__add_str()
1535 btf->hdr->str_len = strset__data_size(btf->strs_set); in btf__add_str()
1537 return btf->start_str_off + off; in btf__add_str()
1542 return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1, in btf_add_type_mem()
1543 btf->hdr->type_len, UINT_MAX, add_sz); in btf_add_type_mem()
1548 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t)); in btf_type_inc_vlen()
1555 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf_commit_type()
1559 btf->hdr->type_len += data_sz; in btf_commit_type()
1560 btf->hdr->str_off += data_sz; in btf_commit_type()
1561 btf->nr_types++; in btf_commit_type()
1562 return btf->start_id + btf->nr_types - 1; in btf_commit_type()
1580 if (p->str_off_map && in btf_rewrite_str()
1581 hashmap__find(p->str_off_map, *str_off, &mapped_off)) { in btf_rewrite_str()
1586 off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off)); in btf_rewrite_str()
1593 if (p->str_off_map) { in btf_rewrite_str()
1594 err = hashmap__append(p->str_off_map, *str_off, off); in btf_rewrite_str()
1615 return libbpf_err(-ENOMEM); in btf__add_type()
1619 return libbpf_err(-ENOMEM); in btf__add_type()
1638 * btf->start_id + btf->nr_types - 1 is the type ID offset we should in btf_rewrite_type_ids()
1641 *type_id += btf->start_id + btf->nr_types - 1; in btf_rewrite_type_ids()
1656 if (src_btf->base_btf) in btf__add_btf()
1657 return libbpf_err(-ENOTSUP); in btf__add_btf()
1661 return libbpf_err(-ENOMEM); in btf__add_btf()
1666 old_strs_len = btf->hdr->str_len; in btf__add_btf()
1668 data_sz = src_btf->hdr->type_len; in btf__add_btf()
1669 cnt = btf__type_cnt(src_btf) - 1; in btf__add_btf()
1671 /* pre-allocate enough memory for new types */ in btf__add_btf()
1674 return libbpf_err(-ENOMEM); in btf__add_btf()
1676 /* pre-allocate enough memory for type offset index for new types */ in btf__add_btf()
1679 return libbpf_err(-ENOMEM); in btf__add_btf()
1684 return libbpf_err(-ENOMEM); in btf__add_btf()
1687 memcpy(t, src_btf->types_data, data_sz); in btf__add_btf()
1698 *off = t - btf->types_data; in btf__add_btf()
1718 * pre-allocated, but it would not be available for querying. But now in btf__add_btf()
1723 btf->hdr->type_len += data_sz; in btf__add_btf()
1724 btf->hdr->str_off += data_sz; in btf__add_btf()
1725 btf->nr_types += cnt; in btf__add_btf()
1730 return btf->start_id + btf->nr_types - cnt; in btf__add_btf()
1735 memset(btf->types_data + btf->hdr->type_len, 0, data_sz); in btf__add_btf()
1736 memset(btf->strs_data + old_strs_len, 0, btf->hdr->str_len - old_strs_len); in btf__add_btf()
1741 btf->hdr->str_len = old_strs_len; in btf__add_btf()
1750 * - *name* - non-empty, non-NULL type name;
1751 * - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes;
1752 * - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL.
1754 * - >0, type ID of newly added BTF type;
1755 * - <0, on error.
1762 /* non-empty name */ in btf__add_int()
1764 return libbpf_err(-EINVAL); in btf__add_int()
1766 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16) in btf__add_int()
1767 return libbpf_err(-EINVAL); in btf__add_int()
1769 return libbpf_err(-EINVAL); in btf__add_int()
1773 return libbpf_err(-ENOMEM); in btf__add_int()
1778 return libbpf_err(-ENOMEM); in btf__add_int()
1788 t->name_off = name_off; in btf__add_int()
1789 t->info = btf_type_info(BTF_KIND_INT, 0, 0); in btf__add_int()
1790 t->size = byte_sz; in btf__add_int()
1799 * - *name* - non-empty, non-NULL type name;
1800 * - *sz* - size of the type, in bytes;
1802 * - >0, type ID of newly added BTF type;
1803 * - <0, on error.
1810 /* non-empty name */ in btf__add_float()
1812 return libbpf_err(-EINVAL); in btf__add_float()
1817 return libbpf_err(-EINVAL); in btf__add_float()
1820 return libbpf_err(-ENOMEM); in btf__add_float()
1825 return libbpf_err(-ENOMEM); in btf__add_float()
1831 t->name_off = name_off; in btf__add_float()
1832 t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0); in btf__add_float()
1833 t->size = byte_sz; in btf__add_float()
1845 return -EINVAL; in validate_type_id()
1856 return libbpf_err(-EINVAL); in btf_add_ref_kind()
1859 return libbpf_err(-ENOMEM); in btf_add_ref_kind()
1864 return libbpf_err(-ENOMEM); in btf_add_ref_kind()
1872 t->name_off = name_off; in btf_add_ref_kind()
1873 t->info = btf_type_info(kind, 0, 0); in btf_add_ref_kind()
1874 t->type = ref_type_id; in btf_add_ref_kind()
1881 * - *ref_type_id* - referenced type ID, it might not exist yet;
1883 * - >0, type ID of newly added BTF type;
1884 * - <0, on error.
1893 * - *index_type_id* - type ID of the type describing array index;
1894 * - *elem_type_id* - type ID of the type describing array element;
1895 * - *nr_elems* - the size of the array;
1897 * - >0, type ID of newly added BTF type;
1898 * - <0, on error.
1907 return libbpf_err(-EINVAL); in btf__add_array()
1910 return libbpf_err(-ENOMEM); in btf__add_array()
1915 return libbpf_err(-ENOMEM); in btf__add_array()
1917 t->name_off = 0; in btf__add_array()
1918 t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0); in btf__add_array()
1919 t->size = 0; in btf__add_array()
1922 a->type = elem_type_id; in btf__add_array()
1923 a->index_type = index_type_id; in btf__add_array()
1924 a->nelems = nr_elems; in btf__add_array()
1936 return libbpf_err(-ENOMEM); in btf_add_composite()
1941 return libbpf_err(-ENOMEM); in btf_add_composite()
1952 t->name_off = name_off; in btf_add_composite()
1953 t->info = btf_type_info(kind, 0, 0); in btf_add_composite()
1954 t->size = bytes_sz; in btf_add_composite()
1961 * - *name* - name of the struct, can be NULL or empty for anonymous structs;
1962 * - *byte_sz* - size of the struct, in bytes;
1968 * - >0, type ID of newly added BTF type;
1969 * - <0, on error.
1978 * - *name* - name of the union, can be NULL or empty for anonymous union;
1979 * - *byte_sz* - size of the union, in bytes;
1986 * - >0, type ID of newly added BTF type;
1987 * - <0, on error.
1996 return btf_type_by_id(btf, btf__type_cnt(btf) - 1); in btf_last_type()
2001 * - *name* - name of the field, can be NULL or empty for anonymous field;
2002 * - *type_id* - type ID for the type describing field type;
2003 * - *bit_offset* - bit offset of the start of the field within struct/union;
2004 * - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields;
2006 * - 0, on success;
2007 * - <0, on error.
2018 if (btf->nr_types == 0) in btf__add_field()
2019 return libbpf_err(-EINVAL); in btf__add_field()
2022 return libbpf_err(-EINVAL); in btf__add_field()
2025 return libbpf_err(-EINVAL); in btf__add_field()
2026 /* best-effort bit field offset/size enforcement */ in btf__add_field()
2029 return libbpf_err(-EINVAL); in btf__add_field()
2033 return libbpf_err(-EINVAL); in btf__add_field()
2037 return libbpf_err(-ENOMEM); in btf__add_field()
2042 return libbpf_err(-ENOMEM); in btf__add_field()
2050 m->name_off = name_off; in btf__add_field()
2051 m->type = type_id; in btf__add_field()
2052 m->offset = bit_offset | (bit_size << 24); in btf__add_field()
2057 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t)); in btf__add_field()
2059 btf->hdr->type_len += sz; in btf__add_field()
2060 btf->hdr->str_off += sz; in btf__add_field()
2071 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8) in btf_add_enum_common()
2072 return libbpf_err(-EINVAL); in btf_add_enum_common()
2075 return libbpf_err(-ENOMEM); in btf_add_enum_common()
2080 return libbpf_err(-ENOMEM); in btf_add_enum_common()
2089 t->name_off = name_off; in btf_add_enum_common()
2090 t->info = btf_type_info(kind, 0, is_signed); in btf_add_enum_common()
2091 t->size = byte_sz; in btf_add_enum_common()
2098 * - *name* - name of the enum, can be NULL or empty for anonymous enums;
2099 * - *byte_sz* - size of the enum, in bytes.
2106 * - >0, type ID of newly added BTF type;
2107 * - <0, on error.
2120 * - *name* - name of the enumerator value, can't be NULL or empty;
2121 * - *value* - integer value corresponding to enum value *name*;
2123 * - 0, on success;
2124 * - <0, on error.
2133 if (btf->nr_types == 0) in btf__add_enum_value()
2134 return libbpf_err(-EINVAL); in btf__add_enum_value()
2137 return libbpf_err(-EINVAL); in btf__add_enum_value()
2139 /* non-empty name */ in btf__add_enum_value()
2141 return libbpf_err(-EINVAL); in btf__add_enum_value()
2143 return libbpf_err(-E2BIG); in btf__add_enum_value()
2147 return libbpf_err(-ENOMEM); in btf__add_enum_value()
2152 return libbpf_err(-ENOMEM); in btf__add_enum_value()
2158 v->name_off = name_off; in btf__add_enum_value()
2159 v->val = value; in btf__add_enum_value()
2167 t->info = btf_type_info(btf_kind(t), btf_vlen(t), true); in btf__add_enum_value()
2169 btf->hdr->type_len += sz; in btf__add_enum_value()
2170 btf->hdr->str_off += sz; in btf__add_enum_value()
2176 * - *name* - name of the enum, can be NULL or empty for anonymous enums;
2177 * - *byte_sz* - size of the enum, in bytes.
2178 * - *is_signed* - whether the enum values are signed or not;
2185 * - >0, type ID of newly added BTF type;
2186 * - <0, on error.
2197 * - *name* - name of the enumerator value, can't be NULL or empty;
2198 * - *value* - integer value corresponding to enum value *name*;
2200 * - 0, on success;
2201 * - <0, on error.
2210 if (btf->nr_types == 0) in btf__add_enum64_value()
2211 return libbpf_err(-EINVAL); in btf__add_enum64_value()
2214 return libbpf_err(-EINVAL); in btf__add_enum64_value()
2216 /* non-empty name */ in btf__add_enum64_value()
2218 return libbpf_err(-EINVAL); in btf__add_enum64_value()
2222 return libbpf_err(-ENOMEM); in btf__add_enum64_value()
2227 return libbpf_err(-ENOMEM); in btf__add_enum64_value()
2233 v->name_off = name_off; in btf__add_enum64_value()
2234 v->val_lo32 = (__u32)value; in btf__add_enum64_value()
2235 v->val_hi32 = value >> 32; in btf__add_enum64_value()
2241 btf->hdr->type_len += sz; in btf__add_enum64_value()
2242 btf->hdr->str_off += sz; in btf__add_enum64_value()
2248 * - *name*, non-empty/non-NULL name;
2249 * - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT,
2252 * - >0, type ID of newly added BTF type;
2253 * - <0, on error.
2258 return libbpf_err(-EINVAL); in btf__add_fwd()
2270 t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION); in btf__add_fwd()
2275 * values; we also assume a standard 4-byte size for it in btf__add_fwd()
2279 return libbpf_err(-EINVAL); in btf__add_fwd()
2285 * - *name*, non-empty/non-NULL name;
2286 * - *ref_type_id* - referenced type ID, it might not exist yet;
2288 * - >0, type ID of newly added BTF type;
2289 * - <0, on error.
2294 return libbpf_err(-EINVAL); in btf__add_typedef()
2301 * - *ref_type_id* - referenced type ID, it might not exist yet;
2303 * - >0, type ID of newly added BTF type;
2304 * - <0, on error.
2313 * - *ref_type_id* - referenced type ID, it might not exist yet;
2315 * - >0, type ID of newly added BTF type;
2316 * - <0, on error.
2325 * - *ref_type_id* - referenced type ID, it might not exist yet;
2327 * - >0, type ID of newly added BTF type;
2328 * - <0, on error.
2337 * - *value*, non-empty/non-NULL tag value;
2338 * - *ref_type_id* - referenced type ID, it might not exist yet;
2340 * - >0, type ID of newly added BTF type;
2341 * - <0, on error.
2346 return libbpf_err(-EINVAL); in btf__add_type_tag()
2353 * - *name*, non-empty/non-NULL name;
2354 * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet;
2356 * - >0, type ID of newly added BTF type;
2357 * - <0, on error.
2365 return libbpf_err(-EINVAL); in btf__add_func()
2368 return libbpf_err(-EINVAL); in btf__add_func()
2374 t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0); in btf__add_func()
2381 * - *ret_type_id* - type ID for return result of a function.
2388 * - >0, type ID of newly added BTF type;
2389 * - <0, on error.
2397 return libbpf_err(-EINVAL); in btf__add_func_proto()
2400 return libbpf_err(-ENOMEM); in btf__add_func_proto()
2405 return libbpf_err(-ENOMEM); in btf__add_func_proto()
2410 t->name_off = 0; in btf__add_func_proto()
2411 t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0); in btf__add_func_proto()
2412 t->type = ret_type_id; in btf__add_func_proto()
2419 * - *name* - parameter name, can be NULL or empty;
2420 * - *type_id* - type ID describing the type of the parameter.
2422 * - 0, on success;
2423 * - <0, on error.
2432 return libbpf_err(-EINVAL); in btf__add_func_param()
2435 if (btf->nr_types == 0) in btf__add_func_param()
2436 return libbpf_err(-EINVAL); in btf__add_func_param()
2439 return libbpf_err(-EINVAL); in btf__add_func_param()
2443 return libbpf_err(-ENOMEM); in btf__add_func_param()
2448 return libbpf_err(-ENOMEM); in btf__add_func_param()
2456 p->name_off = name_off; in btf__add_func_param()
2457 p->type = type_id; in btf__add_func_param()
2463 btf->hdr->type_len += sz; in btf__add_func_param()
2464 btf->hdr->str_off += sz; in btf__add_func_param()
2470 * - *name* - non-empty/non-NULL name;
2471 * - *linkage* - variable linkage, one of BTF_VAR_STATIC,
2473 * - *type_id* - type ID of the type describing the type of the variable.
2475 * - >0, type ID of newly added BTF type;
2476 * - <0, on error.
2484 /* non-empty name */ in btf__add_var()
2486 return libbpf_err(-EINVAL); in btf__add_var()
2489 return libbpf_err(-EINVAL); in btf__add_var()
2491 return libbpf_err(-EINVAL); in btf__add_var()
2495 return libbpf_err(-ENOMEM); in btf__add_var()
2500 return libbpf_err(-ENOMEM); in btf__add_var()
2506 t->name_off = name_off; in btf__add_var()
2507 t->info = btf_type_info(BTF_KIND_VAR, 0, 0); in btf__add_var()
2508 t->type = type_id; in btf__add_var()
2511 v->linkage = linkage; in btf__add_var()
2518 * - *name* - non-empty/non-NULL name;
2519 * - *byte_sz* - data section size, in bytes.
2525 * - >0, type ID of newly added BTF type;
2526 * - <0, on error.
2533 /* non-empty name */ in btf__add_datasec()
2535 return libbpf_err(-EINVAL); in btf__add_datasec()
2538 return libbpf_err(-ENOMEM); in btf__add_datasec()
2543 return libbpf_err(-ENOMEM); in btf__add_datasec()
2550 t->name_off = name_off; in btf__add_datasec()
2551 t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0); in btf__add_datasec()
2552 t->size = byte_sz; in btf__add_datasec()
2559 * - *var_type_id* - type ID, describing type of the variable;
2560 * - *offset* - variable offset within data section, in bytes;
2561 * - *byte_sz* - variable size, in bytes.
2564 * - 0, on success;
2565 * - <0, on error.
2574 if (btf->nr_types == 0) in btf__add_datasec_var_info()
2575 return libbpf_err(-EINVAL); in btf__add_datasec_var_info()
2578 return libbpf_err(-EINVAL); in btf__add_datasec_var_info()
2581 return libbpf_err(-EINVAL); in btf__add_datasec_var_info()
2585 return libbpf_err(-ENOMEM); in btf__add_datasec_var_info()
2590 return libbpf_err(-ENOMEM); in btf__add_datasec_var_info()
2592 v->type = var_type_id; in btf__add_datasec_var_info()
2593 v->offset = offset; in btf__add_datasec_var_info()
2594 v->size = byte_sz; in btf__add_datasec_var_info()
2600 btf->hdr->type_len += sz; in btf__add_datasec_var_info()
2601 btf->hdr->str_off += sz; in btf__add_datasec_var_info()
2607 * - *value* - non-empty/non-NULL string;
2608 * - *ref_type_id* - referenced type ID, it might not exist yet;
2609 * - *component_idx* - -1 for tagging reference type, otherwise struct/union
2612 * - >0, type ID of newly added BTF type;
2613 * - <0, on error.
2621 if (!value || !value[0] || component_idx < -1) in btf__add_decl_tag()
2622 return libbpf_err(-EINVAL); in btf__add_decl_tag()
2625 return libbpf_err(-EINVAL); in btf__add_decl_tag()
2628 return libbpf_err(-ENOMEM); in btf__add_decl_tag()
2633 return libbpf_err(-ENOMEM); in btf__add_decl_tag()
2639 t->name_off = value_off; in btf__add_decl_tag()
2640 t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, false); in btf__add_decl_tag()
2641 t->type = ref_type_id; in btf__add_decl_tag()
2642 btf_decl_tag(t)->component_idx = component_idx; in btf__add_decl_tag()
2665 if (ext_sec->len == 0) in btf_ext_setup_info()
2668 if (ext_sec->off & 0x03) { in btf_ext_setup_info()
2670 ext_sec->desc); in btf_ext_setup_info()
2671 return -EINVAL; in btf_ext_setup_info()
2674 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; in btf_ext_setup_info()
2675 info_left = ext_sec->len; in btf_ext_setup_info()
2677 if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) { in btf_ext_setup_info()
2679 ext_sec->desc, ext_sec->off, ext_sec->len); in btf_ext_setup_info()
2680 return -EINVAL; in btf_ext_setup_info()
2685 pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc); in btf_ext_setup_info()
2686 return -EINVAL; in btf_ext_setup_info()
2691 if (record_size < ext_sec->min_rec_size || in btf_ext_setup_info()
2694 ext_sec->desc, record_size); in btf_ext_setup_info()
2695 return -EINVAL; in btf_ext_setup_info()
2699 info_left -= sizeof(__u32); in btf_ext_setup_info()
2703 pr_debug("%s section in .BTF.ext has no records", ext_sec->desc); in btf_ext_setup_info()
2704 return -EINVAL; in btf_ext_setup_info()
2714 ext_sec->desc); in btf_ext_setup_info()
2715 return -EINVAL; in btf_ext_setup_info()
2718 num_records = sinfo->num_info; in btf_ext_setup_info()
2721 ext_sec->desc); in btf_ext_setup_info()
2722 return -EINVAL; in btf_ext_setup_info()
2728 ext_sec->desc); in btf_ext_setup_info()
2729 return -EINVAL; in btf_ext_setup_info()
2732 info_left -= total_record_size; in btf_ext_setup_info()
2737 ext_info = ext_sec->ext_info; in btf_ext_setup_info()
2738 ext_info->len = ext_sec->len - sizeof(__u32); in btf_ext_setup_info()
2739 ext_info->rec_size = record_size; in btf_ext_setup_info()
2740 ext_info->info = info + sizeof(__u32); in btf_ext_setup_info()
2741 ext_info->sec_cnt = sec_cnt; in btf_ext_setup_info()
2749 .off = btf_ext->hdr->func_info_off, in btf_ext_setup_func_info()
2750 .len = btf_ext->hdr->func_info_len, in btf_ext_setup_func_info()
2752 .ext_info = &btf_ext->func_info, in btf_ext_setup_func_info()
2762 .off = btf_ext->hdr->line_info_off, in btf_ext_setup_line_info()
2763 .len = btf_ext->hdr->line_info_len, in btf_ext_setup_line_info()
2765 .ext_info = &btf_ext->line_info, in btf_ext_setup_line_info()
2775 .off = btf_ext->hdr->core_relo_off, in btf_ext_setup_core_relos()
2776 .len = btf_ext->hdr->core_relo_len, in btf_ext_setup_core_relos()
2778 .ext_info = &btf_ext->core_relo_info, in btf_ext_setup_core_relos()
2790 data_size < hdr->hdr_len) { in btf_ext_parse_hdr()
2792 return -EINVAL; in btf_ext_parse_hdr()
2795 if (hdr->magic == bswap_16(BTF_MAGIC)) { in btf_ext_parse_hdr()
2796 pr_warn("BTF.ext in non-native endianness is not supported\n"); in btf_ext_parse_hdr()
2797 return -ENOTSUP; in btf_ext_parse_hdr()
2798 } else if (hdr->magic != BTF_MAGIC) { in btf_ext_parse_hdr()
2799 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic); in btf_ext_parse_hdr()
2800 return -EINVAL; in btf_ext_parse_hdr()
2803 if (hdr->version != BTF_VERSION) { in btf_ext_parse_hdr()
2804 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version); in btf_ext_parse_hdr()
2805 return -ENOTSUP; in btf_ext_parse_hdr()
2808 if (hdr->flags) { in btf_ext_parse_hdr()
2809 pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags); in btf_ext_parse_hdr()
2810 return -ENOTSUP; in btf_ext_parse_hdr()
2813 if (data_size == hdr->hdr_len) { in btf_ext_parse_hdr()
2815 return -EINVAL; in btf_ext_parse_hdr()
2825 free(btf_ext->func_info.sec_idxs); in btf_ext__free()
2826 free(btf_ext->line_info.sec_idxs); in btf_ext__free()
2827 free(btf_ext->core_relo_info.sec_idxs); in btf_ext__free()
2828 free(btf_ext->data); in btf_ext__free()
2839 return libbpf_err_ptr(-ENOMEM); in btf_ext__new()
2841 btf_ext->data_size = size; in btf_ext__new()
2842 btf_ext->data = malloc(size); in btf_ext__new()
2843 if (!btf_ext->data) { in btf_ext__new()
2844 err = -ENOMEM; in btf_ext__new()
2847 memcpy(btf_ext->data, data, size); in btf_ext__new()
2849 err = btf_ext_parse_hdr(btf_ext->data, size); in btf_ext__new()
2853 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, line_info_len)) { in btf_ext__new()
2854 err = -EINVAL; in btf_ext__new()
2866 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) in btf_ext__new()
2884 *size = btf_ext->data_size; in btf_ext__get_raw_data()
2885 return btf_ext->data; in btf_ext__get_raw_data()
2906 * memory in-place with deduplicated types and strings without any loss of
2916 * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html
2921 * BTF type information is typically emitted either as a result of conversion
2977 * types, in general, can form graphs containing cycles, not just DAGs.
3024 * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that
3028 * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash
3045 return libbpf_err(-EINVAL); in btf__dedup()
3050 return libbpf_err(-EINVAL); in btf__dedup()
3054 err = -ENOMEM; in btf__dedup()
3104 #define BTF_UNPROCESSED_ID ((__u32)-1)
3105 #define BTF_IN_PROGRESS_ID ((__u32)-2)
3108 /* .BTF section to be deduped in-place */
3149 hashmap__for_each_key_entry(d->dedup_table, node, hash)
3153 return hashmap__append(d->dedup_table, hash, type_id); in btf_dedup_table_add()
3159 if (d->hypot_cnt == d->hypot_cap) { in btf_dedup_hypot_map_add()
3162 d->hypot_cap += max((size_t)16, d->hypot_cap / 2); in btf_dedup_hypot_map_add()
3163 new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32)); in btf_dedup_hypot_map_add()
3165 return -ENOMEM; in btf_dedup_hypot_map_add()
3166 d->hypot_list = new_list; in btf_dedup_hypot_map_add()
3168 d->hypot_list[d->hypot_cnt++] = from_id; in btf_dedup_hypot_map_add()
3169 d->hypot_map[from_id] = to_id; in btf_dedup_hypot_map_add()
3177 for (i = 0; i < d->hypot_cnt; i++) in btf_dedup_clear_hypot_map()
3178 d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID; in btf_dedup_clear_hypot_map()
3179 d->hypot_cnt = 0; in btf_dedup_clear_hypot_map()
3180 d->hypot_adjust_canon = false; in btf_dedup_clear_hypot_map()
3185 hashmap__free(d->dedup_table); in btf_dedup_free()
3186 d->dedup_table = NULL; in btf_dedup_free()
3188 free(d->map); in btf_dedup_free()
3189 d->map = NULL; in btf_dedup_free()
3191 free(d->hypot_map); in btf_dedup_free()
3192 d->hypot_map = NULL; in btf_dedup_free()
3194 free(d->hypot_list); in btf_dedup_free()
3195 d->hypot_list = NULL; in btf_dedup_free()
3222 return ERR_PTR(-ENOMEM); in btf_dedup_new()
3227 d->btf = btf; in btf_dedup_new()
3228 d->btf_ext = OPTS_GET(opts, btf_ext, NULL); in btf_dedup_new()
3230 d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL); in btf_dedup_new()
3231 if (IS_ERR(d->dedup_table)) { in btf_dedup_new()
3232 err = PTR_ERR(d->dedup_table); in btf_dedup_new()
3233 d->dedup_table = NULL; in btf_dedup_new()
3238 d->map = malloc(sizeof(__u32) * type_cnt); in btf_dedup_new()
3239 if (!d->map) { in btf_dedup_new()
3240 err = -ENOMEM; in btf_dedup_new()
3244 d->map[0] = 0; in btf_dedup_new()
3246 struct btf_type *t = btf_type_by_id(d->btf, i); in btf_dedup_new()
3248 /* VAR and DATASEC are never deduped and are self-canonical */ in btf_dedup_new()
3250 d->map[i] = i; in btf_dedup_new()
3252 d->map[i] = BTF_UNPROCESSED_ID; in btf_dedup_new()
3255 d->hypot_map = malloc(sizeof(__u32) * type_cnt); in btf_dedup_new()
3256 if (!d->hypot_map) { in btf_dedup_new()
3257 err = -ENOMEM; in btf_dedup_new()
3261 d->hypot_map[i] = BTF_UNPROCESSED_ID; in btf_dedup_new()
3280 for (i = 0; i < d->btf->nr_types; i++) { in btf_for_each_str_off()
3281 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); in btf_for_each_str_off()
3288 if (!d->btf_ext) in btf_for_each_str_off()
3291 r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx); in btf_for_each_str_off()
3306 if (str_off == 0 || str_off < d->btf->start_str_off) in strs_dedup_remap_str_off()
3309 s = btf__str_by_offset(d->btf, str_off); in strs_dedup_remap_str_off()
3310 if (d->btf->base_btf) { in strs_dedup_remap_str_off()
3311 err = btf__find_str(d->btf->base_btf, s); in strs_dedup_remap_str_off()
3316 if (err != -ENOENT) in strs_dedup_remap_str_off()
3320 off = strset__add_str(d->strs_set, s); in strs_dedup_remap_str_off()
3324 *str_off_ptr = d->btf->start_str_off + off; in strs_dedup_remap_str_off()
3343 if (d->btf->strs_deduped) in btf_dedup_strings()
3346 d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0); in btf_dedup_strings()
3347 if (IS_ERR(d->strs_set)) { in btf_dedup_strings()
3348 err = PTR_ERR(d->strs_set); in btf_dedup_strings()
3352 if (!d->btf->base_btf) { in btf_dedup_strings()
3356 err = strset__add_str(d->strs_set, ""); in btf_dedup_strings()
3367 strset__free(d->btf->strs_set); in btf_dedup_strings()
3368 d->btf->hdr->str_len = strset__data_size(d->strs_set); in btf_dedup_strings()
3369 d->btf->strs_set = d->strs_set; in btf_dedup_strings()
3370 d->strs_set = NULL; in btf_dedup_strings()
3371 d->btf->strs_deduped = true; in btf_dedup_strings()
3375 strset__free(d->strs_set); in btf_dedup_strings()
3376 d->strs_set = NULL; in btf_dedup_strings()
3385 h = hash_combine(0, t->name_off); in btf_hash_common()
3386 h = hash_combine(h, t->info); in btf_hash_common()
3387 h = hash_combine(h, t->size); in btf_hash_common()
3393 return t1->name_off == t2->name_off && in btf_equal_common()
3394 t1->info == t2->info && in btf_equal_common()
3395 t1->size == t2->size; in btf_equal_common()
3427 h = hash_combine(0, t->name_off); in btf_hash_enum()
3441 if (m1->name_off != m2->name_off || m1->val != m2->val) in btf_equal_enum_members()
3459 if (m1->name_off != m2->name_off || m1->val_lo32 != m2->val_lo32 || in btf_equal_enum64_members()
3460 m1->val_hi32 != m2->val_hi32) in btf_equal_enum64_members()
3491 * - skip comparing vlen because it is zero for forward declarations; in btf_compat_enum()
3492 * - skip comparing size to allow enum forward declarations in btf_compat_enum()
3494 * - skip comparing kind for the same reason. in btf_compat_enum()
3496 return t1->name_off == t2->name_off && in btf_compat_enum()
3513 h = hash_combine(h, member->name_off); in btf_hash_struct()
3514 h = hash_combine(h, member->offset); in btf_hash_struct()
3539 if (m1->name_off != m2->name_off || m1->offset != m2->offset) in btf_shallow_equal_struct()
3557 h = hash_combine(h, info->type); in btf_hash_array()
3558 h = hash_combine(h, info->index_type); in btf_hash_array()
3559 h = hash_combine(h, info->nelems); in btf_hash_array()
3579 return info1->type == info2->type && in btf_equal_array()
3580 info1->index_type == info2->index_type && in btf_equal_array()
3581 info1->nelems == info2->nelems; in btf_equal_array()
3594 return btf_array(t1)->nelems == btf_array(t2)->nelems; in btf_compat_array()
3610 h = hash_combine(h, member->name_off); in btf_hash_fnproto()
3611 h = hash_combine(h, member->type); in btf_hash_fnproto()
3637 if (m1->name_off != m2->name_off || m1->type != m2->type) in btf_equal_fnproto()
3657 if (t1->name_off != t2->name_off || t1->info != t2->info) in btf_compat_fnproto()
3664 if (m1->name_off != m2->name_off) in btf_compat_fnproto()
3682 if (!d->btf->base_btf) in btf_dedup_prep()
3685 for (type_id = 1; type_id < d->btf->start_id; type_id++) { in btf_dedup_prep()
3686 t = btf_type_by_id(d->btf, type_id); in btf_dedup_prep()
3688 /* all base BTF types are self-canonical by definition */ in btf_dedup_prep()
3689 d->map[type_id] = type_id; in btf_dedup_prep()
3727 return -EINVAL; in btf_dedup_prep()
3730 return -ENOMEM; in btf_dedup_prep()
3740 * canonical and is added into `btf_dedup->dedup_table` as another candidate.
3744 struct btf_type *t = btf_type_by_id(d->btf, type_id); in btf_dedup_prim_type()
3772 cand_id = hash_entry->value; in btf_dedup_prim_type()
3773 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3785 cand_id = hash_entry->value; in btf_dedup_prim_type()
3786 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3798 d->map[cand_id] = type_id; in btf_dedup_prim_type()
3807 cand_id = hash_entry->value; in btf_dedup_prim_type()
3808 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3817 return -EINVAL; in btf_dedup_prim_type()
3820 d->map[type_id] = new_id; in btf_dedup_prim_type()
3822 return -ENOMEM; in btf_dedup_prim_type()
3831 for (i = 0; i < d->btf->nr_types; i++) { in btf_dedup_prim_types()
3832 err = btf_dedup_prim_type(d, d->btf->start_id + i); in btf_dedup_prim_types()
3844 return d->map[type_id] <= BTF_MAX_NR_TYPES; in is_type_mapped()
3854 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) in resolve_type_id()
3855 type_id = d->map[type_id]; in resolve_type_id()
3867 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) in resolve_fwd_id()
3870 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) in resolve_fwd_id()
3871 type_id = d->map[type_id]; in resolve_fwd_id()
3873 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) in resolve_fwd_id()
3890 t1 = btf_type_by_id(d->btf, id1); in btf_dedup_identical_arrays()
3891 t2 = btf_type_by_id(d->btf, id2); in btf_dedup_identical_arrays()
3905 t1 = btf_type_by_id(d->btf, id1); in btf_dedup_identical_structs()
3906 t2 = btf_type_by_id(d->btf, id2); in btf_dedup_identical_structs()
3917 if (m1->type != m2->type && in btf_dedup_identical_structs()
3918 !btf_dedup_identical_arrays(d, m1->type, m2->type) && in btf_dedup_identical_structs()
3919 !btf_dedup_identical_structs(d, m1->type, m2->type)) in btf_dedup_identical_structs()
3933 * - >0, if type graphs are equivalent;
3934 * - 0, if not equivalent;
3935 * - <0, on error.
3937 * Algorithm performs side-by-side DFS traversal of both type graphs and checks
3942 * a `cand_id` is recored in hypothetical mapping (`btf_dedup->hypot_map`).
3949 * - `canon_id` is mapped to exactly the same type as `cand_id`. This will
3950 * happen when type graphs have cycles. In this case we assume those two
3952 * - `canon_id` is mapped to different type. This is contradiction in our
3966 * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently.
3991 * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION
3995 * - Both types in canonical and candidate graphs are FWDs. If they are
3999 * - Both types in canonical and candidate graphs are concrete STRUCT/UNION,
4001 * - Type in canonical graph is FWD, while type in candidate is concrete
4010 * - Type in canonical graph is concrete STRUCT/UNION, while type in candidate
4015 * turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from
4034 hypot_type_id = d->hypot_map[canon_id]; in btf_dedup_is_equiv()
4042 * types sub-graph has a consistent and deduped-by-compiler in btf_dedup_is_equiv()
4053 * complicated and requires a many-to-many equivalence mapping. in btf_dedup_is_equiv()
4061 return -ENOMEM; in btf_dedup_is_equiv()
4063 cand_type = btf_type_by_id(d->btf, cand_id); in btf_dedup_is_equiv()
4064 canon_type = btf_type_by_id(d->btf, canon_id); in btf_dedup_is_equiv()
4068 if (cand_type->name_off != canon_type->name_off) in btf_dedup_is_equiv()
4071 /* FWD <--> STRUCT/UNION equivalence check, if enabled */ in btf_dedup_is_equiv()
4084 if (fwd_kind == real_kind && canon_id < d->btf->start_id) in btf_dedup_is_equiv()
4085 d->hypot_adjust_canon = true; in btf_dedup_is_equiv()
4112 if (cand_type->info != canon_type->info) in btf_dedup_is_equiv()
4114 return btf_dedup_is_equiv(d, cand_type->type, canon_type->type); in btf_dedup_is_equiv()
4123 eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type); in btf_dedup_is_equiv()
4126 return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type); in btf_dedup_is_equiv()
4140 eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); in btf_dedup_is_equiv()
4156 eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type); in btf_dedup_is_equiv()
4163 eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type); in btf_dedup_is_equiv()
4173 return -EINVAL; in btf_dedup_is_equiv()
4183 * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional:
4211 for (i = 0; i < d->hypot_cnt; i++) { in btf_dedup_merge_hypot_map()
4212 canon_type_id = d->hypot_list[i]; in btf_dedup_merge_hypot_map()
4213 targ_type_id = d->hypot_map[canon_type_id]; in btf_dedup_merge_hypot_map()
4216 t_kind = btf_kind(btf__type_by_id(d->btf, t_id)); in btf_dedup_merge_hypot_map()
4217 c_kind = btf_kind(btf__type_by_id(d->btf, c_id)); in btf_dedup_merge_hypot_map()
4222 * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because in btf_dedup_merge_hypot_map()
4239 d->map[c_id] = t_id; in btf_dedup_merge_hypot_map()
4246 if (d->hypot_adjust_canon) in btf_dedup_merge_hypot_map()
4250 d->map[t_id] = c_id; in btf_dedup_merge_hypot_map()
4262 d->map[t_id] = c_id; in btf_dedup_merge_hypot_map()
4279 * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence
4280 * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to
4286 * as canonical for itself and is added into btf_dedup->dedup_table hash map
4299 if (d->map[type_id] <= BTF_MAX_NR_TYPES) in btf_dedup_struct_type()
4302 t = btf_type_by_id(d->btf, type_id); in btf_dedup_struct_type()
4310 __u32 cand_id = hash_entry->value; in btf_dedup_struct_type()
4320 * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because in btf_dedup_struct_type()
4323 cand_type = btf_type_by_id(d->btf, cand_id); in btf_dedup_struct_type()
4334 if (d->hypot_adjust_canon) /* not really equivalent */ in btf_dedup_struct_type()
4340 d->map[type_id] = new_id; in btf_dedup_struct_type()
4342 return -ENOMEM; in btf_dedup_struct_type()
4351 for (i = 0; i < d->btf->nr_types; i++) { in btf_dedup_struct_types()
4352 err = btf_dedup_struct_type(d, d->btf->start_id + i); in btf_dedup_struct_types()
4370 * one by one. There is no danger of encountering cycles because in C type
4381 * btf_dedup->dedup_table as another possible canonical representative.
4392 if (d->map[type_id] == BTF_IN_PROGRESS_ID) in btf_dedup_ref_type()
4393 return -ELOOP; in btf_dedup_ref_type()
4394 if (d->map[type_id] <= BTF_MAX_NR_TYPES) in btf_dedup_ref_type()
4397 t = btf_type_by_id(d->btf, type_id); in btf_dedup_ref_type()
4398 d->map[type_id] = BTF_IN_PROGRESS_ID; in btf_dedup_ref_type()
4408 ref_type_id = btf_dedup_ref_type(d, t->type); in btf_dedup_ref_type()
4411 t->type = ref_type_id; in btf_dedup_ref_type()
4415 cand_id = hash_entry->value; in btf_dedup_ref_type()
4416 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4425 ref_type_id = btf_dedup_ref_type(d, t->type); in btf_dedup_ref_type()
4428 t->type = ref_type_id; in btf_dedup_ref_type()
4432 cand_id = hash_entry->value; in btf_dedup_ref_type()
4433 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4444 ref_type_id = btf_dedup_ref_type(d, info->type); in btf_dedup_ref_type()
4447 info->type = ref_type_id; in btf_dedup_ref_type()
4449 ref_type_id = btf_dedup_ref_type(d, info->index_type); in btf_dedup_ref_type()
4452 info->index_type = ref_type_id; in btf_dedup_ref_type()
4456 cand_id = hash_entry->value; in btf_dedup_ref_type()
4457 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4471 ref_type_id = btf_dedup_ref_type(d, t->type); in btf_dedup_ref_type()
4474 t->type = ref_type_id; in btf_dedup_ref_type()
4479 ref_type_id = btf_dedup_ref_type(d, param->type); in btf_dedup_ref_type()
4482 param->type = ref_type_id; in btf_dedup_ref_type()
4488 cand_id = hash_entry->value; in btf_dedup_ref_type()
4489 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4499 return -EINVAL; in btf_dedup_ref_type()
4502 d->map[type_id] = new_id; in btf_dedup_ref_type()
4504 return -ENOMEM; in btf_dedup_ref_type()
4513 for (i = 0; i < d->btf->nr_types; i++) { in btf_dedup_ref_types()
4514 err = btf_dedup_ref_type(d, d->btf->start_id + i); in btf_dedup_ref_types()
4518 /* we won't need d->dedup_table anymore */ in btf_dedup_ref_types()
4519 hashmap__free(d->dedup_table); in btf_dedup_ref_types()
4520 d->dedup_table = NULL; in btf_dedup_ref_types()
4531 __u32 nr_types = btf__type_cnt(d->btf); in btf_dedup_fill_unique_names_map()
4542 t = btf_type_by_id(d->btf, type_id); in btf_dedup_fill_unique_names_map()
4548 /* Skip non-canonical types */ in btf_dedup_fill_unique_names_map()
4549 if (type_id != d->map[type_id]) in btf_dedup_fill_unique_names_map()
4552 err = hashmap__add(names_map, t->name_off, type_id); in btf_dedup_fill_unique_names_map()
4553 if (err == -EEXIST) in btf_dedup_fill_unique_names_map()
4554 err = hashmap__set(names_map, t->name_off, 0, NULL, NULL); in btf_dedup_fill_unique_names_map()
4565 struct btf_type *t = btf_type_by_id(d->btf, type_id); in btf_dedup_resolve_fwd()
4575 if (type_id != d->map[type_id]) in btf_dedup_resolve_fwd()
4578 if (!hashmap__find(names_map, t->name_off, &cand_id)) in btf_dedup_resolve_fwd()
4585 cand_t = btf_type_by_id(d->btf, cand_id); in btf_dedup_resolve_fwd()
4591 d->map[type_id] = cand_id; in btf_dedup_resolve_fwd()
4641 for (i = 0; i < d->btf->nr_types; i++) { in btf_dedup_resolve_fwds()
4642 err = btf_dedup_resolve_fwd(d, names_map, d->btf->start_id + i); in btf_dedup_resolve_fwds()
4658 * duplicates. During compaction btf_dedup->hypot_map array is reused to store
4666 __u32 next_type_id = d->btf->start_id; in btf_dedup_compact_types()
4672 d->hypot_map[0] = 0; in btf_dedup_compact_types()
4674 for (id = 1; id < d->btf->start_id; id++) in btf_dedup_compact_types()
4675 d->hypot_map[id] = id; in btf_dedup_compact_types()
4676 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) in btf_dedup_compact_types()
4677 d->hypot_map[id] = BTF_UNPROCESSED_ID; in btf_dedup_compact_types()
4679 p = d->btf->types_data; in btf_dedup_compact_types()
4681 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) { in btf_dedup_compact_types()
4682 if (d->map[id] != id) in btf_dedup_compact_types()
4685 t = btf__type_by_id(d->btf, id); in btf_dedup_compact_types()
4691 d->hypot_map[id] = next_type_id; in btf_dedup_compact_types()
4692 d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data; in btf_dedup_compact_types()
4698 d->btf->nr_types = next_type_id - d->btf->start_id; in btf_dedup_compact_types()
4699 d->btf->type_offs_cap = d->btf->nr_types; in btf_dedup_compact_types()
4700 d->btf->hdr->type_len = p - d->btf->types_data; in btf_dedup_compact_types()
4701 new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap, in btf_dedup_compact_types()
4703 if (d->btf->type_offs_cap && !new_offs) in btf_dedup_compact_types()
4704 return -ENOMEM; in btf_dedup_compact_types()
4705 d->btf->type_offs = new_offs; in btf_dedup_compact_types()
4706 d->btf->hdr->str_off = d->btf->hdr->type_len; in btf_dedup_compact_types()
4707 d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len; in btf_dedup_compact_types()
4714 * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map,
4723 new_type_id = d->hypot_map[resolved_type_id]; in btf_dedup_remap_type_id()
4725 return -EINVAL; in btf_dedup_remap_type_id()
4736 * deduped type ID is stored in btf_dedup->hypot_map and is populated during
4745 for (i = 0; i < d->btf->nr_types; i++) { in btf_dedup_remap_types()
4746 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); in btf_dedup_remap_types()
4753 if (!d->btf_ext) in btf_dedup_remap_types()
4756 r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d); in btf_dedup_remap_types()
4764 * Probe few well-known locations for vmlinux kernel image and try to load BTF
4773 "/boot/vmlinux-%1$s", in btf__load_vmlinux_btf()
4774 "/lib/modules/%1$s/vmlinux-%1$s", in btf__load_vmlinux_btf()
4777 "/usr/lib/debug/boot/vmlinux-%1$s", in btf__load_vmlinux_btf()
4778 "/usr/lib/debug/boot/vmlinux-%1$s.debug", in btf__load_vmlinux_btf()
4804 return libbpf_err_ptr(-ESRCH); in btf__load_vmlinux_btf()
4838 return visit(&t->type, ctx); in btf_type_visit_type_ids()
4843 err = visit(&a->type, ctx); in btf_type_visit_type_ids()
4844 err = err ?: visit(&a->index_type, ctx); in btf_type_visit_type_ids()
4853 err = visit(&m->type, ctx); in btf_type_visit_type_ids()
4863 err = visit(&t->type, ctx); in btf_type_visit_type_ids()
4867 err = visit(&m->type, ctx); in btf_type_visit_type_ids()
4878 err = visit(&m->type, ctx); in btf_type_visit_type_ids()
4886 return -EINVAL; in btf_type_visit_type_ids()
4894 err = visit(&t->name_off, ctx); in btf_type_visit_str_offs()
4904 err = visit(&m->name_off, ctx); in btf_type_visit_str_offs()
4914 err = visit(&m->name_off, ctx); in btf_type_visit_str_offs()
4924 err = visit(&m->name_off, ctx); in btf_type_visit_str_offs()
4934 err = visit(&m->name_off, ctx); in btf_type_visit_str_offs()
4953 seg = &btf_ext->func_info; in btf_ext_visit_type_ids()
4958 err = visit(&rec->type_id, ctx); in btf_ext_visit_type_ids()
4964 seg = &btf_ext->core_relo_info; in btf_ext_visit_type_ids()
4969 err = visit(&rec->type_id, ctx); in btf_ext_visit_type_ids()
4984 seg = &btf_ext->func_info; in btf_ext_visit_str_offs()
4986 err = visit(&sec->sec_name_off, ctx); in btf_ext_visit_str_offs()
4991 seg = &btf_ext->line_info; in btf_ext_visit_str_offs()
4995 err = visit(&sec->sec_name_off, ctx); in btf_ext_visit_str_offs()
5000 err = visit(&rec->file_name_off, ctx); in btf_ext_visit_str_offs()
5003 err = visit(&rec->line_off, ctx); in btf_ext_visit_str_offs()
5009 seg = &btf_ext->core_relo_info; in btf_ext_visit_str_offs()
5013 err = visit(&sec->sec_name_off, ctx); in btf_ext_visit_str_offs()
5018 err = visit(&rec->access_str_off, ctx); in btf_ext_visit_str_offs()