Lines Matching +full:three +full:- +full:conversion +full:- +full:cycles
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
33 /* raw BTF data in non-native endianness */
45 * +--------------------------------+
47 * +--------------------------------+
51 * types_data-+ |
52 * strs_data------------+
56 * representation is broken up into three independently allocated
63 * +----------+ +---------+ +-----------+
65 * +----------+ +---------+ +-----------+
69 * types_data----+ |
70 * strs_data------------------+
72 * +----------+---------+-----------+
74 * raw_data----->+----------+---------+-----------+
79 size_t types_data_cap; /* used size stored in hdr->type_len */
87 size_t strs_data_cap; /* used size stored in hdr->str_len */
142 memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz); in btf_add_mem()
159 p = btf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); in btf_ensure_mem()
161 return -ENOMEM; in btf_ensure_mem()
170 p = btf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), in btf_add_type_idx_entry()
171 btf->nr_types + 1, BTF_MAX_NR_TYPES, 1); in btf_add_type_idx_entry()
173 return -ENOMEM; in btf_add_type_idx_entry()
181 h->magic = bswap_16(h->magic); in btf_bswap_hdr()
182 h->hdr_len = bswap_32(h->hdr_len); in btf_bswap_hdr()
183 h->type_off = bswap_32(h->type_off); in btf_bswap_hdr()
184 h->type_len = bswap_32(h->type_len); in btf_bswap_hdr()
185 h->str_off = bswap_32(h->str_off); in btf_bswap_hdr()
186 h->str_len = bswap_32(h->str_len); in btf_bswap_hdr()
191 struct btf_header *hdr = btf->hdr; in btf_parse_hdr()
194 if (btf->raw_size < sizeof(struct btf_header)) { in btf_parse_hdr()
196 return -EINVAL; in btf_parse_hdr()
199 if (hdr->magic == bswap_16(BTF_MAGIC)) { in btf_parse_hdr()
200 btf->swapped_endian = true; in btf_parse_hdr()
201 if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) { in btf_parse_hdr()
202 pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", in btf_parse_hdr()
203 bswap_32(hdr->hdr_len)); in btf_parse_hdr()
204 return -ENOTSUP; in btf_parse_hdr()
207 } else if (hdr->magic != BTF_MAGIC) { in btf_parse_hdr()
208 pr_debug("Invalid BTF magic: %x\n", hdr->magic); in btf_parse_hdr()
209 return -EINVAL; in btf_parse_hdr()
212 if (btf->raw_size < hdr->hdr_len) { in btf_parse_hdr()
214 hdr->hdr_len, btf->raw_size); in btf_parse_hdr()
215 return -EINVAL; in btf_parse_hdr()
218 meta_left = btf->raw_size - hdr->hdr_len; in btf_parse_hdr()
219 if (meta_left < (long long)hdr->str_off + hdr->str_len) { in btf_parse_hdr()
220 pr_debug("Invalid BTF total size: %u\n", btf->raw_size); in btf_parse_hdr()
221 return -EINVAL; in btf_parse_hdr()
224 if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) { in btf_parse_hdr()
226 hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len); in btf_parse_hdr()
227 return -EINVAL; in btf_parse_hdr()
230 if (hdr->type_off % 4) { in btf_parse_hdr()
232 return -EINVAL; in btf_parse_hdr()
240 const struct btf_header *hdr = btf->hdr; in btf_parse_str_sec()
241 const char *start = btf->strs_data; in btf_parse_str_sec()
242 const char *end = start + btf->hdr->str_len; in btf_parse_str_sec()
244 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || in btf_parse_str_sec()
245 start[0] || end[-1]) { in btf_parse_str_sec()
247 return -EINVAL; in btf_parse_str_sec()
284 return -EINVAL; in btf_type_size()
290 t->name_off = bswap_32(t->name_off); in btf_bswap_type_base()
291 t->info = bswap_32(t->info); in btf_bswap_type_base()
292 t->type = bswap_32(t->type); in btf_bswap_type_base()
319 e->name_off = bswap_32(e->name_off); in btf_bswap_type_rest()
320 e->val = bswap_32(e->val); in btf_bswap_type_rest()
325 a->type = bswap_32(a->type); in btf_bswap_type_rest()
326 a->index_type = bswap_32(a->index_type); in btf_bswap_type_rest()
327 a->nelems = bswap_32(a->nelems); in btf_bswap_type_rest()
332 m->name_off = bswap_32(m->name_off); in btf_bswap_type_rest()
333 m->type = bswap_32(m->type); in btf_bswap_type_rest()
334 m->offset = bswap_32(m->offset); in btf_bswap_type_rest()
339 p->name_off = bswap_32(p->name_off); in btf_bswap_type_rest()
340 p->type = bswap_32(p->type); in btf_bswap_type_rest()
344 btf_var(t)->linkage = bswap_32(btf_var(t)->linkage); in btf_bswap_type_rest()
348 v->type = bswap_32(v->type); in btf_bswap_type_rest()
349 v->offset = bswap_32(v->offset); in btf_bswap_type_rest()
350 v->size = bswap_32(v->size); in btf_bswap_type_rest()
355 return -EINVAL; in btf_bswap_type_rest()
361 struct btf_header *hdr = btf->hdr; in btf_parse_type_sec()
362 void *next_type = btf->types_data; in btf_parse_type_sec()
363 void *end_type = next_type + hdr->type_len; in btf_parse_type_sec()
377 if (btf->swapped_endian) in btf_parse_type_sec()
385 return -EINVAL; in btf_parse_type_sec()
388 if (btf->swapped_endian && btf_bswap_type_rest(next_type)) in btf_parse_type_sec()
389 return -EINVAL; in btf_parse_type_sec()
391 err = btf_add_type_idx_entry(btf, next_type - btf->types_data); in btf_parse_type_sec()
396 btf->nr_types++; in btf_parse_type_sec()
401 return -EINVAL; in btf_parse_type_sec()
409 return btf->nr_types; in btf__get_nr_types()
412 /* internal helper returning non-const pointer to a type */
418 return btf->types_data + btf->type_offs[type_id]; in btf_type_by_id()
423 if (type_id > btf->nr_types) in btf__type_by_id()
434 for (i = 1; i <= btf->nr_types; i++) { in determine_ptr_size()
439 name = btf__name_by_offset(btf, t->name_off); in determine_ptr_size()
445 if (t->size != 4 && t->size != 8) in determine_ptr_size()
447 return t->size; in determine_ptr_size()
451 return -1; in determine_ptr_size()
456 if (!btf->ptr_sz) in btf_ptr_sz()
457 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); in btf_ptr_sz()
458 return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz; in btf_ptr_sz()
471 if (!btf->ptr_sz) in btf__pointer_size()
472 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); in btf__pointer_size()
474 if (btf->ptr_sz < 0) in btf__pointer_size()
478 return btf->ptr_sz; in btf__pointer_size()
487 return -EINVAL; in btf__set_pointer_size()
488 btf->ptr_sz = ptr_sz; in btf__set_pointer_size()
506 return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; in btf__endianness()
508 return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; in btf__endianness()
514 return -EINVAL; in btf__set_endianness()
516 btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); in btf__set_endianness()
517 if (!btf->swapped_endian) { in btf__set_endianness()
518 free(btf->raw_data_swapped); in btf__set_endianness()
519 btf->raw_data_swapped = NULL; in btf__set_endianness()
541 __s64 size = -1; in btf__resolve_size()
553 size = t->size; in btf__resolve_size()
563 type_id = t->type; in btf__resolve_size()
567 if (nelems && array->nelems > UINT32_MAX / nelems) in btf__resolve_size()
568 return -E2BIG; in btf__resolve_size()
569 nelems *= array->nelems; in btf__resolve_size()
570 type_id = array->type; in btf__resolve_size()
573 return -EINVAL; in btf__resolve_size()
581 return -EINVAL; in btf__resolve_size()
583 return -E2BIG; in btf__resolve_size()
596 return min(btf_ptr_sz(btf), (size_t)t->size); in btf__align_of()
603 return btf__align_of(btf, t->type); in btf__align_of()
605 return btf__align_of(btf, btf_array(t)->type); in btf__align_of()
613 align = btf__align_of(btf, m->type); in btf__align_of()
622 (m->offset % (8 * align)) != 0) in btf__align_of()
629 if ((t->size % max_align) != 0) in btf__align_of()
649 type_id = t->type; in btf__resolve_type()
655 return -EINVAL; in btf__resolve_type()
667 for (i = 1; i <= btf->nr_types; i++) { in btf__find_by_name()
669 const char *name = btf__name_by_offset(btf, t->name_off); in btf__find_by_name()
675 return -ENOENT; in btf__find_by_name()
686 for (i = 1; i <= btf->nr_types; i++) { in btf__find_by_name_kind()
692 name = btf__name_by_offset(btf, t->name_off); in btf__find_by_name_kind()
697 return -ENOENT; in btf__find_by_name_kind()
702 return (void *)btf->hdr != btf->raw_data; in btf_is_modifiable()
710 if (btf->fd >= 0) in btf__free()
711 close(btf->fd); in btf__free()
715 * in-memory representation for header, types, and strings in btf__free()
720 free(btf->hdr); in btf__free()
721 free(btf->types_data); in btf__free()
722 free(btf->strs_data); in btf__free()
724 free(btf->raw_data); in btf__free()
725 free(btf->raw_data_swapped); in btf__free()
726 free(btf->type_offs); in btf__free()
736 return ERR_PTR(-ENOMEM); in btf__new_empty()
738 btf->fd = -1; in btf__new_empty()
739 btf->ptr_sz = sizeof(void *); in btf__new_empty()
740 btf->swapped_endian = false; in btf__new_empty()
743 btf->raw_size = sizeof(struct btf_header) + 1; in btf__new_empty()
744 btf->raw_data = calloc(1, btf->raw_size); in btf__new_empty()
745 if (!btf->raw_data) { in btf__new_empty()
747 return ERR_PTR(-ENOMEM); in btf__new_empty()
750 btf->hdr = btf->raw_data; in btf__new_empty()
751 btf->hdr->hdr_len = sizeof(struct btf_header); in btf__new_empty()
752 btf->hdr->magic = BTF_MAGIC; in btf__new_empty()
753 btf->hdr->version = BTF_VERSION; in btf__new_empty()
755 btf->types_data = btf->raw_data + btf->hdr->hdr_len; in btf__new_empty()
756 btf->strs_data = btf->raw_data + btf->hdr->hdr_len; in btf__new_empty()
757 btf->hdr->str_len = 1; /* empty string at offset 0 */ in btf__new_empty()
769 return ERR_PTR(-ENOMEM); in btf__new()
771 btf->raw_data = malloc(size); in btf__new()
772 if (!btf->raw_data) { in btf__new()
773 err = -ENOMEM; in btf__new()
776 memcpy(btf->raw_data, data, size); in btf__new()
777 btf->raw_size = size; in btf__new()
779 btf->hdr = btf->raw_data; in btf__new()
784 btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off; in btf__new()
785 btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off; in btf__new()
792 btf->fd = -1; in btf__new()
806 int err = 0, fd = -1, idx = 0; in btf__parse_elf()
814 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); in btf__parse_elf()
819 err = -errno; in btf__parse_elf()
824 err = -LIBBPF_ERRNO__FORMAT; in btf__parse_elf()
878 err = -ENOENT; in btf__parse_elf()
881 btf = btf__new(btf_data->d_buf, btf_data->d_size); in btf__parse_elf()
898 *btf_ext = btf_ext__new(btf_ext_data->d_buf, in btf__parse_elf()
899 btf_ext_data->d_size); in btf__parse_elf()
937 err = -errno; in btf__parse_raw()
943 err = -EIO; in btf__parse_raw()
948 err = -EPROTO; in btf__parse_raw()
954 err = -errno; in btf__parse_raw()
959 err = -errno; in btf__parse_raw()
964 err = -errno; in btf__parse_raw()
968 /* pre-alloc memory and read all of BTF data */ in btf__parse_raw()
971 err = -ENOMEM; in btf__parse_raw()
975 err = -EIO; in btf__parse_raw()
997 if (!IS_ERR(btf) || PTR_ERR(btf) != -EPROTO) in btf__parse()
1008 return a->offset - b->offset; in compare_vsi_off()
1015 const char *name = btf__name_by_offset(btf, t->name_off); in btf_fixup_datasec()
1023 return -ENOENT; in btf_fixup_datasec()
1029 if (t->size) in btf_fixup_datasec()
1033 if (ret || !size || (t->size && t->size != size)) { in btf_fixup_datasec()
1035 return -ENOENT; in btf_fixup_datasec()
1038 t->size = size; in btf_fixup_datasec()
1041 t_var = btf__type_by_id(btf, vsi->type); in btf_fixup_datasec()
1045 pr_debug("Non-VAR type seen in section %s\n", name); in btf_fixup_datasec()
1046 return -EINVAL; in btf_fixup_datasec()
1049 if (var->linkage == BTF_VAR_STATIC) in btf_fixup_datasec()
1052 name = btf__name_by_offset(btf, t_var->name_off); in btf_fixup_datasec()
1055 return -ENOENT; in btf_fixup_datasec()
1062 return -ENOENT; in btf_fixup_datasec()
1065 vsi->offset = off; in btf_fixup_datasec()
1078 for (i = 1; i <= btf->nr_types; i++) { in btf__finalize_data()
1105 if (btf->fd >= 0) in btf__load()
1106 return -EEXIST; in btf__load()
1112 return -ENOMEM; in btf__load()
1119 err = -ENOMEM; in btf__load()
1123 btf->raw_size = raw_size; in btf__load()
1124 btf->raw_data = raw_data; in btf__load()
1126 btf->fd = bpf_load_btf(raw_data, raw_size, log_buf, log_buf_size, false); in btf__load()
1127 if (btf->fd < 0) { in btf__load()
1135 err = -errno; in btf__load()
1149 return btf->fd; in btf__fd()
1154 btf->fd = fd; in btf__set_fd()
1159 struct btf_header *hdr = btf->hdr; in btf_get_raw_data()
1165 data = swap_endian ? btf->raw_data_swapped : btf->raw_data; in btf_get_raw_data()
1167 *size = btf->raw_size; in btf_get_raw_data()
1171 data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len; in btf_get_raw_data()
1177 memcpy(p, hdr, hdr->hdr_len); in btf_get_raw_data()
1180 p += hdr->hdr_len; in btf_get_raw_data()
1182 memcpy(p, btf->types_data, hdr->type_len); in btf_get_raw_data()
1184 for (i = 1; i <= btf->nr_types; i++) { in btf_get_raw_data()
1185 t = p + btf->type_offs[i]; in btf_get_raw_data()
1186 /* btf_bswap_type_rest() relies on native t->info, so in btf_get_raw_data()
1195 p += hdr->type_len; in btf_get_raw_data()
1197 memcpy(p, btf->strs_data, hdr->str_len); in btf_get_raw_data()
1198 p += hdr->str_len; in btf_get_raw_data()
1213 data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian); in btf__get_raw_data()
1217 btf->raw_size = data_sz; in btf__get_raw_data()
1218 if (btf->swapped_endian) in btf__get_raw_data()
1219 btf->raw_data_swapped = data; in btf__get_raw_data()
1221 btf->raw_data = data; in btf__get_raw_data()
1228 if (offset < btf->hdr->str_len) in btf__str_by_offset()
1229 return btf->strs_data + offset; in btf__str_by_offset()
1255 * let's start with a sane default - 4KiB here - and resize it only if in btf__get_from_id()
1262 err = -ENOMEM; in btf__get_from_id()
1276 err = -ENOMEM; in btf__get_from_id()
1318 return -EINVAL; in btf__get_map_kv_tids()
1332 return -EINVAL; in btf__get_map_kv_tids()
1338 return -EINVAL; in btf__get_map_kv_tids()
1344 key_size = btf__resolve_size(btf, key->type); in btf__get_map_kv_tids()
1353 return -EINVAL; in btf__get_map_kv_tids()
1356 value_size = btf__resolve_size(btf, value->type); in btf__get_map_kv_tids()
1365 return -EINVAL; in btf__get_map_kv_tids()
1368 *key_type_id = key->type; in btf__get_map_kv_tids()
1369 *value_type_id = value->type; in btf__get_map_kv_tids()
1377 const char *str = btf->strs_data + (long)key; in strs_hash_fn()
1385 const char *str1 = btf->strs_data + (long)key1; in strs_hash_equal_fn()
1386 const char *str2 = btf->strs_data + (long)key2; in strs_hash_equal_fn()
1393 if (btf->raw_data) { in btf_invalidate_raw_data()
1394 free(btf->raw_data); in btf_invalidate_raw_data()
1395 btf->raw_data = NULL; in btf_invalidate_raw_data()
1397 if (btf->raw_data_swapped) { in btf_invalidate_raw_data()
1398 free(btf->raw_data_swapped); in btf_invalidate_raw_data()
1399 btf->raw_data_swapped = NULL; in btf_invalidate_raw_data()
1403 /* Ensure BTF is ready to be modified (by splitting into a three memory
1420 /* split raw data into three memory regions */ in btf_ensure_modifiable()
1421 hdr = malloc(btf->hdr->hdr_len); in btf_ensure_modifiable()
1422 types = malloc(btf->hdr->type_len); in btf_ensure_modifiable()
1423 strs = malloc(btf->hdr->str_len); in btf_ensure_modifiable()
1427 memcpy(hdr, btf->hdr, btf->hdr->hdr_len); in btf_ensure_modifiable()
1428 memcpy(types, btf->types_data, btf->hdr->type_len); in btf_ensure_modifiable()
1429 memcpy(strs, btf->strs_data, btf->hdr->str_len); in btf_ensure_modifiable()
1439 strs_end = strs + btf->hdr->str_len; in btf_ensure_modifiable()
1445 if (err == -EEXIST) in btf_ensure_modifiable()
1452 btf->hdr = hdr; in btf_ensure_modifiable()
1453 btf->types_data = types; in btf_ensure_modifiable()
1454 btf->types_data_cap = btf->hdr->type_len; in btf_ensure_modifiable()
1455 btf->strs_data = strs; in btf_ensure_modifiable()
1456 btf->strs_data_cap = btf->hdr->str_len; in btf_ensure_modifiable()
1457 btf->strs_hash = hash; in btf_ensure_modifiable()
1461 btf->strs_deduped = btf->hdr->str_len <= 1; in btf_ensure_modifiable()
1473 return -ENOMEM; in btf_ensure_modifiable()
1478 return btf_add_mem(&btf->strs_data, &btf->strs_data_cap, 1, in btf_add_str_mem()
1479 btf->hdr->str_len, BTF_MAX_STR_OFFSET, add_sz); in btf_add_str_mem()
1484 * - >0 offset into string section, if string is found;
1485 * - -ENOENT, if string is not in the string section;
1486 * - <0, on any other error.
1495 return -ENOMEM; in btf__find_str()
1501 return -ENOMEM; in btf__find_str()
1503 new_off = btf->hdr->str_len; in btf__find_str()
1506 if (hashmap__find(btf->strs_hash, (void *)new_off, (void **)&old_off)) in btf__find_str()
1509 return -ENOENT; in btf__find_str()
1514 * - > 0 offset into string section, on success;
1515 * - < 0, on error.
1524 return -ENOMEM; in btf__add_str()
1526 /* Hashmap keys are always offsets within btf->strs_data, so to even in btf__add_str()
1529 * until btf->hdr->str_len is incremented, that string is just a piece in btf__add_str()
1532 * ready to be used, only a simple btf->hdr->str_len increment away. in btf__add_str()
1537 return -ENOMEM; in btf__add_str()
1539 new_off = btf->hdr->str_len; in btf__add_str()
1546 err = hashmap__insert(btf->strs_hash, (void *)new_off, (void *)new_off, in btf__add_str()
1548 if (err == -EEXIST) in btf__add_str()
1553 btf->hdr->str_len += len; /* new unique string, adjust data length */ in btf__add_str()
1559 return btf_add_mem(&btf->types_data, &btf->types_data_cap, 1, in btf_add_type_mem()
1560 btf->hdr->type_len, UINT_MAX, add_sz); in btf_add_type_mem()
1570 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t)); in btf_type_inc_vlen()
1575 * - *name* - non-empty, non-NULL type name;
1576 * - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes;
1577 * - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL.
1579 * - >0, type ID of newly added BTF type;
1580 * - <0, on error.
1587 /* non-empty name */ in btf__add_int()
1589 return -EINVAL; in btf__add_int()
1591 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16) in btf__add_int()
1592 return -EINVAL; in btf__add_int()
1594 return -EINVAL; in btf__add_int()
1598 return -ENOMEM; in btf__add_int()
1603 return -ENOMEM; in btf__add_int()
1613 t->name_off = name_off; in btf__add_int()
1614 t->info = btf_type_info(BTF_KIND_INT, 0, 0); in btf__add_int()
1615 t->size = byte_sz; in btf__add_int()
1619 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf__add_int()
1623 btf->hdr->type_len += sz; in btf__add_int()
1624 btf->hdr->str_off += sz; in btf__add_int()
1625 btf->nr_types++; in btf__add_int()
1626 return btf->nr_types; in btf__add_int()
1636 return -EINVAL; in validate_type_id()
1647 return -EINVAL; in btf_add_ref_kind()
1650 return -ENOMEM; in btf_add_ref_kind()
1655 return -ENOMEM; in btf_add_ref_kind()
1663 t->name_off = name_off; in btf_add_ref_kind()
1664 t->info = btf_type_info(kind, 0, 0); in btf_add_ref_kind()
1665 t->type = ref_type_id; in btf_add_ref_kind()
1667 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf_add_ref_kind()
1671 btf->hdr->type_len += sz; in btf_add_ref_kind()
1672 btf->hdr->str_off += sz; in btf_add_ref_kind()
1673 btf->nr_types++; in btf_add_ref_kind()
1674 return btf->nr_types; in btf_add_ref_kind()
1679 * - *ref_type_id* - referenced type ID, it might not exist yet;
1681 * - >0, type ID of newly added BTF type;
1682 * - <0, on error.
1691 * - *index_type_id* - type ID of the type describing array index;
1692 * - *elem_type_id* - type ID of the type describing array element;
1693 * - *nr_elems* - the size of the array;
1695 * - >0, type ID of newly added BTF type;
1696 * - <0, on error.
1705 return -EINVAL; in btf__add_array()
1708 return -ENOMEM; in btf__add_array()
1713 return -ENOMEM; in btf__add_array()
1715 t->name_off = 0; in btf__add_array()
1716 t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0); in btf__add_array()
1717 t->size = 0; in btf__add_array()
1720 a->type = elem_type_id; in btf__add_array()
1721 a->index_type = index_type_id; in btf__add_array()
1722 a->nelems = nr_elems; in btf__add_array()
1724 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf__add_array()
1728 btf->hdr->type_len += sz; in btf__add_array()
1729 btf->hdr->str_off += sz; in btf__add_array()
1730 btf->nr_types++; in btf__add_array()
1731 return btf->nr_types; in btf__add_array()
1741 return -ENOMEM; in btf_add_composite()
1746 return -ENOMEM; in btf_add_composite()
1757 t->name_off = name_off; in btf_add_composite()
1758 t->info = btf_type_info(kind, 0, 0); in btf_add_composite()
1759 t->size = bytes_sz; in btf_add_composite()
1761 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf_add_composite()
1765 btf->hdr->type_len += sz; in btf_add_composite()
1766 btf->hdr->str_off += sz; in btf_add_composite()
1767 btf->nr_types++; in btf_add_composite()
1768 return btf->nr_types; in btf_add_composite()
1773 * - *name* - name of the struct, can be NULL or empty for anonymous structs;
1774 * - *byte_sz* - size of the struct, in bytes;
1780 * - >0, type ID of newly added BTF type;
1781 * - <0, on error.
1790 * - *name* - name of the union, can be NULL or empty for anonymous union;
1791 * - *byte_sz* - size of the union, in bytes;
1798 * - >0, type ID of newly added BTF type;
1799 * - <0, on error.
1808 * - *name* - name of the field, can be NULL or empty for anonymous field;
1809 * - *type_id* - type ID for the type describing field type;
1810 * - *bit_offset* - bit offset of the start of the field within struct/union;
1811 * - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields;
1813 * - 0, on success;
1814 * - <0, on error.
1825 if (btf->nr_types == 0) in btf__add_field()
1826 return -EINVAL; in btf__add_field()
1827 t = btf_type_by_id(btf, btf->nr_types); in btf__add_field()
1829 return -EINVAL; in btf__add_field()
1832 return -EINVAL; in btf__add_field()
1833 /* best-effort bit field offset/size enforcement */ in btf__add_field()
1836 return -EINVAL; in btf__add_field()
1840 return -EINVAL; in btf__add_field()
1844 return -ENOMEM; in btf__add_field()
1849 return -ENOMEM; in btf__add_field()
1857 m->name_off = name_off; in btf__add_field()
1858 m->type = type_id; in btf__add_field()
1859 m->offset = bit_offset | (bit_size << 24); in btf__add_field()
1862 t = btf_type_by_id(btf, btf->nr_types); in btf__add_field()
1864 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t)); in btf__add_field()
1866 btf->hdr->type_len += sz; in btf__add_field()
1867 btf->hdr->str_off += sz; in btf__add_field()
1873 * - *name* - name of the enum, can be NULL or empty for anonymous enums;
1874 * - *byte_sz* - size of the enum, in bytes.
1881 * - >0, type ID of newly added BTF type;
1882 * - <0, on error.
1890 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8) in btf__add_enum()
1891 return -EINVAL; in btf__add_enum()
1894 return -ENOMEM; in btf__add_enum()
1899 return -ENOMEM; in btf__add_enum()
1908 t->name_off = name_off; in btf__add_enum()
1909 t->info = btf_type_info(BTF_KIND_ENUM, 0, 0); in btf__add_enum()
1910 t->size = byte_sz; in btf__add_enum()
1912 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf__add_enum()
1916 btf->hdr->type_len += sz; in btf__add_enum()
1917 btf->hdr->str_off += sz; in btf__add_enum()
1918 btf->nr_types++; in btf__add_enum()
1919 return btf->nr_types; in btf__add_enum()
1924 * - *name* - name of the enumerator value, can't be NULL or empty;
1925 * - *value* - integer value corresponding to enum value *name*;
1927 * - 0, on success;
1928 * - <0, on error.
1937 if (btf->nr_types == 0) in btf__add_enum_value()
1938 return -EINVAL; in btf__add_enum_value()
1939 t = btf_type_by_id(btf, btf->nr_types); in btf__add_enum_value()
1941 return -EINVAL; in btf__add_enum_value()
1943 /* non-empty name */ in btf__add_enum_value()
1945 return -EINVAL; in btf__add_enum_value()
1947 return -E2BIG; in btf__add_enum_value()
1951 return -ENOMEM; in btf__add_enum_value()
1956 return -ENOMEM; in btf__add_enum_value()
1962 v->name_off = name_off; in btf__add_enum_value()
1963 v->val = value; in btf__add_enum_value()
1966 t = btf_type_by_id(btf, btf->nr_types); in btf__add_enum_value()
1969 btf->hdr->type_len += sz; in btf__add_enum_value()
1970 btf->hdr->str_off += sz; in btf__add_enum_value()
1976 * - *name*, non-empty/non-NULL name;
1977 * - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT,
1980 * - >0, type ID of newly added BTF type;
1981 * - <0, on error.
1986 return -EINVAL; in btf__add_fwd()
1998 t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION); in btf__add_fwd()
2003 * values; we also assume a standard 4-byte size for it in btf__add_fwd()
2007 return -EINVAL; in btf__add_fwd()
2013 * - *name*, non-empty/non-NULL name;
2014 * - *ref_type_id* - referenced type ID, it might not exist yet;
2016 * - >0, type ID of newly added BTF type;
2017 * - <0, on error.
2022 return -EINVAL; in btf__add_typedef()
2029 * - *ref_type_id* - referenced type ID, it might not exist yet;
2031 * - >0, type ID of newly added BTF type;
2032 * - <0, on error.
2041 * - *ref_type_id* - referenced type ID, it might not exist yet;
2043 * - >0, type ID of newly added BTF type;
2044 * - <0, on error.
2053 * - *ref_type_id* - referenced type ID, it might not exist yet;
2055 * - >0, type ID of newly added BTF type;
2056 * - <0, on error.
2065 * - *name*, non-empty/non-NULL name;
2066 * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet;
2068 * - >0, type ID of newly added BTF type;
2069 * - <0, on error.
2077 return -EINVAL; in btf__add_func()
2080 return -EINVAL; in btf__add_func()
2086 t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0); in btf__add_func()
2093 * - *ret_type_id* - type ID for return result of a function.
2100 * - >0, type ID of newly added BTF type;
2101 * - <0, on error.
2109 return -EINVAL; in btf__add_func_proto()
2112 return -ENOMEM; in btf__add_func_proto()
2117 return -ENOMEM; in btf__add_func_proto()
2122 t->name_off = 0; in btf__add_func_proto()
2123 t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0); in btf__add_func_proto()
2124 t->type = ret_type_id; in btf__add_func_proto()
2126 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf__add_func_proto()
2130 btf->hdr->type_len += sz; in btf__add_func_proto()
2131 btf->hdr->str_off += sz; in btf__add_func_proto()
2132 btf->nr_types++; in btf__add_func_proto()
2133 return btf->nr_types; in btf__add_func_proto()
2138 * - *name* - parameter name, can be NULL or empty;
2139 * - *type_id* - type ID describing the type of the parameter.
2141 * - 0, on success;
2142 * - <0, on error.
2151 return -EINVAL; in btf__add_func_param()
2154 if (btf->nr_types == 0) in btf__add_func_param()
2155 return -EINVAL; in btf__add_func_param()
2156 t = btf_type_by_id(btf, btf->nr_types); in btf__add_func_param()
2158 return -EINVAL; in btf__add_func_param()
2162 return -ENOMEM; in btf__add_func_param()
2167 return -ENOMEM; in btf__add_func_param()
2175 p->name_off = name_off; in btf__add_func_param()
2176 p->type = type_id; in btf__add_func_param()
2179 t = btf_type_by_id(btf, btf->nr_types); in btf__add_func_param()
2182 btf->hdr->type_len += sz; in btf__add_func_param()
2183 btf->hdr->str_off += sz; in btf__add_func_param()
2189 * - *name* - non-empty/non-NULL name;
2190 * - *linkage* - variable linkage, one of BTF_VAR_STATIC,
2192 * - *type_id* - type ID of the type describing the type of the variable.
2194 * - >0, type ID of newly added BTF type;
2195 * - <0, on error.
2203 /* non-empty name */ in btf__add_var()
2205 return -EINVAL; in btf__add_var()
2208 return -EINVAL; in btf__add_var()
2210 return -EINVAL; in btf__add_var()
2214 return -ENOMEM; in btf__add_var()
2219 return -ENOMEM; in btf__add_var()
2225 t->name_off = name_off; in btf__add_var()
2226 t->info = btf_type_info(BTF_KIND_VAR, 0, 0); in btf__add_var()
2227 t->type = type_id; in btf__add_var()
2230 v->linkage = linkage; in btf__add_var()
2232 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf__add_var()
2236 btf->hdr->type_len += sz; in btf__add_var()
2237 btf->hdr->str_off += sz; in btf__add_var()
2238 btf->nr_types++; in btf__add_var()
2239 return btf->nr_types; in btf__add_var()
2244 * - *name* - non-empty/non-NULL name;
2245 * - *byte_sz* - data section size, in bytes.
2251 * - >0, type ID of newly added BTF type;
2252 * - <0, on error.
2259 /* non-empty name */ in btf__add_datasec()
2261 return -EINVAL; in btf__add_datasec()
2264 return -ENOMEM; in btf__add_datasec()
2269 return -ENOMEM; in btf__add_datasec()
2276 t->name_off = name_off; in btf__add_datasec()
2277 t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0); in btf__add_datasec()
2278 t->size = byte_sz; in btf__add_datasec()
2280 err = btf_add_type_idx_entry(btf, btf->hdr->type_len); in btf__add_datasec()
2284 btf->hdr->type_len += sz; in btf__add_datasec()
2285 btf->hdr->str_off += sz; in btf__add_datasec()
2286 btf->nr_types++; in btf__add_datasec()
2287 return btf->nr_types; in btf__add_datasec()
2292 * - *var_type_id* - type ID, describing type of the variable;
2293 * - *offset* - variable offset within data section, in bytes;
2294 * - *byte_sz* - variable size, in bytes.
2297 * - 0, on success;
2298 * - <0, on error.
2307 if (btf->nr_types == 0) in btf__add_datasec_var_info()
2308 return -EINVAL; in btf__add_datasec_var_info()
2309 t = btf_type_by_id(btf, btf->nr_types); in btf__add_datasec_var_info()
2311 return -EINVAL; in btf__add_datasec_var_info()
2314 return -EINVAL; in btf__add_datasec_var_info()
2318 return -ENOMEM; in btf__add_datasec_var_info()
2323 return -ENOMEM; in btf__add_datasec_var_info()
2325 v->type = var_type_id; in btf__add_datasec_var_info()
2326 v->offset = offset; in btf__add_datasec_var_info()
2327 v->size = byte_sz; in btf__add_datasec_var_info()
2330 t = btf_type_by_id(btf, btf->nr_types); in btf__add_datasec_var_info()
2333 btf->hdr->type_len += sz; in btf__add_datasec_var_info()
2334 btf->hdr->str_off += sz; in btf__add_datasec_var_info()
2355 if (ext_sec->len == 0) in btf_ext_setup_info()
2358 if (ext_sec->off & 0x03) { in btf_ext_setup_info()
2360 ext_sec->desc); in btf_ext_setup_info()
2361 return -EINVAL; in btf_ext_setup_info()
2364 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; in btf_ext_setup_info()
2365 info_left = ext_sec->len; in btf_ext_setup_info()
2367 if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) { in btf_ext_setup_info()
2369 ext_sec->desc, ext_sec->off, ext_sec->len); in btf_ext_setup_info()
2370 return -EINVAL; in btf_ext_setup_info()
2375 pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc); in btf_ext_setup_info()
2376 return -EINVAL; in btf_ext_setup_info()
2381 if (record_size < ext_sec->min_rec_size || in btf_ext_setup_info()
2384 ext_sec->desc, record_size); in btf_ext_setup_info()
2385 return -EINVAL; in btf_ext_setup_info()
2389 info_left -= sizeof(__u32); in btf_ext_setup_info()
2393 pr_debug("%s section in .BTF.ext has no records", ext_sec->desc); in btf_ext_setup_info()
2394 return -EINVAL; in btf_ext_setup_info()
2404 ext_sec->desc); in btf_ext_setup_info()
2405 return -EINVAL; in btf_ext_setup_info()
2408 num_records = sinfo->num_info; in btf_ext_setup_info()
2411 ext_sec->desc); in btf_ext_setup_info()
2412 return -EINVAL; in btf_ext_setup_info()
2419 ext_sec->desc); in btf_ext_setup_info()
2420 return -EINVAL; in btf_ext_setup_info()
2423 info_left -= total_record_size; in btf_ext_setup_info()
2427 ext_info = ext_sec->ext_info; in btf_ext_setup_info()
2428 ext_info->len = ext_sec->len - sizeof(__u32); in btf_ext_setup_info()
2429 ext_info->rec_size = record_size; in btf_ext_setup_info()
2430 ext_info->info = info + sizeof(__u32); in btf_ext_setup_info()
2438 .off = btf_ext->hdr->func_info_off, in btf_ext_setup_func_info()
2439 .len = btf_ext->hdr->func_info_len, in btf_ext_setup_func_info()
2441 .ext_info = &btf_ext->func_info, in btf_ext_setup_func_info()
2451 .off = btf_ext->hdr->line_info_off, in btf_ext_setup_line_info()
2452 .len = btf_ext->hdr->line_info_len, in btf_ext_setup_line_info()
2454 .ext_info = &btf_ext->line_info, in btf_ext_setup_line_info()
2464 .off = btf_ext->hdr->core_relo_off, in btf_ext_setup_core_relos()
2465 .len = btf_ext->hdr->core_relo_len, in btf_ext_setup_core_relos()
2467 .ext_info = &btf_ext->core_relo_info, in btf_ext_setup_core_relos()
2479 data_size < hdr->hdr_len) { in btf_ext_parse_hdr()
2481 return -EINVAL; in btf_ext_parse_hdr()
2484 if (hdr->magic == bswap_16(BTF_MAGIC)) { in btf_ext_parse_hdr()
2485 pr_warn("BTF.ext in non-native endianness is not supported\n"); in btf_ext_parse_hdr()
2486 return -ENOTSUP; in btf_ext_parse_hdr()
2487 } else if (hdr->magic != BTF_MAGIC) { in btf_ext_parse_hdr()
2488 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic); in btf_ext_parse_hdr()
2489 return -EINVAL; in btf_ext_parse_hdr()
2492 if (hdr->version != BTF_VERSION) { in btf_ext_parse_hdr()
2493 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version); in btf_ext_parse_hdr()
2494 return -ENOTSUP; in btf_ext_parse_hdr()
2497 if (hdr->flags) { in btf_ext_parse_hdr()
2498 pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags); in btf_ext_parse_hdr()
2499 return -ENOTSUP; in btf_ext_parse_hdr()
2502 if (data_size == hdr->hdr_len) { in btf_ext_parse_hdr()
2504 return -EINVAL; in btf_ext_parse_hdr()
2514 free(btf_ext->data); in btf_ext__free()
2529 return ERR_PTR(-ENOMEM); in btf_ext__new()
2531 btf_ext->data_size = size; in btf_ext__new()
2532 btf_ext->data = malloc(size); in btf_ext__new()
2533 if (!btf_ext->data) { in btf_ext__new()
2534 err = -ENOMEM; in btf_ext__new()
2537 memcpy(btf_ext->data, data, size); in btf_ext__new()
2539 if (btf_ext->hdr->hdr_len < in btf_ext__new()
2550 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) in btf_ext__new()
2567 *size = btf_ext->data_size; in btf_ext__get_raw_data()
2568 return btf_ext->data; in btf_ext__get_raw_data()
2583 record_size = ext_info->rec_size; in btf_ext_reloc_info()
2584 sinfo = ext_info->info; in btf_ext_reloc_info()
2585 remain_len = ext_info->len; in btf_ext_reloc_info()
2587 records_len = sinfo->num_info * record_size; in btf_ext_reloc_info()
2588 info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); in btf_ext_reloc_info()
2590 remain_len -= sec_hdrlen + records_len; in btf_ext_reloc_info()
2598 return -ENOMEM; in btf_ext_reloc_info()
2600 memcpy(data + existing_len, sinfo->data, records_len); in btf_ext_reloc_info()
2604 for (i = 0; i < sinfo->num_info; i++) { in btf_ext_reloc_info()
2612 *cnt += sinfo->num_info; in btf_ext_reloc_info()
2616 return -ENOENT; in btf_ext_reloc_info()
2624 return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name, in btf_ext__reloc_func_info()
2633 return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name, in btf_ext__reloc_line_info()
2639 return btf_ext->func_info.rec_size; in btf_ext__func_info_rec_size()
2644 return btf_ext->line_info.rec_size; in btf_ext__line_info_rec_size()
2664 * memory in-place with deduplicated types and strings without any loss of
2674 * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html
2679 * BTF type information is typically emitted either as a result of conversion
2735 * types, in general, can form graphs containing cycles, not just DAGs.
2781 * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that
2785 * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash
2804 return -EINVAL; in btf__dedup()
2808 return -ENOMEM; in btf__dedup()
2846 #define BTF_UNPROCESSED_ID ((__u32)-1)
2847 #define BTF_IN_PROGRESS_ID ((__u32)-2)
2850 /* .BTF section to be deduped in-place */
2895 hashmap__for_each_key_entry(d->dedup_table, node, (void *)hash)
2899 return hashmap__append(d->dedup_table, in btf_dedup_table_add()
2906 if (d->hypot_cnt == d->hypot_cap) { in btf_dedup_hypot_map_add()
2909 d->hypot_cap += max((size_t)16, d->hypot_cap / 2); in btf_dedup_hypot_map_add()
2910 new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32)); in btf_dedup_hypot_map_add()
2912 return -ENOMEM; in btf_dedup_hypot_map_add()
2913 d->hypot_list = new_list; in btf_dedup_hypot_map_add()
2915 d->hypot_list[d->hypot_cnt++] = from_id; in btf_dedup_hypot_map_add()
2916 d->hypot_map[from_id] = to_id; in btf_dedup_hypot_map_add()
2924 for (i = 0; i < d->hypot_cnt; i++) in btf_dedup_clear_hypot_map()
2925 d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID; in btf_dedup_clear_hypot_map()
2926 d->hypot_cnt = 0; in btf_dedup_clear_hypot_map()
2931 hashmap__free(d->dedup_table); in btf_dedup_free()
2932 d->dedup_table = NULL; in btf_dedup_free()
2934 free(d->map); in btf_dedup_free()
2935 d->map = NULL; in btf_dedup_free()
2937 free(d->hypot_map); in btf_dedup_free()
2938 d->hypot_map = NULL; in btf_dedup_free()
2940 free(d->hypot_list); in btf_dedup_free()
2941 d->hypot_list = NULL; in btf_dedup_free()
2969 return ERR_PTR(-ENOMEM); in btf_dedup_new()
2971 d->opts.dont_resolve_fwds = opts && opts->dont_resolve_fwds; in btf_dedup_new()
2973 if (opts && opts->dedup_table_size == 1) in btf_dedup_new()
2976 d->btf = btf; in btf_dedup_new()
2977 d->btf_ext = btf_ext; in btf_dedup_new()
2979 d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL); in btf_dedup_new()
2980 if (IS_ERR(d->dedup_table)) { in btf_dedup_new()
2981 err = PTR_ERR(d->dedup_table); in btf_dedup_new()
2982 d->dedup_table = NULL; in btf_dedup_new()
2986 d->map = malloc(sizeof(__u32) * (1 + btf->nr_types)); in btf_dedup_new()
2987 if (!d->map) { in btf_dedup_new()
2988 err = -ENOMEM; in btf_dedup_new()
2992 d->map[0] = 0; in btf_dedup_new()
2993 for (i = 1; i <= btf->nr_types; i++) { in btf_dedup_new()
2994 struct btf_type *t = btf_type_by_id(d->btf, i); in btf_dedup_new()
2996 /* VAR and DATASEC are never deduped and are self-canonical */ in btf_dedup_new()
2998 d->map[i] = i; in btf_dedup_new()
3000 d->map[i] = BTF_UNPROCESSED_ID; in btf_dedup_new()
3003 d->hypot_map = malloc(sizeof(__u32) * (1 + btf->nr_types)); in btf_dedup_new()
3004 if (!d->hypot_map) { in btf_dedup_new()
3005 err = -ENOMEM; in btf_dedup_new()
3008 for (i = 0; i <= btf->nr_types; i++) in btf_dedup_new()
3009 d->hypot_map[i] = BTF_UNPROCESSED_ID; in btf_dedup_new()
3032 for (i = 1; i <= d->btf->nr_types; i++) { in btf_for_each_str_off()
3033 t = btf_type_by_id(d->btf, i); in btf_for_each_str_off()
3034 r = fn(&t->name_off, ctx); in btf_for_each_str_off()
3045 r = fn(&m->name_off, ctx); in btf_for_each_str_off()
3057 r = fn(&m->name_off, ctx); in btf_for_each_str_off()
3069 r = fn(&m->name_off, ctx); in btf_for_each_str_off()
3081 if (!d->btf_ext) in btf_for_each_str_off()
3084 line_data_cur = d->btf_ext->line_info.info; in btf_for_each_str_off()
3085 line_data_end = d->btf_ext->line_info.info + d->btf_ext->line_info.len; in btf_for_each_str_off()
3086 rec_size = d->btf_ext->line_info.rec_size; in btf_for_each_str_off()
3091 __u32 num_info = sec->num_info; in btf_for_each_str_off()
3093 r = fn(&sec->sec_name_off, ctx); in btf_for_each_str_off()
3100 r = fn(&line_info->file_name_off, ctx); in btf_for_each_str_off()
3103 r = fn(&line_info->line_off, ctx); in btf_for_each_str_off()
3118 return strcmp(p1->str, p2->str); in str_sort_by_content()
3126 if (p1->str != p2->str) in str_sort_by_offset()
3127 return p1->str < p2->str ? -1 : 1; in str_sort_by_offset()
3135 if (str_ptr != p->str) in btf_dedup_str_ptr_cmp()
3136 return (const char *)str_ptr < p->str ? -1 : 1; in btf_dedup_str_ptr_cmp()
3149 s = bsearch(strs->data + *str_off_ptr, strs->ptrs, strs->cnt, in btf_str_mark_as_used()
3152 return -EINVAL; in btf_str_mark_as_used()
3153 s->used = true; in btf_str_mark_as_used()
3166 s = bsearch(strs->data + *str_off_ptr, strs->ptrs, strs->cnt, in btf_str_remap_offset()
3169 return -EINVAL; in btf_str_remap_offset()
3170 *str_off_ptr = s->new_off; in btf_str_remap_offset()
3187 char *start = d->btf->strs_data; in btf_dedup_strings()
3188 char *end = start + d->btf->hdr->str_len; in btf_dedup_strings()
3199 if (d->btf->strs_deduped) in btf_dedup_strings()
3210 err = -ENOMEM; in btf_dedup_strings()
3224 tmp_strs = malloc(d->btf->hdr->str_len); in btf_dedup_strings()
3226 err = -ENOMEM; in btf_dedup_strings()
3265 int new_off = p - tmp_strs; in btf_dedup_strings()
3281 d->btf->hdr->str_len = p - tmp_strs; in btf_dedup_strings()
3282 memmove(start, tmp_strs, d->btf->hdr->str_len); in btf_dedup_strings()
3283 end = start + d->btf->hdr->str_len; in btf_dedup_strings()
3293 d->btf->hdr->str_len = end - start; in btf_dedup_strings()
3294 d->btf->strs_deduped = true; in btf_dedup_strings()
3306 h = hash_combine(0, t->name_off); in btf_hash_common()
3307 h = hash_combine(h, t->info); in btf_hash_common()
3308 h = hash_combine(h, t->size); in btf_hash_common()
3314 return t1->name_off == t2->name_off && in btf_equal_common()
3315 t1->info == t2->info && in btf_equal_common()
3316 t1->size == t2->size; in btf_equal_common()
3348 h = hash_combine(0, t->name_off); in btf_hash_enum()
3349 h = hash_combine(h, t->info & ~0xffff); in btf_hash_enum()
3350 h = hash_combine(h, t->size); in btf_hash_enum()
3368 if (m1->name_off != m2->name_off || m1->val != m2->val) in btf_equal_enum()
3386 return t1->name_off == t2->name_off && in btf_compat_enum()
3387 (t1->info & ~0xffff) == (t2->info & ~0xffff) && in btf_compat_enum()
3388 t1->size == t2->size; in btf_compat_enum()
3404 h = hash_combine(h, member->name_off); in btf_hash_struct()
3405 h = hash_combine(h, member->offset); in btf_hash_struct()
3430 if (m1->name_off != m2->name_off || m1->offset != m2->offset) in btf_shallow_equal_struct()
3448 h = hash_combine(h, info->type); in btf_hash_array()
3449 h = hash_combine(h, info->index_type); in btf_hash_array()
3450 h = hash_combine(h, info->nelems); in btf_hash_array()
3470 return info1->type == info2->type && in btf_equal_array()
3471 info1->index_type == info2->index_type && in btf_equal_array()
3472 info1->nelems == info2->nelems; in btf_equal_array()
3485 return btf_array(t1)->nelems == btf_array(t2)->nelems; in btf_compat_array()
3501 h = hash_combine(h, member->name_off); in btf_hash_fnproto()
3502 h = hash_combine(h, member->type); in btf_hash_fnproto()
3528 if (m1->name_off != m2->name_off || m1->type != m2->type) in btf_equal_fnproto()
3548 if (t1->name_off != t2->name_off || t1->info != t2->info) in btf_compat_fnproto()
3555 if (m1->name_off != m2->name_off) in btf_compat_fnproto()
3567 * canonical and is added into `btf_dedup->dedup_table` as another candidate.
3571 struct btf_type *t = btf_type_by_id(d->btf, type_id); in btf_dedup_prim_type()
3597 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_prim_type()
3598 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3609 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_prim_type()
3610 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3615 if (d->opts.dont_resolve_fwds) in btf_dedup_prim_type()
3624 d->map[cand_id] = type_id; in btf_dedup_prim_type()
3632 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_prim_type()
3633 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_prim_type()
3642 return -EINVAL; in btf_dedup_prim_type()
3645 d->map[type_id] = new_id; in btf_dedup_prim_type()
3647 return -ENOMEM; in btf_dedup_prim_type()
3656 for (i = 1; i <= d->btf->nr_types; i++) { in btf_dedup_prim_types()
3669 return d->map[type_id] <= BTF_MAX_NR_TYPES; in is_type_mapped()
3679 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) in resolve_type_id()
3680 type_id = d->map[type_id]; in resolve_type_id()
3692 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) in resolve_fwd_id()
3695 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) in resolve_fwd_id()
3696 type_id = d->map[type_id]; in resolve_fwd_id()
3698 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) in resolve_fwd_id()
3718 * - >0, if type graphs are equivalent;
3719 * - 0, if not equivalent;
3720 * - <0, on error.
3722 * Algorithm performs side-by-side DFS traversal of both type graphs and checks
3727 * a `cand_id` is recored in hypothetical mapping (`btf_dedup->hypot_map`).
3734 * - `canon_id` is mapped to exactly the same type as `cand_id`. This will
3735 * happen when type graphs have cycles. In this case we assume those two
3737 * - `canon_id` is mapped to different type. This is contradiction in our
3751 * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently.
3776 * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION
3780 * - Both types in canonical and candidate graphs are FWDs. If they are
3784 * - Both types in canonical and candidate graphs are concrete STRUCT/UNION,
3786 * - Type in canonical graph is FWD, while type in candidate is concrete
3795 * - Type in canonical graph is concrete STRUCT/UNION, while type in candidate
3800 * turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from
3819 hypot_type_id = d->hypot_map[canon_id]; in btf_dedup_is_equiv()
3824 return -ENOMEM; in btf_dedup_is_equiv()
3826 cand_type = btf_type_by_id(d->btf, cand_id); in btf_dedup_is_equiv()
3827 canon_type = btf_type_by_id(d->btf, canon_id); in btf_dedup_is_equiv()
3831 if (cand_type->name_off != canon_type->name_off) in btf_dedup_is_equiv()
3834 /* FWD <--> STRUCT/UNION equivalence check, if enabled */ in btf_dedup_is_equiv()
3835 if (!d->opts.dont_resolve_fwds in btf_dedup_is_equiv()
3859 if (d->opts.dont_resolve_fwds) in btf_dedup_is_equiv()
3873 if (cand_type->info != canon_type->info) in btf_dedup_is_equiv()
3875 return btf_dedup_is_equiv(d, cand_type->type, canon_type->type); in btf_dedup_is_equiv()
3885 cand_arr->index_type, canon_arr->index_type); in btf_dedup_is_equiv()
3888 return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type); in btf_dedup_is_equiv()
3902 eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); in btf_dedup_is_equiv()
3918 eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type); in btf_dedup_is_equiv()
3925 eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type); in btf_dedup_is_equiv()
3935 return -EINVAL; in btf_dedup_is_equiv()
3945 * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional:
3973 for (i = 0; i < d->hypot_cnt; i++) { in btf_dedup_merge_hypot_map()
3974 cand_type_id = d->hypot_list[i]; in btf_dedup_merge_hypot_map()
3975 targ_type_id = d->hypot_map[cand_type_id]; in btf_dedup_merge_hypot_map()
3978 t_kind = btf_kind(btf__type_by_id(d->btf, t_id)); in btf_dedup_merge_hypot_map()
3979 c_kind = btf_kind(btf__type_by_id(d->btf, c_id)); in btf_dedup_merge_hypot_map()
3984 * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because in btf_dedup_merge_hypot_map()
3993 d->map[c_id] = t_id; in btf_dedup_merge_hypot_map()
3995 d->map[t_id] = c_id; in btf_dedup_merge_hypot_map()
4007 d->map[t_id] = c_id; in btf_dedup_merge_hypot_map()
4024 * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence
4025 * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to
4031 * as canonical for itself and is added into btf_dedup->dedup_table hash map
4044 if (d->map[type_id] <= BTF_MAX_NR_TYPES) in btf_dedup_struct_type()
4047 t = btf_type_by_id(d->btf, type_id); in btf_dedup_struct_type()
4055 __u32 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_struct_type()
4065 * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because in btf_dedup_struct_type()
4068 cand_type = btf_type_by_id(d->btf, cand_id); in btf_dedup_struct_type()
4083 d->map[type_id] = new_id; in btf_dedup_struct_type()
4085 return -ENOMEM; in btf_dedup_struct_type()
4094 for (i = 1; i <= d->btf->nr_types; i++) { in btf_dedup_struct_types()
4113 * one by one. There is no danger of encountering cycles because in C type
4124 * btf_dedup->dedup_table as another possible canonical representative.
4135 if (d->map[type_id] == BTF_IN_PROGRESS_ID) in btf_dedup_ref_type()
4136 return -ELOOP; in btf_dedup_ref_type()
4137 if (d->map[type_id] <= BTF_MAX_NR_TYPES) in btf_dedup_ref_type()
4140 t = btf_type_by_id(d->btf, type_id); in btf_dedup_ref_type()
4141 d->map[type_id] = BTF_IN_PROGRESS_ID; in btf_dedup_ref_type()
4150 ref_type_id = btf_dedup_ref_type(d, t->type); in btf_dedup_ref_type()
4153 t->type = ref_type_id; in btf_dedup_ref_type()
4157 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_ref_type()
4158 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4169 ref_type_id = btf_dedup_ref_type(d, info->type); in btf_dedup_ref_type()
4172 info->type = ref_type_id; in btf_dedup_ref_type()
4174 ref_type_id = btf_dedup_ref_type(d, info->index_type); in btf_dedup_ref_type()
4177 info->index_type = ref_type_id; in btf_dedup_ref_type()
4181 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_ref_type()
4182 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4196 ref_type_id = btf_dedup_ref_type(d, t->type); in btf_dedup_ref_type()
4199 t->type = ref_type_id; in btf_dedup_ref_type()
4204 ref_type_id = btf_dedup_ref_type(d, param->type); in btf_dedup_ref_type()
4207 param->type = ref_type_id; in btf_dedup_ref_type()
4213 cand_id = (__u32)(long)hash_entry->value; in btf_dedup_ref_type()
4214 cand = btf_type_by_id(d->btf, cand_id); in btf_dedup_ref_type()
4224 return -EINVAL; in btf_dedup_ref_type()
4227 d->map[type_id] = new_id; in btf_dedup_ref_type()
4229 return -ENOMEM; in btf_dedup_ref_type()
4238 for (i = 1; i <= d->btf->nr_types; i++) { in btf_dedup_ref_types()
4243 /* we won't need d->dedup_table anymore */ in btf_dedup_ref_types()
4244 hashmap__free(d->dedup_table); in btf_dedup_ref_types()
4245 d->dedup_table = NULL; in btf_dedup_ref_types()
4255 * duplicates. During compaction btf_dedup->hypot_map array is reused to store
4268 d->hypot_map[0] = 0; in btf_dedup_compact_types()
4269 for (i = 1; i <= d->btf->nr_types; i++) in btf_dedup_compact_types()
4270 d->hypot_map[i] = BTF_UNPROCESSED_ID; in btf_dedup_compact_types()
4272 p = d->btf->types_data; in btf_dedup_compact_types()
4274 for (i = 1; i <= d->btf->nr_types; i++) { in btf_dedup_compact_types()
4275 if (d->map[i] != i) in btf_dedup_compact_types()
4278 len = btf_type_size(btf__type_by_id(d->btf, i)); in btf_dedup_compact_types()
4282 memmove(p, btf__type_by_id(d->btf, i), len); in btf_dedup_compact_types()
4283 d->hypot_map[i] = next_type_id; in btf_dedup_compact_types()
4284 d->btf->type_offs[next_type_id] = p - d->btf->types_data; in btf_dedup_compact_types()
4290 d->btf->nr_types = next_type_id - 1; in btf_dedup_compact_types()
4291 d->btf->type_offs_cap = d->btf->nr_types + 1; in btf_dedup_compact_types()
4292 d->btf->hdr->type_len = p - d->btf->types_data; in btf_dedup_compact_types()
4293 new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap, in btf_dedup_compact_types()
4296 return -ENOMEM; in btf_dedup_compact_types()
4297 d->btf->type_offs = new_offs; in btf_dedup_compact_types()
4298 d->btf->hdr->str_off = d->btf->hdr->type_len; in btf_dedup_compact_types()
4299 d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len; in btf_dedup_compact_types()
4306 * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map,
4314 new_type_id = d->hypot_map[resolved_type_id]; in btf_dedup_remap_type_id()
4316 return -EINVAL; in btf_dedup_remap_type_id()
4325 * deduped type ID is stored in btf_dedup->hypot_map and is populated during
4332 struct btf_type *t = btf_type_by_id(d->btf, type_id); in btf_dedup_remap_type()
4348 r = btf_dedup_remap_type_id(d, t->type); in btf_dedup_remap_type()
4351 t->type = r; in btf_dedup_remap_type()
4357 r = btf_dedup_remap_type_id(d, arr_info->type); in btf_dedup_remap_type()
4360 arr_info->type = r; in btf_dedup_remap_type()
4361 r = btf_dedup_remap_type_id(d, arr_info->index_type); in btf_dedup_remap_type()
4364 arr_info->index_type = r; in btf_dedup_remap_type()
4374 r = btf_dedup_remap_type_id(d, member->type); in btf_dedup_remap_type()
4377 member->type = r; in btf_dedup_remap_type()
4387 r = btf_dedup_remap_type_id(d, t->type); in btf_dedup_remap_type()
4390 t->type = r; in btf_dedup_remap_type()
4393 r = btf_dedup_remap_type_id(d, param->type); in btf_dedup_remap_type()
4396 param->type = r; in btf_dedup_remap_type()
4407 r = btf_dedup_remap_type_id(d, var->type); in btf_dedup_remap_type()
4410 var->type = r; in btf_dedup_remap_type()
4417 return -EINVAL; in btf_dedup_remap_type()
4427 for (i = 1; i <= d->btf->nr_types; i++) { in btf_dedup_remap_types()
4436 * Probe few well-known locations for vmlinux kernel image and try to load BTF
4448 { "/boot/vmlinux-%1$s" }, in libbpf_find_kernel_btf()
4449 { "/lib/modules/%1$s/vmlinux-%1$s" }, in libbpf_find_kernel_btf()
4452 { "/usr/lib/debug/boot/vmlinux-%1$s" }, in libbpf_find_kernel_btf()
4453 { "/usr/lib/debug/boot/vmlinux-%1$s.debug" }, in libbpf_find_kernel_btf()
4483 return ERR_PTR(-ESRCH); in libbpf_find_kernel_btf()