Lines Matching +full:libc6 +full:- +full:dev +full:- +full:i386
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
71 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
271 if (err != -EPERM || geteuid() != 0) in pr_perm_msg()
288 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", in pr_perm_msg()
304 fd = -1; \
315 /* as of v1.0 libbpf_set_strict_mode() is a no-op */ in libbpf_set_strict_mode()
361 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
380 /* BPF program support non-linear XDP buffer */
418 * program. For the entry-point (main) BPF program, this is always
419 * zero. For a sub-program, this gets reset before each of main BPF
421 * whether sub-program was already appended to the main program, and
433 * entry-point BPF programs this includes the size of main program
434 * itself plus all the used sub-programs, appended at the end
485 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
591 /* BTF fd index to be patched in for insn->off, this is
592 * 0 for vmlinux BTF, index in obj->fd_array for module
691 /* Path to the custom BTF to be used for BPF CO-RE relocations as an
695 /* vmlinux BTF override for CO-RE relocations */
747 zclose(prog->fd); in bpf_program__unload()
749 zfree(&prog->func_info); in bpf_program__unload()
750 zfree(&prog->line_info); in bpf_program__unload()
759 zfree(&prog->name); in bpf_program__exit()
760 zfree(&prog->sec_name); in bpf_program__exit()
761 zfree(&prog->insns); in bpf_program__exit()
762 zfree(&prog->reloc_desc); in bpf_program__exit()
764 prog->nr_reloc = 0; in bpf_program__exit()
765 prog->insns_cnt = 0; in bpf_program__exit()
766 prog->sec_idx = -1; in bpf_program__exit()
771 return BPF_CLASS(insn->code) == BPF_JMP && in insn_is_subprog_call()
772 BPF_OP(insn->code) == BPF_CALL && in insn_is_subprog_call()
773 BPF_SRC(insn->code) == BPF_K && in insn_is_subprog_call()
774 insn->src_reg == BPF_PSEUDO_CALL && in insn_is_subprog_call()
775 insn->dst_reg == 0 && in insn_is_subprog_call()
776 insn->off == 0; in insn_is_subprog_call()
781 return insn->code == (BPF_JMP | BPF_CALL); in is_call_insn()
786 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; in insn_is_pseudo_func()
797 return -EINVAL; in bpf_object__init_prog()
801 prog->obj = obj; in bpf_object__init_prog()
803 prog->sec_idx = sec_idx; in bpf_object__init_prog()
804 prog->sec_insn_off = sec_off / BPF_INSN_SZ; in bpf_object__init_prog()
805 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; in bpf_object__init_prog()
807 prog->insns_cnt = prog->sec_insn_cnt; in bpf_object__init_prog()
809 prog->type = BPF_PROG_TYPE_UNSPEC; in bpf_object__init_prog()
810 prog->fd = -1; in bpf_object__init_prog()
811 prog->exception_cb_idx = -1; in bpf_object__init_prog()
818 prog->autoload = false; in bpf_object__init_prog()
822 prog->autoload = true; in bpf_object__init_prog()
825 prog->autoattach = true; in bpf_object__init_prog()
828 prog->log_level = obj->log_level; in bpf_object__init_prog()
830 prog->sec_name = strdup(sec_name); in bpf_object__init_prog()
831 if (!prog->sec_name) in bpf_object__init_prog()
834 prog->name = strdup(name); in bpf_object__init_prog()
835 if (!prog->name) in bpf_object__init_prog()
838 prog->insns = malloc(insn_data_sz); in bpf_object__init_prog()
839 if (!prog->insns) in bpf_object__init_prog()
841 memcpy(prog->insns, insn_data, insn_data_sz); in bpf_object__init_prog()
847 return -ENOMEM; in bpf_object__init_prog()
854 Elf_Data *symbols = obj->efile.symbols; in bpf_object__add_programs()
856 void *data = sec_data->d_buf; in bpf_object__add_programs()
857 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; in bpf_object__add_programs()
862 progs = obj->programs; in bpf_object__add_programs()
863 nr_progs = obj->nr_programs; in bpf_object__add_programs()
864 nr_syms = symbols->d_size / sizeof(Elf64_Sym); in bpf_object__add_programs()
869 if (sym->st_shndx != sec_idx) in bpf_object__add_programs()
871 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) in bpf_object__add_programs()
874 prog_sz = sym->st_size; in bpf_object__add_programs()
875 sec_off = sym->st_value; in bpf_object__add_programs()
877 name = elf_sym_str(obj, sym->st_name); in bpf_object__add_programs()
881 return -LIBBPF_ERRNO__FORMAT; in bpf_object__add_programs()
887 return -LIBBPF_ERRNO__FORMAT; in bpf_object__add_programs()
890 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { in bpf_object__add_programs()
892 return -ENOTSUP; in bpf_object__add_programs()
901 * In this case the original obj->programs in bpf_object__add_programs()
907 return -ENOMEM; in bpf_object__add_programs()
909 obj->programs = progs; in bpf_object__add_programs()
918 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL) in bpf_object__add_programs()
919 prog->sym_global = true; in bpf_object__add_programs()
926 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN in bpf_object__add_programs()
927 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) in bpf_object__add_programs()
928 prog->mark_btf_static = true; in bpf_object__add_programs()
931 obj->nr_programs = nr_progs; in bpf_object__add_programs()
959 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) in find_member_by_name()
1011 if (kern_data_member->type == kern_type_id) in find_struct_ops_kern_types()
1017 return -EINVAL; in find_struct_ops_kern_types()
1031 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; in bpf_map__is_struct_ops()
1047 st_ops = map->st_ops; in bpf_map__init_kern_struct_ops()
1048 type = st_ops->type; in bpf_map__init_kern_struct_ops()
1049 tname = st_ops->tname; in bpf_map__init_kern_struct_ops()
1058 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); in bpf_map__init_kern_struct_ops()
1060 map->def.value_size = kern_vtype->size; in bpf_map__init_kern_struct_ops()
1061 map->btf_vmlinux_value_type_id = kern_vtype_id; in bpf_map__init_kern_struct_ops()
1063 st_ops->kern_vdata = calloc(1, kern_vtype->size); in bpf_map__init_kern_struct_ops()
1064 if (!st_ops->kern_vdata) in bpf_map__init_kern_struct_ops()
1065 return -ENOMEM; in bpf_map__init_kern_struct_ops()
1067 data = st_ops->data; in bpf_map__init_kern_struct_ops()
1068 kern_data_off = kern_data_member->offset / 8; in bpf_map__init_kern_struct_ops()
1069 kern_data = st_ops->kern_vdata + kern_data_off; in bpf_map__init_kern_struct_ops()
1081 mname = btf__name_by_offset(btf, member->name_off); in bpf_map__init_kern_struct_ops()
1085 map->name, mname); in bpf_map__init_kern_struct_ops()
1086 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1089 kern_member_idx = kern_member - btf_members(kern_type); in bpf_map__init_kern_struct_ops()
1093 map->name, mname); in bpf_map__init_kern_struct_ops()
1094 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1097 moff = member->offset / 8; in bpf_map__init_kern_struct_ops()
1098 kern_moff = kern_member->offset / 8; in bpf_map__init_kern_struct_ops()
1103 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); in bpf_map__init_kern_struct_ops()
1104 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, in bpf_map__init_kern_struct_ops()
1106 if (BTF_INFO_KIND(mtype->info) != in bpf_map__init_kern_struct_ops()
1107 BTF_INFO_KIND(kern_mtype->info)) { in bpf_map__init_kern_struct_ops()
1109 map->name, mname, BTF_INFO_KIND(mtype->info), in bpf_map__init_kern_struct_ops()
1110 BTF_INFO_KIND(kern_mtype->info)); in bpf_map__init_kern_struct_ops()
1111 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1117 prog = st_ops->progs[i]; in bpf_map__init_kern_struct_ops()
1122 kern_mtype->type, in bpf_map__init_kern_struct_ops()
1125 /* mtype->type must be a func_proto which was in bpf_map__init_kern_struct_ops()
1131 map->name, mname); in bpf_map__init_kern_struct_ops()
1132 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1135 prog->attach_btf_id = kern_type_id; in bpf_map__init_kern_struct_ops()
1136 prog->expected_attach_type = kern_member_idx; in bpf_map__init_kern_struct_ops()
1138 st_ops->kern_func_off[i] = kern_data_off + kern_moff; in bpf_map__init_kern_struct_ops()
1141 map->name, mname, prog->name, moff, in bpf_map__init_kern_struct_ops()
1151 map->name, mname, (ssize_t)msize, in bpf_map__init_kern_struct_ops()
1153 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1157 map->name, mname, (unsigned int)msize, in bpf_map__init_kern_struct_ops()
1171 for (i = 0; i < obj->nr_maps; i++) { in bpf_object__init_kern_struct_ops_maps()
1172 map = &obj->maps[i]; in bpf_object__init_kern_struct_ops_maps()
1177 err = bpf_map__init_kern_struct_ops(map, obj->btf, in bpf_object__init_kern_struct_ops_maps()
1178 obj->btf_vmlinux); in bpf_object__init_kern_struct_ops_maps()
1198 if (shndx == -1) in init_struct_ops_maps()
1201 btf = obj->btf; in init_struct_ops_maps()
1207 return -EINVAL; in init_struct_ops_maps()
1213 type = btf__type_by_id(obj->btf, vsi->type); in init_struct_ops_maps()
1214 var_name = btf__name_by_offset(obj->btf, type->name_off); in init_struct_ops_maps()
1216 type_id = btf__resolve_type(obj->btf, vsi->type); in init_struct_ops_maps()
1219 vsi->type, sec_name); in init_struct_ops_maps()
1220 return -EINVAL; in init_struct_ops_maps()
1223 type = btf__type_by_id(obj->btf, type_id); in init_struct_ops_maps()
1224 tname = btf__name_by_offset(obj->btf, type->name_off); in init_struct_ops_maps()
1227 return -ENOTSUP; in init_struct_ops_maps()
1231 return -EINVAL; in init_struct_ops_maps()
1238 map->sec_idx = shndx; in init_struct_ops_maps()
1239 map->sec_offset = vsi->offset; in init_struct_ops_maps()
1240 map->name = strdup(var_name); in init_struct_ops_maps()
1241 if (!map->name) in init_struct_ops_maps()
1242 return -ENOMEM; in init_struct_ops_maps()
1244 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; in init_struct_ops_maps()
1245 map->def.key_size = sizeof(int); in init_struct_ops_maps()
1246 map->def.value_size = type->size; in init_struct_ops_maps()
1247 map->def.max_entries = 1; in init_struct_ops_maps()
1248 map->def.map_flags = map_flags; in init_struct_ops_maps()
1250 map->st_ops = calloc(1, sizeof(*map->st_ops)); in init_struct_ops_maps()
1251 if (!map->st_ops) in init_struct_ops_maps()
1252 return -ENOMEM; in init_struct_ops_maps()
1253 st_ops = map->st_ops; in init_struct_ops_maps()
1254 st_ops->data = malloc(type->size); in init_struct_ops_maps()
1255 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); in init_struct_ops_maps()
1256 st_ops->kern_func_off = malloc(btf_vlen(type) * in init_struct_ops_maps()
1257 sizeof(*st_ops->kern_func_off)); in init_struct_ops_maps()
1258 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) in init_struct_ops_maps()
1259 return -ENOMEM; in init_struct_ops_maps()
1261 if (vsi->offset + type->size > data->d_size) { in init_struct_ops_maps()
1264 return -EINVAL; in init_struct_ops_maps()
1267 memcpy(st_ops->data, in init_struct_ops_maps()
1268 data->d_buf + vsi->offset, in init_struct_ops_maps()
1269 type->size); in init_struct_ops_maps()
1270 st_ops->tname = tname; in init_struct_ops_maps()
1271 st_ops->type = type; in init_struct_ops_maps()
1272 st_ops->type_id = type_id; in init_struct_ops_maps()
1275 tname, type_id, var_name, vsi->offset); in init_struct_ops_maps()
1285 err = init_struct_ops_maps(obj, STRUCT_OPS_SEC, obj->efile.st_ops_shndx, in bpf_object_init_struct_ops()
1286 obj->efile.st_ops_data, 0); in bpf_object_init_struct_ops()
1288 obj->efile.st_ops_link_shndx, in bpf_object_init_struct_ops()
1289 obj->efile.st_ops_link_data, in bpf_object_init_struct_ops()
1305 return ERR_PTR(-ENOMEM); in bpf_object__new()
1308 strcpy(obj->path, path); in bpf_object__new()
1310 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); in bpf_object__new()
1313 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); in bpf_object__new()
1314 end = strchr(obj->name, '.'); in bpf_object__new()
1319 obj->efile.fd = -1; in bpf_object__new()
1326 obj->efile.obj_buf = obj_buf; in bpf_object__new()
1327 obj->efile.obj_buf_sz = obj_buf_sz; in bpf_object__new()
1328 obj->efile.btf_maps_shndx = -1; in bpf_object__new()
1329 obj->efile.st_ops_shndx = -1; in bpf_object__new()
1330 obj->efile.st_ops_link_shndx = -1; in bpf_object__new()
1331 obj->kconfig_map_idx = -1; in bpf_object__new()
1333 obj->kern_version = get_kernel_version(); in bpf_object__new()
1334 obj->loaded = false; in bpf_object__new()
1341 if (!obj->efile.elf) in bpf_object__elf_finish()
1344 elf_end(obj->efile.elf); in bpf_object__elf_finish()
1346 if (obj->efile.shstring) { in bpf_object__elf_finish()
1347 elfio_string_section_accessor_delete(obj->efile.shstring); in bpf_object__elf_finish()
1349 if (obj->efile.strstring) { in bpf_object__elf_finish()
1350 elfio_string_section_accessor_delete(obj->efile.strstring); in bpf_object__elf_finish()
1352 elfio_delete(obj->efile.elf); in bpf_object__elf_finish()
1354 obj->efile.elf = NULL; in bpf_object__elf_finish()
1355 obj->efile.symbols = NULL; in bpf_object__elf_finish()
1356 obj->efile.st_ops_data = NULL; in bpf_object__elf_finish()
1357 obj->efile.st_ops_link_data = NULL; in bpf_object__elf_finish()
1359 zfree(&obj->efile.secs); in bpf_object__elf_finish()
1360 obj->efile.sec_cnt = 0; in bpf_object__elf_finish()
1361 zclose(obj->efile.fd); in bpf_object__elf_finish()
1362 obj->efile.obj_buf = NULL; in bpf_object__elf_finish()
1363 obj->efile.obj_buf_sz = 0; in bpf_object__elf_finish()
1376 if (obj->efile.elf) {
1378 return -LIBBPF_ERRNO__LIBELF;
1381 if (obj->efile.obj_buf_sz > 0) {
1384 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1389 ftruncate(fdm, obj->efile.obj_buf_sz);
1390 write(fdm, (char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1395 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1396 if (obj->efile.fd < 0) {
1399 err = -errno;
1401 pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1405 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1410 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1411 err = -LIBBPF_ERRNO__LIBELF;
1415 obj->efile.elf = elf;
1418 err = -LIBBPF_ERRNO__FORMAT;
1419 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1427 err = -LIBBPF_ERRNO__FORMAT;
1428 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1432 obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1434 obj->efile.ehdr = ehdr = (Elf64_Ehdr*)obj->efile.obj_buf;
1436 if (!obj->efile.ehdr) {
1437 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1438 err = -LIBBPF_ERRNO__FORMAT;
1443 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1445 obj->path, elf_errmsg(-1));
1446 err = -LIBBPF_ERRNO__FORMAT;
1451 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1453 obj->path, elf_errmsg(-1));
1454 err = -LIBBPF_ERRNO__FORMAT;
1458 obj->efile.shstrndx = elfio_get_section_name_str_index(elf);
1461 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1462 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1463 err = -LIBBPF_ERRNO__FORMAT;
1476 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1479 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1484 pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1485 return -LIBBPF_ERRNO__ENDIAN;
1492 pr_warn("invalid license section in %s\n", obj->path);
1493 return -LIBBPF_ERRNO__FORMAT;
1495 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1498 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1499 pr_debug("license of %s is %s\n", obj->path, obj->license);
1509 pr_warn("invalid kver section in %s\n", obj->path);
1510 return -LIBBPF_ERRNO__FORMAT;
1513 obj->kern_version = kver;
1514 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1534 return -EINVAL;
1544 *size = data->d_size;
1548 return -ENOENT;
1553 Elf_Data *symbols = obj->efile.symbols;
1557 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1560 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1563 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1564 ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1567 sname = elf_sym_str(obj, sym->st_name);
1570 return ERR_PTR(-EIO);
1576 return ERR_PTR(-ENOENT);
1584 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1585 sizeof(*obj->maps), obj->nr_maps + 1);
1589 map = &obj->maps[obj->nr_maps++];
1590 map->obj = obj;
1591 map->fd = -1;
1592 map->inner_map_fd = -1;
1593 map->autocreate = true;
1612 if (!map->mmaped)
1613 return -EINVAL;
1618 mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1620 return -errno;
1622 memcpy(mmaped, map->mmaped, min(old_sz, new_sz));
1623 munmap(map->mmaped, old_sz);
1624 map->mmaped = mmaped;
1658 * '.rodata.abracad' kernel and user-visible name.
1667 sfx_len = BPF_OBJ_NAME_LEN - 1;
1673 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1675 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1700 if (!map->btf_value_type_id)
1703 t = btf__type_by_id(obj->btf, map->btf_value_type_id);
1709 vt = btf__type_by_id(obj->btf, vsi->type);
1713 if (btf_var(vt)->linkage != BTF_VAR_STATIC)
1733 map->libbpf_type = type;
1734 map->sec_idx = sec_idx;
1735 map->sec_offset = 0;
1736 map->real_name = strdup(real_name);
1737 map->name = internal_map_name(obj, real_name);
1738 if (!map->real_name || !map->name) {
1739 zfree(&map->real_name);
1740 zfree(&map->name);
1741 return -ENOMEM;
1744 def = &map->def;
1745 def->type = BPF_MAP_TYPE_ARRAY;
1746 def->key_size = sizeof(int);
1747 def->value_size = data_sz;
1748 def->max_entries = 1;
1749 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1756 def->map_flags |= BPF_F_MMAPABLE;
1759 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1761 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
1762 map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1763 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1764 if (map->mmaped == MAP_FAILED) {
1765 err = -errno;
1766 map->mmaped = NULL;
1768 map->name, err);
1769 zfree(&map->real_name);
1770 zfree(&map->name);
1775 memcpy(map->mmaped, data, data_sz);
1777 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1788 * Populate obj->maps with libbpf internal maps.
1790 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
1791 sec_desc = &obj->efile.secs[sec_idx];
1794 if (!sec_desc->data || sec_desc->data->d_size == 0)
1797 switch (sec_desc->sec_type) {
1806 sec_desc->data->d_buf,
1807 sec_desc->data->d_size);
1810 obj->has_rodata = true;
1818 sec_desc->data->d_buf,
1819 sec_desc->data->d_size);
1830 sec_desc->data->d_size);
1848 for (i = 0; i < obj->nr_extern; i++) {
1849 if (strcmp(obj->externs[i].name, name) == 0)
1850 return &obj->externs[i];
1858 switch (ext->kcfg.type) {
1862 ext->name, value);
1863 return -EINVAL;
1883 ext->name, value);
1884 return -EINVAL;
1886 ext->is_set = true;
1895 if (ext->kcfg.type != KCFG_CHAR_ARR) {
1897 ext->name, value);
1898 return -EINVAL;
1902 if (value[len - 1] != '"') {
1904 ext->name, value);
1905 return -EINVAL;
1909 len -= 2;
1910 if (len >= ext->kcfg.sz) {
1912 ext->name, value, len, ext->kcfg.sz - 1);
1913 len = ext->kcfg.sz - 1;
1917 ext->is_set = true;
1929 err = -errno;
1935 return -EINVAL;
1942 int bit_sz = ext->kcfg.sz * 8;
1944 if (ext->kcfg.sz == 8)
1947 /* Validate that value stored in u64 fits in integer of `ext->sz`
1952 * -2^(Y-1) <= X <= 2^(Y-1) - 1
1953 * 0 <= X + 2^(Y-1) <= 2^Y - 1
1954 * 0 <= X + 2^(Y-1) < 2^Y
1956 * For unsigned target integer, check that all the (64 - Y) bits are
1959 if (ext->kcfg.is_signed)
1960 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1968 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
1969 ext->kcfg.type != KCFG_BOOL) {
1971 ext->name, (unsigned long long)value);
1972 return -EINVAL;
1974 if (ext->kcfg.type == KCFG_BOOL && value > 1) {
1976 ext->name, (unsigned long long)value);
1977 return -EINVAL;
1982 ext->name, (unsigned long long)value, ext->kcfg.sz);
1983 return -ERANGE;
1985 switch (ext->kcfg.sz) {
1999 return -EINVAL;
2001 ext->is_set = true;
2020 return -EINVAL;
2025 if (buf[len - 1] == '\n')
2026 buf[len - 1] = '\0';
2032 return -EINVAL;
2036 if (!ext || ext->is_set)
2039 ext_val = data + ext->kcfg.data_off;
2053 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
2056 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
2057 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
2058 return -EINVAL;
2065 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
2077 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
2079 return -EINVAL;
2081 return -ENAMETOOLONG;
2090 return -ENOENT;
2116 err = -errno;
2117 pr_warn("failed to open in-memory Kconfig: %d\n", err);
2124 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
2140 for (i = 0; i < obj->nr_extern; i++) {
2141 ext = &obj->externs[i];
2142 if (ext->type == EXT_KCFG)
2149 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
2151 ".kconfig", obj->efile.symbols_shndx,
2156 obj->kconfig_map_idx = obj->nr_maps - 1;
2171 *res_id = t->type;
2172 t = btf__type_by_id(btf, t->type);
2187 t = skip_mods_and_typedefs(btf, t->type, res_id);
2234 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2235 const char *name = btf__name_by_offset(btf, m->name_off);
2245 arr_t = btf__type_by_id(btf, t->type);
2248 map_name, name, t->type);
2257 *res = arr_info->nelems;
2267 return -EINVAL;
2269 return -ENAMETOOLONG;
2308 const char *name = btf__name_by_offset(btf, m->name_off);
2312 return -EINVAL;
2315 if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2316 return -EINVAL;
2317 map_def->parts |= MAP_DEF_MAP_TYPE;
2319 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2320 return -EINVAL;
2321 map_def->parts |= MAP_DEF_MAX_ENTRIES;
2323 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2324 return -EINVAL;
2325 map_def->parts |= MAP_DEF_MAP_FLAGS;
2327 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2328 return -EINVAL;
2329 map_def->parts |= MAP_DEF_NUMA_NODE;
2334 return -EINVAL;
2335 if (map_def->key_size && map_def->key_size != sz) {
2337 map_name, map_def->key_size, sz);
2338 return -EINVAL;
2340 map_def->key_size = sz;
2341 map_def->parts |= MAP_DEF_KEY_SIZE;
2345 t = btf__type_by_id(btf, m->type);
2348 map_name, m->type);
2349 return -EINVAL;
2354 return -EINVAL;
2356 sz = btf__resolve_size(btf, t->type);
2359 map_name, t->type, (ssize_t)sz);
2362 if (map_def->key_size && map_def->key_size != sz) {
2364 map_name, map_def->key_size, (ssize_t)sz);
2365 return -EINVAL;
2367 map_def->key_size = sz;
2368 map_def->key_type_id = t->type;
2369 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2374 return -EINVAL;
2375 if (map_def->value_size && map_def->value_size != sz) {
2377 map_name, map_def->value_size, sz);
2378 return -EINVAL;
2380 map_def->value_size = sz;
2381 map_def->parts |= MAP_DEF_VALUE_SIZE;
2385 t = btf__type_by_id(btf, m->type);
2388 map_name, m->type);
2389 return -EINVAL;
2394 return -EINVAL;
2396 sz = btf__resolve_size(btf, t->type);
2399 map_name, t->type, (ssize_t)sz);
2402 if (map_def->value_size && map_def->value_size != sz) {
2404 map_name, map_def->value_size, (ssize_t)sz);
2405 return -EINVAL;
2407 map_def->value_size = sz;
2408 map_def->value_type_id = t->type;
2409 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2412 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2413 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2414 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2419 pr_warn("map '%s': multi-level inner maps not supported.\n",
2421 return -ENOTSUP;
2423 if (i != vlen - 1) {
2426 return -EINVAL;
2429 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2431 return -ENOTSUP;
2433 if (map_def->value_size && map_def->value_size != 4) {
2435 map_name, map_def->value_size);
2436 return -EINVAL;
2438 map_def->value_size = 4;
2439 t = btf__type_by_id(btf, m->type);
2442 map_name, desc, m->type);
2443 return -EINVAL;
2445 if (!btf_is_array(t) || btf_array(t)->nelems) {
2446 pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2448 return -EINVAL;
2450 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2454 return -EINVAL;
2456 t = skip_mods_and_typedefs(btf, t->type, NULL);
2459 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2461 return -EINVAL;
2466 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2468 return -EINVAL;
2476 map_def->parts |= MAP_DEF_INNER_MAP;
2482 return -EINVAL;
2485 return -EINVAL;
2489 return -EINVAL;
2491 map_def->pinning = val;
2492 map_def->parts |= MAP_DEF_PINNING;
2497 return -EINVAL;
2498 map_def->map_extra = map_extra;
2499 map_def->parts |= MAP_DEF_MAP_EXTRA;
2503 return -ENOTSUP;
2509 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2511 return -EINVAL;
2526 * a power-of-2 multiple of kernel's page size. If user diligently
2533 * user-set size to satisfy both user size request and kernel
2542 * very close to UINT_MAX but is not a power-of-2 multiple of
2550 return map->def.type == BPF_MAP_TYPE_RINGBUF ||
2551 map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2556 map->def.type = def->map_type;
2557 map->def.key_size = def->key_size;
2558 map->def.value_size = def->value_size;
2559 map->def.max_entries = def->max_entries;
2560 map->def.map_flags = def->map_flags;
2561 map->map_extra = def->map_extra;
2563 map->numa_node = def->numa_node;
2564 map->btf_key_type_id = def->key_type_id;
2565 map->btf_value_type_id = def->value_type_id;
2567 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2569 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2571 if (def->parts & MAP_DEF_MAP_TYPE)
2572 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2574 if (def->parts & MAP_DEF_KEY_TYPE)
2576 map->name, def->key_type_id, def->key_size);
2577 else if (def->parts & MAP_DEF_KEY_SIZE)
2578 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2580 if (def->parts & MAP_DEF_VALUE_TYPE)
2582 map->name, def->value_type_id, def->value_size);
2583 else if (def->parts & MAP_DEF_VALUE_SIZE)
2584 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2586 if (def->parts & MAP_DEF_MAX_ENTRIES)
2587 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2588 if (def->parts & MAP_DEF_MAP_FLAGS)
2589 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2590 if (def->parts & MAP_DEF_MAP_EXTRA)
2591 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2592 (unsigned long long)def->map_extra);
2593 if (def->parts & MAP_DEF_PINNING)
2594 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2595 if (def->parts & MAP_DEF_NUMA_NODE)
2596 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2598 if (def->parts & MAP_DEF_INNER_MAP)
2599 pr_debug("map '%s': found inner map definition.\n", map->name);
2627 var = btf__type_by_id(obj->btf, vi->type);
2629 map_name = btf__name_by_offset(obj->btf, var->name_off);
2633 return -EINVAL;
2635 if ((__u64)vi->offset + vi->size > data->d_size) {
2637 return -EINVAL;
2642 return -EINVAL;
2644 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2646 map_name, btf_var_linkage_str(var_extra->linkage));
2647 return -EOPNOTSUPP;
2650 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2654 return -EINVAL;
2656 if (def->size > vi->size) {
2658 return -EINVAL;
2664 map->name = strdup(map_name);
2665 if (!map->name) {
2667 return -ENOMEM;
2669 map->libbpf_type = LIBBPF_MAP_UNSPEC;
2670 map->def.type = BPF_MAP_TYPE_UNSPEC;
2671 map->sec_idx = sec_idx;
2672 map->sec_offset = vi->offset;
2673 map->btf_var_idx = var_idx;
2675 map_name, map->sec_idx, map->sec_offset);
2677 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2686 pr_warn("map '%s': couldn't build pin path.\n", map->name);
2692 map->inner_map = calloc(1, sizeof(*map->inner_map));
2693 if (!map->inner_map)
2694 return -ENOMEM;
2695 map->inner_map->fd = -1;
2696 map->inner_map->sec_idx = sec_idx;
2697 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2698 if (!map->inner_map->name)
2699 return -ENOMEM;
2700 sprintf(map->inner_map->name, "%s.inner", map_name);
2702 fill_map_from_def(map->inner_map, &inner_def);
2724 if (obj->efile.btf_maps_shndx < 0)
2727 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2732 data = elf_sec_data_by_idx(obj, obj->efile.btf_maps_shndx, &realdata);
2736 MAPS_ELF_SEC, obj->path);
2737 return -EINVAL;
2740 nr_types = btf__type_cnt(obj->btf);
2742 t = btf__type_by_id(obj->btf, i);
2745 name = btf__name_by_offset(obj->btf, t->name_off);
2748 obj->efile.btf_maps_sec_btf_id = i;
2755 return -ENOENT;
2761 obj->efile.btf_maps_shndx,
2801 return sh->sh_flags & SHF_EXECINSTR;
2836 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2842 t->size = 1;
2851 name = (char *)btf__name_by_offset(btf, t->name_off);
2859 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2862 m->offset = v->offset * 8;
2863 m->type = v->type;
2865 vt = (void *)btf__type_by_id(btf, v->type);
2866 m->name_off = vt->name_off;
2871 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2872 t->size = sizeof(__u32); /* kernel enforced */
2875 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2878 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2880 /* replace FLOAT with an equally-sized empty STRUCT;
2884 t->name_off = 0;
2885 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
2888 t->name_off = 0;
2889 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
2892 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
2907 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
2909 m->type = enum64_placeholder_id;
2910 m->offset = 0;
2920 return obj->efile.btf_maps_shndx >= 0 ||
2921 obj->efile.st_ops_shndx >= 0 ||
2922 obj->efile.st_ops_link_shndx >= 0 ||
2923 obj->nr_extern > 0;
2928 return obj->efile.st_ops_shndx >= 0 || obj->efile.st_ops_link_shndx >= 0;
2935 int err = -ENOENT;
2938 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2939 err = libbpf_get_error(obj->btf);
2941 obj->btf = NULL;
2945 /* enforce 8-byte pointers for BPF-targeted BTFs */
2946 btf__set_pointer_size(obj->btf, 8);
2952 if (!obj->btf) {
2957 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
2958 err = libbpf_get_error(obj->btf_ext);
2962 obj->btf_ext = NULL;
2967 ext_segs[0] = &obj->btf_ext->func_info;
2968 ext_segs[1] = &obj->btf_ext->line_info;
2969 ext_segs[2] = &obj->btf_ext->core_relo_info;
2980 if (seg->sec_cnt == 0)
2983 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
2984 if (!seg->sec_idxs) {
2985 err = -ENOMEM;
2996 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
3004 pelfio_t elf = obj->efile.elf;
3010 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
3012 seg->sec_idxs[sec_num - 1] = elfio_section_get_index(sec_obj);
3030 return a->offset - b->offset;
3037 const char *sec_name = btf__name_by_offset(btf, t->name_off);
3044 return -ENOENT;
3047 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and
3059 * to be optional. But the STV_HIDDEN handling is non-optional for any
3060 * non-extern DATASEC, so the variable fixup loop below handles both
3061 * functions at the same time, paying the cost of BTF VAR <-> ELF
3064 if (t->size == 0) {
3069 return -ENOENT;
3072 t->size = size;
3082 t_var = btf__type_by_id(btf, vsi->type);
3084 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);
3085 return -EINVAL;
3089 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)
3092 var_name = btf__name_by_offset(btf, t_var->name_off);
3096 return -ENOENT;
3103 return -ENOENT;
3107 vsi->offset = sym->st_value;
3116 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
3117 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)
3118 var->linkage = BTF_VAR_STATIC;
3130 if (!obj->btf)
3133 n = btf__type_cnt(obj->btf);
3135 struct btf_type *t = btf_type_by_id(obj->btf, i);
3143 err = btf_fixup_datasec(obj, obj->btf, t);
3154 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3155 prog->type == BPF_PROG_TYPE_LSM)
3161 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3172 /* CO-RE relocations need kernel BTF, only when btf_custom_path
3175 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
3179 for (i = 0; i < obj->nr_extern; i++) {
3182 ext = &obj->externs[i];
3183 if (ext->type == EXT_KSYM && ext->ksym.type_id)
3188 if (!prog->autoload)
3202 if (obj->btf_vmlinux || obj->gen_loader)
3208 obj->btf_vmlinux = btf__load_vmlinux_btf();
3209 err = libbpf_get_error(obj->btf_vmlinux);
3212 obj->btf_vmlinux = NULL;
3220 struct btf *kern_btf = obj->btf;
3224 if (!obj->btf)
3229 err = -EOPNOTSUPP;
3244 for (i = 0; i < obj->nr_programs; i++) {
3245 struct bpf_program *prog = &obj->programs[i];
3250 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3253 n = btf__type_cnt(obj->btf);
3255 t = btf_type_by_id(obj->btf, j);
3259 name = btf__str_by_offset(obj->btf, t->name_off);
3260 if (strcmp(name, prog->name) != 0)
3263 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3270 for (i = 0; i < obj->nr_programs; i++) {
3271 struct bpf_program *prog = &obj->programs[i];
3276 n = btf__type_cnt(obj->btf);
3282 t = btf_type_by_id(obj->btf, j);
3283 if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1)
3286 name = btf__str_by_offset(obj->btf, t->name_off);
3290 t = btf_type_by_id(obj->btf, t->type);
3293 prog->name);
3294 return -EINVAL;
3296 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)))
3302 if (prog->exception_cb_idx >= 0) {
3303 prog->exception_cb_idx = -1;
3310 prog->name);
3311 return -EINVAL;
3314 for (k = 0; k < obj->nr_programs; k++) {
3315 struct bpf_program *subprog = &obj->programs[k];
3319 if (strcmp(name, subprog->name))
3321 /* Enforce non-hidden, as from verifier point of
3325 if (!subprog->sym_global || subprog->mark_btf_static) {
3326 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n",
3327 prog->name, subprog->name);
3328 return -EINVAL;
3331 if (prog->exception_cb_idx >= 0) {
3333 prog->name, subprog->name);
3334 return -EINVAL;
3336 prog->exception_cb_idx = k;
3340 if (prog->exception_cb_idx >= 0)
3342 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name);
3343 return -ENOENT;
3354 raw_data = btf__raw_data(obj->btf, &sz);
3360 /* enforce 8-byte pointers for BPF-targeted BTFs */
3361 btf__set_pointer_size(obj->btf, 8);
3367 if (obj->gen_loader) {
3372 return -ENOMEM;
3373 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3375 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3380 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3381 obj->log_level ? 1 : 0);
3386 btf__set_fd(obj->btf, btf__fd(kern_btf));
3387 btf__set_fd(kern_btf, -1);
3407 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3409 name = elfio_string_get_string(obj->efile.strstring, off);
3413 off, obj->path, elf_errmsg(-1));
3424 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3426 name = elfio_string_get_string(obj->efile.shstring, off);
3431 off, obj->path, elf_errmsg(-1));
3443 scn = elf_getscn(obj->efile.elf, idx);
3446 idx, obj->path, elf_errmsg(-1));
3455 Elf *elf = obj->efile.elf;
3481 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3500 name = elf_sec_str(obj, sh->sh_name);
3503 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3512 psection_t psection = elfio_get_section_by_index(obj->efile.elf, idx);
3514 sheader->sh_name = elfio_section_get_name_string_offset(psection);
3515 sheader->sh_type = elfio_section_get_type(psection);
3516 sheader->sh_flags = elfio_section_get_flags(psection);
3517 sheader->sh_addr = elfio_section_get_address(psection);
3518 sheader->sh_offset = elfio_section_get_offset(psection);
3519 sheader->sh_size = elfio_section_get_size(psection);
3520 sheader->sh_link = elfio_section_get_link(psection);
3521 sheader->sh_info = elfio_section_get_info(psection);
3522 sheader->sh_addralign = elfio_section_get_addr_align(psection);
3523 sheader->sh_entsize = elfio_section_get_entry_size(psection);
3538 idx, obj->path, elf_errmsg(-1));
3558 obj->path, elf_errmsg(-1));
3567 pelfio_t elf = obj->efile.elf;
3569 data->d_buf = (void*)elfio_section_get_data(psection_name);
3570 data->d_size = elfio_section_get_size(psection_name);
3577 pelfio_t elf = obj->efile.elf;
3579 data->d_buf = (void*)elfio_section_get_data(psection_index);
3580 data->d_size = elfio_section_get_size(psection_index);
3588 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3591 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3596 if (idx >= data->d_size / sizeof(Elf64_Rel))
3599 return (Elf64_Rel *)data->d_buf + idx;
3611 if (hdr->sh_type == SHT_STRTAB)
3615 if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3619 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3628 name += sizeof(".rel") - 1;
3647 if (a->sec_idx != b->sec_idx)
3648 return a->sec_idx < b->sec_idx ? -1 : 1;
3651 return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3658 Elf *elf = obj->efile.elf;
3660 pelfio_t elf = obj->efile.elf;
3676 /* ELF section indices are 0-based, but sec #0 is special "invalid"
3682 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3684 obj->path, elf_errmsg(-1));
3685 return -LIBBPF_ERRNO__FORMAT;
3688 obj->efile.sec_cnt = elfio_get_sections_num(elf);
3690 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3691 if (!obj->efile.secs)
3692 return -ENOMEM;
3708 return -LIBBPF_ERRNO__FORMAT;
3710 if (sh->sh_type == SHT_SYMTAB) {
3711 if (obj->efile.symbols) {
3712 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3713 return -LIBBPF_ERRNO__FORMAT;
3721 return -LIBBPF_ERRNO__FORMAT;
3727 obj->efile.symbols = data;
3729 obj->efile.realsymbols.d_buf = data->d_buf;
3730 obj->efile.realsymbols.d_size = data->d_size;
3731 obj->efile.symbols = &(obj->efile.realsymbols);
3735 obj->efile.symbols_shndx = idx;
3737 obj->efile.symbols_shndx = i;
3739 obj->efile.strtabidx = sh->sh_link;
3747 psection_t psection = elfio_get_section_by_index(elf, obj->efile.strtabidx);
3749 return -LIBBPF_ERRNO__FORMAT;
3752 psection = elfio_get_section_by_index(elf, obj->efile.shstrndx);
3754 return -LIBBPF_ERRNO__FORMAT;
3758 return -LIBBPF_ERRNO__FORMAT;
3759 obj->efile.strstring = strstring;
3760 obj->efile.shstring = shstring;
3763 if (!obj->efile.symbols) {
3765 obj->path);
3766 return -ENOENT;
3783 sec_desc = &obj->efile.secs[idx];
3792 return -LIBBPF_ERRNO__FORMAT;
3794 name = elf_sec_str(obj, sh->sh_name);
3796 return -LIBBPF_ERRNO__FORMAT;
3804 data = elf_sec_data_by_idx(obj, i, &sec_desc->realdata);
3807 return -LIBBPF_ERRNO__FORMAT;
3810 idx, name, (unsigned long)data->d_size,
3811 (int)sh->sh_link, (unsigned long)sh->sh_flags,
3812 (int)sh->sh_type);
3815 err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3819 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3824 return -ENOTSUP;
3826 obj->efile.btf_maps_shndx = idx;
3828 if (sh->sh_type != SHT_PROGBITS)
3829 return -LIBBPF_ERRNO__FORMAT;
3832 if (sh->sh_type != SHT_PROGBITS)
3833 return -LIBBPF_ERRNO__FORMAT;
3835 } else if (sh->sh_type == SHT_SYMTAB) {
3837 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
3838 if (sh->sh_flags & SHF_EXECINSTR) {
3840 obj->efile.text_shndx = idx;
3846 sec_desc->sec_type = SEC_DATA;
3848 sec_desc->shdr = sh;
3849 sec_desc->data = data;
3851 sec_desc->psection = ptmpsection;
3852 sec_desc->realdata.d_buf = data->d_buf;
3853 sec_desc->realdata.d_size = data->d_size;
3854 sec_desc->data = &(sec_desc->realdata);
3858 sec_desc->sec_type = SEC_RODATA;
3860 sec_desc->shdr = sh;
3861 sec_desc->data = data;
3863 sec_desc->psection = ptmpsection;
3864 sec_desc->realdata.d_buf = data->d_buf;
3865 sec_desc->realdata.d_size = data->d_size;
3866 sec_desc->data = &(sec_desc->realdata);
3871 obj->efile.st_ops_data = data;
3873 obj->efile.realst_ops_data.d_buf = data->d_buf;
3874 obj->efile.realst_ops_data.d_size = data->d_size;
3875 obj->efile.st_ops_data = &(obj->efile.realst_ops_data);
3877 obj->efile.st_ops_shndx = idx;
3882 } else if (sh->sh_type == SHT_REL) {
3883 int targ_sec_idx = sh->sh_info; /* points to other section */
3885 if (sh->sh_entsize != sizeof(Elf64_Rel) ||
3886 targ_sec_idx >= obj->efile.sec_cnt)
3887 return -LIBBPF_ERRNO__FORMAT;
3905 sec_desc->sec_type = SEC_RELO;
3907 sec_desc->shdr = sh;
3909 sec_desc->psection = ptmpsection;
3911 sec_desc->data = data;
3912 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||
3914 sec_desc->sec_type = SEC_BSS;
3916 sec_desc->shdr = sh;
3918 sec_desc->psection = ptmpsection;
3920 sec_desc->data = data;
3923 (size_t)sh->sh_size);
3927 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3928 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3929 return -LIBBPF_ERRNO__FORMAT;
3932 /* sort BPF programs by section name and in-section instruction offset
3935 if (obj->nr_programs)
3936 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3943 int bind = ELF64_ST_BIND(sym->st_info);
3945 return sym->st_shndx == SHN_UNDEF &&
3947 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
3952 int bind = ELF64_ST_BIND(sym->st_info);
3953 int type = ELF64_ST_TYPE(sym->st_info);
3956 if (sym->st_shndx != text_shndx)
3974 return -ESRCH;
3983 tname = btf__name_by_offset(btf, t->name_off);
3988 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
3989 return -EINVAL;
3992 return -EINVAL;
3997 return -ENOENT;
4006 return -ESRCH;
4017 if (vs->type == ext_btf_id)
4022 return -ENOENT;
4032 name = btf__name_by_offset(btf, t->name_off);
4041 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
4044 if (t->size == 1)
4046 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
4051 if (t->size != 4)
4061 if (btf_array(t)->nelems == 0)
4063 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
4076 if (a->type != b->type)
4077 return a->type < b->type ? -1 : 1;
4079 if (a->type == EXT_KCFG) {
4081 if (a->kcfg.align != b->kcfg.align)
4082 return a->kcfg.align > b->kcfg.align ? -1 : 1;
4084 if (a->kcfg.sz != b->kcfg.sz)
4085 return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
4089 return strcmp(a->name, b->name);
4127 vt = btf__type_by_id(btf, vs->type);
4161 if (!obj->efile.symbols)
4165 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
4169 sh = elf_sec_hdr_by_idx(obj, obj->efile.symbols_shndx, sh);
4172 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
4173 return -LIBBPF_ERRNO__FORMAT;
4175 dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
4179 n = sh->sh_size / sh->sh_entsize;
4186 return -LIBBPF_ERRNO__FORMAT;
4189 ext_name = elf_sym_str(obj, sym->st_name);
4193 ext = obj->externs;
4194 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
4196 return -ENOMEM;
4197 obj->externs = ext;
4198 ext = &ext[obj->nr_extern];
4200 obj->nr_extern++;
4202 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
4203 if (ext->btf_id <= 0) {
4205 ext_name, ext->btf_id);
4206 return ext->btf_id;
4208 t = btf__type_by_id(obj->btf, ext->btf_id);
4209 ext->name = btf__name_by_offset(obj->btf, t->name_off);
4210 ext->sym_idx = i;
4211 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
4213 ext_essent_len = bpf_core_essential_name_len(ext->name);
4214 ext->essent_name = NULL;
4215 if (ext_essent_len != strlen(ext->name)) {
4216 ext->essent_name = strndup(ext->name, ext_essent_len);
4217 if (!ext->essent_name)
4218 return -ENOMEM;
4221 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
4222 if (ext->sec_btf_id <= 0) {
4224 ext_name, ext->btf_id, ext->sec_btf_id);
4225 return ext->sec_btf_id;
4227 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
4228 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
4233 ext->name, KCONFIG_SEC);
4234 return -ENOTSUP;
4237 ext->type = EXT_KCFG;
4238 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
4239 if (ext->kcfg.sz <= 0) {
4241 ext_name, ext->kcfg.sz);
4242 return ext->kcfg.sz;
4244 ext->kcfg.align = btf__align_of(obj->btf, t->type);
4245 if (ext->kcfg.align <= 0) {
4247 ext_name, ext->kcfg.align);
4248 return -EINVAL;
4250 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
4251 &ext->kcfg.is_signed);
4252 if (ext->kcfg.type == KCFG_UNKNOWN) {
4254 return -ENOTSUP;
4258 ext->type = EXT_KSYM;
4259 skip_mods_and_typedefs(obj->btf, t->type,
4260 &ext->ksym.type_id);
4263 return -ENOTSUP;
4266 pr_debug("collected %d externs total\n", obj->nr_extern);
4268 if (!obj->nr_extern)
4272 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
4276 * pretending that each extern is a 8-byte variable
4279 /* find existing 4-byte integer type in BTF to use for fake
4282 int int_btf_id = find_int_btf_id(obj->btf);
4284 * will be used to replace the vs->type and
4290 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
4291 for (i = 0; i < obj->nr_extern; i++) {
4292 ext = &obj->externs[i];
4293 if (ext->type != EXT_KSYM)
4296 i, ext->sym_idx, ext->name);
4305 vt = (void *)btf__type_by_id(obj->btf, vs->type);
4306 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
4311 return -ESRCH;
4318 func_proto = btf__type_by_id(obj->btf,
4319 vt->type);
4327 dummy_var->name_off;
4328 vs->type = dummy_var_btf_id;
4329 vt->info &= ~0xffff;
4330 vt->info |= BTF_FUNC_GLOBAL;
4332 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4333 vt->type = int_btf_id;
4335 vs->offset = off;
4336 vs->size = sizeof(int);
4338 sec->size = off;
4345 for (i = 0; i < obj->nr_extern; i++) {
4346 ext = &obj->externs[i];
4347 if (ext->type != EXT_KCFG)
4350 ext->kcfg.data_off = roundup(off, ext->kcfg.align);
4351 off = ext->kcfg.data_off + ext->kcfg.sz;
4353 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
4355 sec->size = off;
4360 t = btf__type_by_id(obj->btf, vs->type);
4361 ext_name = btf__name_by_offset(obj->btf, t->name_off);
4366 return -ESRCH;
4368 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4369 vs->offset = ext->kcfg.data_off;
4377 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
4389 if (!strcmp(prog->name, name))
4398 switch (obj->efile.secs[shndx].sec_type) {
4411 return shndx == obj->efile.btf_maps_shndx;
4417 if (shndx == obj->efile.symbols_shndx)
4420 switch (obj->efile.secs[shndx].sec_type) {
4437 struct bpf_insn *insn = &prog->insns[insn_idx];
4438 size_t map_idx, nr_maps = prog->obj->nr_maps;
4439 struct bpf_object *obj = prog->obj;
4440 __u32 shdr_idx = sym->st_shndx;
4447 prog->name, sym_name, insn_idx, insn->code);
4448 return -LIBBPF_ERRNO__RELOC;
4452 int sym_idx = ELF64_R_SYM(rel->r_info);
4453 int i, n = obj->nr_extern;
4457 ext = &obj->externs[i];
4458 if (ext->sym_idx == sym_idx)
4463 prog->name, sym_name, sym_idx);
4464 return -LIBBPF_ERRNO__RELOC;
4467 prog->name, i, ext->name, ext->sym_idx, insn_idx);
4468 if (insn->code == (BPF_JMP | BPF_CALL))
4469 reloc_desc->type = RELO_EXTERN_CALL;
4471 reloc_desc->type = RELO_EXTERN_LD64;
4472 reloc_desc->insn_idx = insn_idx;
4473 reloc_desc->ext_idx = i;
4477 /* sub-program call relocation */
4479 if (insn->src_reg != BPF_PSEUDO_CALL) {
4480 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4481 return -LIBBPF_ERRNO__RELOC;
4484 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
4491 prog->name, sym_name, sym_sec_name);
4492 return -LIBBPF_ERRNO__RELOC;
4494 if (sym->st_value % BPF_INSN_SZ) {
4496 prog->name, sym_name, (size_t)sym->st_value);
4497 return -LIBBPF_ERRNO__RELOC;
4499 reloc_desc->type = RELO_CALL;
4500 reloc_desc->insn_idx = insn_idx;
4501 reloc_desc->sym_off = sym->st_value;
4507 prog->name, sym_name, shdr_idx);
4508 return -LIBBPF_ERRNO__RELOC;
4512 if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4513 /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4514 * local_func: sym->st_value = 0, insn->imm = offset in the section.
4516 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4518 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4519 return -LIBBPF_ERRNO__RELOC;
4522 reloc_desc->type = RELO_SUBPROG_ADDR;
4523 reloc_desc->insn_idx = insn_idx;
4524 reloc_desc->sym_off = sym->st_value;
4538 prog->name, sym_name, sym_sec_name);
4539 return -LIBBPF_ERRNO__RELOC;
4542 map = &obj->maps[map_idx];
4543 if (map->libbpf_type != type ||
4544 map->sec_idx != sym->st_shndx ||
4545 map->sec_offset != sym->st_value)
4548 prog->name, map_idx, map->name, map->sec_idx,
4549 map->sec_offset, insn_idx);
4554 prog->name, sym_sec_name, (size_t)sym->st_value);
4555 return -LIBBPF_ERRNO__RELOC;
4557 reloc_desc->type = RELO_LD64;
4558 reloc_desc->insn_idx = insn_idx;
4559 reloc_desc->map_idx = map_idx;
4560 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4567 prog->name, sym_sec_name);
4568 return -LIBBPF_ERRNO__RELOC;
4571 map = &obj->maps[map_idx];
4572 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4575 prog->name, map_idx, map->name, map->sec_idx,
4576 map->sec_offset, insn_idx);
4581 prog->name, sym_sec_name);
4582 return -LIBBPF_ERRNO__RELOC;
4585 reloc_desc->type = RELO_DATA;
4586 reloc_desc->insn_idx = insn_idx;
4587 reloc_desc->map_idx = map_idx;
4588 reloc_desc->sym_off = sym->st_value;
4594 return insn_idx >= prog->sec_insn_off &&
4595 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4601 int l = 0, r = obj->nr_programs - 1, m;
4604 if (!obj->nr_programs)
4608 m = l + (r - l + 1) / 2;
4609 prog = &obj->programs[m];
4611 if (prog->sec_idx < sec_idx ||
4612 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4615 r = m - 1;
4620 prog = &obj->programs[l];
4621 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4630 size_t sec_idx = shdr->sh_info, sym_idx;
4643 if (sec_idx >= obj->efile.sec_cnt)
4644 return -EINVAL;
4650 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4653 return -EINVAL;
4658 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4661 return -EINVAL;
4666 nrels = shdr->sh_size / shdr->sh_entsize;
4672 return -LIBBPF_ERRNO__FORMAT;
4675 sym_idx = ELF64_R_SYM(rel->r_info);
4680 return -LIBBPF_ERRNO__FORMAT;
4683 if (sym->st_shndx >= obj->efile.sec_cnt) {
4685 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4686 return -LIBBPF_ERRNO__FORMAT;
4689 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4691 relo_sec_name, (size_t)rel->r_offset, i);
4692 return -LIBBPF_ERRNO__FORMAT;
4695 insn_idx = rel->r_offset / BPF_INSN_SZ;
4702 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4704 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4706 sym_name = elf_sec_name_by_idx(obj, sym->st_shndx);
4709 sym_name = elf_sym_str(obj, sym->st_name);
4722 relos = libbpf_reallocarray(prog->reloc_desc,
4723 prog->nr_reloc + 1, sizeof(*relos));
4725 return -ENOMEM;
4726 prog->reloc_desc = relos;
4729 insn_idx -= prog->sec_insn_off;
4730 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4735 prog->nr_reloc++;
4744 if (!obj->btf)
4745 return -ENOENT;
4747 /* if it's BTF-defined map, we don't need to search for type IDs.
4751 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4759 return -ENOENT;
4761 id = btf__find_by_name(obj->btf, map->real_name);
4765 map->btf_key_type_id = 0;
4766 map->btf_value_type_id = id;
4782 err = -errno;
4790 info->type = val;
4792 info->key_size = val;
4794 info->value_size = val;
4796 info->max_entries = val;
4798 info->map_flags = val;
4808 return map->autocreate;
4813 if (map->obj->loaded)
4814 return libbpf_err(-EBUSY);
4816 map->autocreate = autocreate;
4835 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
4836 new_name = strdup(map->name);
4841 return libbpf_err(-errno);
4850 err = -errno;
4854 err = zclose(map->fd);
4856 err = -errno;
4859 free(map->name);
4861 map->fd = new_fd;
4862 map->name = new_name;
4863 map->def.type = info.type;
4864 map->def.key_size = info.key_size;
4865 map->def.value_size = info.value_size;
4866 map->def.max_entries = info.max_entries;
4867 map->def.map_flags = info.map_flags;
4868 map->btf_key_type_id = info.btf_key_type_id;
4869 map->btf_value_type_id = info.btf_value_type_id;
4870 map->reused = true;
4871 map->map_extra = info.map_extra;
4884 return map->def.max_entries;
4889 if (!bpf_map_type__is_map_in_map(map->def.type))
4892 return map->inner_map;
4897 if (map->obj->loaded)
4898 return libbpf_err(-EBUSY);
4900 map->def.max_entries = max_entries;
4902 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
4904 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
4919 if (obj->gen_loader)
4937 return -ret;
4986 ret = -errno;
4989 __func__, cp, -ret);
5089 BTF_TYPE_DECL_TAG_ENC(1, 2, -1),
5131 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
5143 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */
5166 ret = -errno;
5169 __func__, cp, -ret);
5227 return -errno;
5232 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL);
5233 err = -errno; /* close() can clobber errno */
5239 return link_fd < 0 && err == -EBADF;
5258 return -errno;
5260 /* Creating uprobe in '/' binary should fail with -EBADF. */
5265 link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts);
5266 err = -errno; /* close() can clobber errno */
5272 return link_fd < 0 && err == -EBADF;
5360 "memcg-based memory accounting", probe_memcg_account,
5372 "BPF multi-uprobe link support", probe_uprobe_multi_link,
5381 if (obj && obj->gen_loader)
5387 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
5388 ret = feat->probe();
5390 WRITE_ONCE(feat->res, FEAT_SUPPORTED);
5392 WRITE_ONCE(feat->res, FEAT_MISSING);
5394 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
5395 WRITE_ONCE(feat->res, FEAT_MISSING);
5399 return READ_ONCE(feat->res) == FEAT_SUPPORTED;
5419 return (map_info.type == map->def.type &&
5420 map_info.key_size == map->def.key_size &&
5421 map_info.value_size == map->def.value_size &&
5422 map_info.max_entries == map->def.max_entries &&
5423 map_info.map_flags == map->def.map_flags &&
5424 map_info.map_extra == map->map_extra);
5433 pin_fd = bpf_obj_get(map->pin_path);
5435 err = -errno;
5436 if (err == -ENOENT) {
5438 map->pin_path);
5442 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
5444 map->pin_path, cp);
5450 map->pin_path);
5452 return -EINVAL;
5460 map->pinned = true;
5461 pr_debug("reused pinned map at '%s'\n", map->pin_path);
5469 enum libbpf_map_type map_type = map->libbpf_type;
5473 if (obj->gen_loader) {
5474 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
5475 map->mmaped, map->def.value_size);
5477 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
5480 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
5482 err = -errno;
5485 map->name, cp);
5489 /* Freeze .rodata and .kconfig map as read-only from syscall side. */
5491 err = bpf_map_freeze(map->fd);
5493 err = -errno;
5495 pr_warn("Error freezing map(%s) as read-only: %s\n",
5496 map->name, cp);
5508 struct bpf_map_def *def = &map->def;
5513 map_name = map->name;
5514 create_attr.map_ifindex = map->map_ifindex;
5515 create_attr.map_flags = def->map_flags;
5516 create_attr.numa_node = map->numa_node;
5517 create_attr.map_extra = map->map_extra;
5520 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
5522 if (obj->btf && btf__fd(obj->btf) >= 0) {
5523 create_attr.btf_fd = btf__fd(obj->btf);
5524 create_attr.btf_key_type_id = map->btf_key_type_id;
5525 create_attr.btf_value_type_id = map->btf_value_type_id;
5528 if (bpf_map_type__is_map_in_map(def->type)) {
5529 if (map->inner_map) {
5530 err = bpf_object__create_map(obj, map->inner_map, true);
5533 map->name, err);
5536 map->inner_map_fd = bpf_map__fd(map->inner_map);
5538 if (map->inner_map_fd >= 0)
5539 create_attr.inner_map_fd = map->inner_map_fd;
5542 switch (def->type) {
5559 map->btf_key_type_id = 0;
5560 map->btf_value_type_id = 0;
5566 if (obj->gen_loader) {
5567 bpf_gen__map_create(obj->gen_loader, def->type, map_name,
5568 def->key_size, def->value_size, def->max_entries,
5569 &create_attr, is_inner ? -1 : map - obj->maps);
5571 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
5573 map->fd = 0;
5575 map->fd = bpf_map_create(def->type, map_name,
5576 def->key_size, def->value_size,
5577 def->max_entries, &create_attr);
5579 if (map->fd < 0 && (create_attr.btf_key_type_id ||
5583 err = -errno;
5586 map->name, cp, err);
5590 map->btf_key_type_id = 0;
5591 map->btf_value_type_id = 0;
5592 map->fd = bpf_map_create(def->type, map_name,
5593 def->key_size, def->value_size,
5594 def->max_entries, &create_attr);
5597 err = map->fd < 0 ? -errno : 0;
5599 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
5600 if (obj->gen_loader)
5601 map->inner_map->fd = -1;
5602 bpf_map__destroy(map->inner_map);
5603 zfree(&map->inner_map);
5615 for (i = 0; i < map->init_slots_sz; i++) {
5616 if (!map->init_slots[i])
5619 targ_map = map->init_slots[i];
5622 if (obj->gen_loader) {
5623 bpf_gen__populate_outer_map(obj->gen_loader,
5624 map - obj->maps, i,
5625 targ_map - obj->maps);
5627 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5630 err = -errno;
5632 map->name, i, targ_map->name, fd, err);
5636 map->name, i, targ_map->name, fd);
5639 zfree(&map->init_slots);
5640 map->init_slots_sz = 0;
5651 if (obj->gen_loader)
5652 return -ENOTSUP;
5654 for (i = 0; i < map->init_slots_sz; i++) {
5655 if (!map->init_slots[i])
5658 targ_prog = map->init_slots[i];
5661 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5663 err = -errno;
5665 map->name, i, targ_prog->name, fd, err);
5669 map->name, i, targ_prog->name, fd);
5672 zfree(&map->init_slots);
5673 map->init_slots_sz = 0;
5683 for (i = 0; i < obj->nr_maps; i++) {
5684 map = &obj->maps[i];
5686 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
5691 zclose(map->fd);
5700 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
5706 map->name, nr_cpus);
5709 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
5710 map->def.max_entries = nr_cpus;
5725 for (i = 0; i < obj->nr_maps; i++) {
5726 map = &obj->maps[i];
5730 * loading, if we detect that at least one of the to-be-loaded
5735 * but also it allows to have CO-RE applications that use
5737 * If those global variable-using programs are not loaded at
5743 map->autocreate = false;
5745 if (!map->autocreate) {
5746 pr_debug("map '%s': skipped auto-creating...\n", map->name);
5756 if (map->pin_path) {
5760 map->name);
5763 if (retried && map->fd < 0) {
5765 map->name);
5766 err = -ENOENT;
5771 if (map->fd >= 0) {
5773 map->name, map->fd);
5780 map->name, map->fd);
5785 zclose(map->fd);
5790 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
5793 zclose(map->fd);
5799 if (map->pin_path && !map->pinned) {
5802 zclose(map->fd);
5803 if (!retried && err == -EEXIST) {
5807 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
5808 map->name, map->pin_path, err);
5818 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
5821 zclose(obj->maps[j].fd);
5835 * underscore is ignored by BPF CO-RE relocation during relocation matching.
5842 for (i = n - 5; i >= 0; i--) {
5854 free(cands->cands);
5871 local_t = btf__type_by_id(local_cand->btf, local_cand->id);
5872 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
5880 targ_name = btf__name_by_offset(targ_btf, t->name_off);
5891 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
5892 local_cand->id, btf_kind_str(local_t),
5895 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
5896 sizeof(*cands->cands));
5898 return -ENOMEM;
5900 cand = &new_cands[cands->len];
5901 cand->btf = targ_btf;
5902 cand->id = i;
5904 cands->cands = new_cands;
5905 cands->len++;
5919 if (obj->btf_modules_loaded)
5922 if (obj->gen_loader)
5926 obj->btf_modules_loaded = true;
5941 err = -errno;
5950 err = -errno;
5962 err = -errno;
5967 /* ignore non-module BTFs */
5973 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5981 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5982 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5986 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
5988 mod_btf->btf = btf;
5989 mod_btf->id = id;
5990 mod_btf->fd = fd;
5991 mod_btf->name = strdup(name);
5992 if (!mod_btf->name) {
5993 err = -ENOMEM;
6021 return ERR_PTR(-EINVAL);
6023 local_name = btf__name_by_offset(local_btf, local_t->name_off);
6025 return ERR_PTR(-EINVAL);
6030 return ERR_PTR(-ENOMEM);
6033 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
6039 if (cands->len)
6043 if (obj->btf_vmlinux_override)
6051 for (i = 0; i < obj->btf_module_cnt; i++) {
6053 obj->btf_modules[i].btf,
6054 obj->btf_modules[i].name,
6055 btf__type_cnt(obj->btf_vmlinux),
6068 * type-based CO-RE relocations and follow slightly different rules than
6069 * field-based relocations. This function assumes that root types were already
6070 * checked for name match. Beyond that initial root-level name check, names
6072 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
6075 * - for ENUMs, the size is ignored;
6076 * - for INT, size and signedness are ignored;
6077 * - for ARRAY, dimensionality is ignored, element types are checked for
6079 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
6080 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
6081 * - FUNC_PROTOs are compatible if they have compatible signature: same
6084 * more experience with using BPF CO-RE relocations.
6113 relos = libbpf_reallocarray(prog->reloc_desc,
6114 prog->nr_reloc + 1, sizeof(*relos));
6116 return -ENOMEM;
6117 relo = &relos[prog->nr_reloc];
6118 relo->type = RELO_CORE;
6119 relo->insn_idx = insn_idx;
6120 relo->core_relo = core_relo;
6121 prog->reloc_desc = relos;
6122 prog->nr_reloc++;
6131 for (i = 0; i < prog->nr_reloc; i++) {
6132 relo = &prog->reloc_desc[i];
6133 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)
6136 return relo->core_relo;
6151 const char *prog_name = prog->name;
6154 __u32 local_id = relo->type_id;
6159 return -EINVAL;
6161 local_name = btf__name_by_offset(local_btf, local_type->name_off);
6163 return -EINVAL;
6165 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
6167 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
6199 if (obj->btf_ext->core_relo_info.len == 0)
6204 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
6206 err = libbpf_get_error(obj->btf_vmlinux_override);
6219 seg = &obj->btf_ext->core_relo_info;
6222 sec_idx = seg->sec_idxs[sec_num];
6225 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6227 err = -EINVAL;
6231 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);
6234 if (rec->insn_off % BPF_INSN_SZ)
6235 return -EINVAL;
6236 insn_idx = rec->insn_off / BPF_INSN_SZ;
6243 * This is similar to what x86-64 linker does for relocations.
6247 …pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subpro…
6251 /* no need to apply CO-RE relocation if the program is
6254 if (!prog->autoload)
6258 * program's frame of reference; (sub-)program code is not yet
6259 * relocated, so it's enough to just subtract in-section offset
6261 insn_idx = insn_idx - prog->sec_insn_off;
6262 if (insn_idx >= prog->insns_cnt)
6263 return -EINVAL;
6264 insn = &prog->insns[insn_idx];
6269 prog->name, i, err);
6273 if (prog->obj->gen_loader)
6276 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
6279 prog->name, i, err);
6283 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
6286 prog->name, i, insn_idx, err);
6293 /* obj->btf_vmlinux and module BTFs are freed after object load */
6294 btf__free(obj->btf_vmlinux_override);
6295 obj->btf_vmlinux_override = NULL;
6299 bpf_core_free_cands(entry->pvalue);
6317 prog->name, relo_idx, insn_idx, map_idx, map->name);
6321 insn->code = BPF_JMP | BPF_CALL;
6322 insn->dst_reg = 0;
6323 insn->src_reg = 0;
6324 insn->off = 0;
6328 * where lower 123 is map index into obj->maps[] array
6330 insn->imm = POISON_LDIMM64_MAP_BASE + map_idx;
6345 prog->name, relo_idx, insn_idx, ext->name);
6348 insn->code = BPF_JMP | BPF_CALL;
6349 insn->dst_reg = 0;
6350 insn->src_reg = 0;
6351 insn->off = 0;
6355 * where lower 123 is extern index into obj->externs[] array
6357 insn->imm = POISON_CALL_KFUNC_BASE + ext_idx;
6361 * - map references;
6362 * - global variable references;
6363 * - extern references.
6370 for (i = 0; i < prog->nr_reloc; i++) {
6371 struct reloc_desc *relo = &prog->reloc_desc[i];
6372 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6376 switch (relo->type) {
6378 map = &obj->maps[relo->map_idx];
6379 if (obj->gen_loader) {
6381 insn[0].imm = relo->map_idx;
6382 } else if (map->autocreate) {
6384 insn[0].imm = map->fd;
6386 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6387 relo->map_idx, map);
6391 map = &obj->maps[relo->map_idx];
6392 insn[1].imm = insn[0].imm + relo->sym_off;
6393 if (obj->gen_loader) {
6395 insn[0].imm = relo->map_idx;
6396 } else if (map->autocreate) {
6398 insn[0].imm = map->fd;
6400 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6401 relo->map_idx, map);
6405 ext = &obj->externs[relo->ext_idx];
6406 if (ext->type == EXT_KCFG) {
6407 if (obj->gen_loader) {
6409 insn[0].imm = obj->kconfig_map_idx;
6412 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6414 insn[1].imm = ext->kcfg.data_off;
6416 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
6418 insn[0].imm = ext->ksym.kernel_btf_id;
6419 insn[1].imm = ext->ksym.kernel_btf_obj_fd;
6421 insn[0].imm = (__u32)ext->ksym.addr;
6422 insn[1].imm = ext->ksym.addr >> 32;
6427 ext = &obj->externs[relo->ext_idx];
6429 if (ext->is_set) {
6430 insn[0].imm = ext->ksym.kernel_btf_id;
6431 insn[0].off = ext->ksym.btf_fd_idx;
6433 poison_kfunc_call(prog, i, relo->insn_idx, insn,
6434 relo->ext_idx, ext);
6440 prog->name, i);
6441 return -EINVAL;
6453 prog->name, i, relo->type);
6454 return -EINVAL;
6475 sec_idx = ext_info->sec_idxs[sec_num];
6477 if (prog->sec_idx != sec_idx)
6483 if (insn_off < prog->sec_insn_off)
6485 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6490 copy_end = rec + ext_info->rec_size;
6494 return -ENOENT;
6496 /* append func/line info of a given (sub-)program to the main
6499 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
6500 new_sz = old_sz + (copy_end - copy_start);
6503 return -ENOMEM;
6505 *prog_rec_cnt = new_sz / ext_info->rec_size;
6506 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6508 /* Kernel instruction offsets are in units of 8-byte
6514 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6517 for (; rec < rec_end; rec += ext_info->rec_size) {
6522 *prog_rec_sz = ext_info->rec_size;
6526 return -ENOENT;
6539 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
6545 if (main_prog != prog && !main_prog->func_info)
6548 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6549 &main_prog->func_info,
6550 &main_prog->func_info_cnt,
6551 &main_prog->func_info_rec_size);
6553 if (err != -ENOENT) {
6555 prog->name, err);
6558 if (main_prog->func_info) {
6563 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6568 prog->name);
6573 if (main_prog != prog && !main_prog->line_info)
6576 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6577 &main_prog->line_info,
6578 &main_prog->line_info_cnt,
6579 &main_prog->line_info_rec_size);
6581 if (err != -ENOENT) {
6583 prog->name, err);
6586 if (main_prog->line_info) {
6591 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6596 prog->name);
6606 if (insn_idx == relo->insn_idx)
6608 return insn_idx < relo->insn_idx ? -1 : 1;
6613 if (!prog->nr_reloc)
6615 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6616 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6621 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
6627 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
6633 return -ENOMEM;
6634 if (subprog->nr_reloc)
6635 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
6636 sizeof(*relos) * subprog->nr_reloc);
6638 for (i = main_prog->nr_reloc; i < new_cnt; i++)
6639 relos[i].insn_idx += subprog->sub_insn_off;
6643 main_prog->reloc_desc = relos;
6644 main_prog->nr_reloc = new_cnt;
6656 subprog->sub_insn_off = main_prog->insns_cnt;
6658 new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6659 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6661 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6662 return -ENOMEM;
6664 main_prog->insns = insns;
6665 main_prog->insns_cnt = new_cnt;
6667 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6668 subprog->insns_cnt * sizeof(*insns));
6670 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6671 main_prog->name, subprog->insns_cnt, subprog->name);
6694 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6695 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6700 if (relo && relo->type == RELO_EXTERN_CALL)
6705 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6707 prog->name, insn_idx, relo->type);
6708 return -LIBBPF_ERRNO__RELOC;
6711 /* sub-program instruction index is a combination of
6714 * call always has imm = -1, but for static functions
6715 * relocation is against STT_SECTION and insn->imm
6718 * for subprog addr relocation, the relo->sym_off + insn->imm is
6721 if (relo->type == RELO_CALL)
6722 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6724 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6731 prog->name, insn_idx);
6732 return -LIBBPF_ERRNO__RELOC;
6737 * offset necessary, insns->imm is relative to
6740 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6743 /* we enforce that sub-programs should be in .text section */
6744 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6746 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6747 prog->name);
6748 return -LIBBPF_ERRNO__RELOC;
6754 * - append it at the end of main program's instructions blog;
6755 * - process is recursively, while current program is put on hold;
6756 * - if that subprogram calls some other not yet processes
6761 if (subprog->sub_insn_off == 0) {
6770 /* main_prog->insns memory could have been re-allocated, so
6773 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6780 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6783 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6790 * Relocate sub-program calls.
6792 * Algorithm operates as follows. Each entry-point BPF program (referred to as
6793 * main prog) is processed separately. For each subprog (non-entry functions,
6802 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6818 * subprog->sub_insn_off as zero at all times and won't be appended to current
6827 * +--------+ +-------+
6829 * +--+---+ +--+-+-+ +---+--+
6831 * +--+---+ +------+ +---+--+
6834 * +---+-------+ +------+----+
6836 * +-----------+ +-----------+
6841 * +-----------+------+
6843 * +-----------+------+
6848 * +-----------+------+------+
6850 * +-----------+------+------+
6859 * +-----------+------+
6861 * +-----------+------+
6864 * +-----------+------+------+
6866 * +-----------+------+------+
6879 for (i = 0; i < obj->nr_programs; i++) {
6880 subprog = &obj->programs[i];
6884 subprog->sub_insn_off = 0;
6901 for (i = 0; i < obj->nr_programs; i++) {
6902 prog = &obj->programs[i];
6903 zfree(&prog->reloc_desc);
6904 prog->nr_reloc = 0;
6913 if (a->insn_idx != b->insn_idx)
6914 return a->insn_idx < b->insn_idx ? -1 : 1;
6917 if (a->type != b->type)
6918 return a->type < b->type ? -1 : 1;
6927 for (i = 0; i < obj->nr_programs; i++) {
6928 struct bpf_program *p = &obj->programs[i];
6930 if (!p->nr_reloc)
6933 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6944 if (obj->btf_ext) {
6947 pr_warn("failed to perform CO-RE relocations: %d\n",
6954 /* Before relocating calls pre-process relocations and mark
6961 for (i = 0; i < obj->nr_programs; i++) {
6962 prog = &obj->programs[i];
6963 for (j = 0; j < prog->nr_reloc; j++) {
6964 struct reloc_desc *relo = &prog->reloc_desc[j];
6965 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6968 if (relo->type == RELO_SUBPROG_ADDR)
6980 for (i = 0; i < obj->nr_programs; i++) {
6981 prog = &obj->programs[i];
6982 /* sub-program's sub-calls are relocated within the context of
6987 if (!prog->autoload)
6993 prog->name, err);
6998 if (prog->exception_cb_idx >= 0) {
6999 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx];
7006 if (subprog->sub_insn_off == 0) {
7017 for (i = 0; i < obj->nr_programs; i++) {
7018 prog = &obj->programs[i];
7021 if (!prog->autoload)
7026 prog->name, err);
7054 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
7055 return -EINVAL;
7056 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
7058 return -EINVAL;
7060 nrels = shdr->sh_size / shdr->sh_entsize;
7065 return -LIBBPF_ERRNO__FORMAT;
7068 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
7071 i, (size_t)ELF64_R_SYM(rel->r_info));
7072 return -LIBBPF_ERRNO__FORMAT;
7074 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
7076 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",
7077 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
7078 (size_t)rel->r_offset, sym->st_name, name);
7080 for (j = 0; j < obj->nr_maps; j++) {
7081 map = &obj->maps[j];
7082 if (map->sec_idx != obj->efile.btf_maps_shndx)
7085 vi = btf_var_secinfos(sec) + map->btf_var_idx;
7086 if (vi->offset <= rel->r_offset &&
7087 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
7090 if (j == obj->nr_maps) {
7091 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
7092 i, name, (size_t)rel->r_offset);
7093 return -EINVAL;
7096 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
7097 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
7100 if (sym->st_shndx != obj->efile.btf_maps_shndx) {
7101 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
7103 return -LIBBPF_ERRNO__RELOC;
7105 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
7106 map->def.key_size != sizeof(int)) {
7107 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
7108 i, map->name, sizeof(int));
7109 return -EINVAL;
7115 return -ESRCH;
7122 return -ESRCH;
7124 if (targ_prog->sec_idx != sym->st_shndx ||
7125 targ_prog->sec_insn_off * 8 != sym->st_value ||
7127 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
7129 return -LIBBPF_ERRNO__RELOC;
7132 return -EINVAL;
7135 var = btf__type_by_id(obj->btf, vi->type);
7136 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
7138 return -EINVAL;
7139 member = btf_members(def) + btf_vlen(def) - 1;
7140 mname = btf__name_by_offset(obj->btf, member->name_off);
7142 return -EINVAL;
7144 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
7145 if (rel->r_offset - vi->offset < moff)
7146 return -EINVAL;
7148 moff = rel->r_offset - vi->offset - moff;
7153 return -EINVAL;
7155 if (moff >= map->init_slots_sz) {
7157 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
7159 return -ENOMEM;
7160 map->init_slots = tmp;
7161 memset(map->init_slots + map->init_slots_sz, 0,
7162 (new_sz - map->init_slots_sz) * host_ptr_sz);
7163 map->init_slots_sz = new_sz;
7165 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
7168 i, map->name, moff, type, name);
7178 for (i = 0; i < obj->efile.sec_cnt; i++) {
7179 struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
7185 if (sec_desc->sec_type != SEC_RELO)
7189 shdr = sec_desc->shdr;
7193 data = sec_desc->data;
7194 idx = shdr->sh_info;
7196 if (shdr->sh_type != SHT_REL) {
7198 return -LIBBPF_ERRNO__INTERNAL;
7201 if (idx == obj->efile.st_ops_shndx || idx == obj->efile.st_ops_link_shndx)
7203 else if (idx == obj->efile.btf_maps_shndx)
7217 if (BPF_CLASS(insn->code) == BPF_JMP &&
7218 BPF_OP(insn->code) == BPF_CALL &&
7219 BPF_SRC(insn->code) == BPF_K &&
7220 insn->src_reg == 0 &&
7221 insn->dst_reg == 0) {
7222 *func_id = insn->imm;
7230 struct bpf_insn *insn = prog->insns;
7234 if (obj->gen_loader)
7237 for (i = 0; i < prog->insns_cnt; i++, insn++) {
7249 insn->imm = BPF_FUNC_probe_read;
7254 insn->imm = BPF_FUNC_probe_read_str;
7266 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
7273 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
7274 opts->expected_attach_type = 0;
7277 opts->prog_flags |= BPF_F_SLEEPABLE;
7279 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
7280 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
7283 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))
7284 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7286 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
7290 attach_name = strchr(prog->sec_name, '/');
7301 …pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\…
7302 prog->name);
7303 return -EINVAL;
7312 prog->attach_btf_obj_fd = btf_obj_fd;
7313 prog->attach_btf_id = btf_type_id;
7316 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
7320 opts->attach_btf_obj_fd = btf_obj_fd;
7321 opts->attach_btf_id = btf_type_id;
7339 __u32 log_level = prog->log_level;
7341 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
7347 prog->name, prog->sec_name);
7348 return -EINVAL;
7352 return -EINVAL;
7355 prog_name = prog->name;
7356 load_attr.attach_prog_fd = prog->attach_prog_fd;
7357 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
7358 load_attr.attach_btf_id = prog->attach_btf_id;
7360 load_attr.prog_ifindex = prog->prog_ifindex;
7366 load_attr.func_info = prog->func_info;
7367 load_attr.func_info_rec_size = prog->func_info_rec_size;
7368 load_attr.func_info_cnt = prog->func_info_cnt;
7369 load_attr.line_info = prog->line_info;
7370 load_attr.line_info_rec_size = prog->line_info_rec_size;
7371 load_attr.line_info_cnt = prog->line_info_cnt;
7374 load_attr.prog_flags = prog->prog_flags;
7375 load_attr.fd_array = obj->fd_array;
7378 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
7379 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
7382 prog->name, err);
7385 insns = prog->insns;
7386 insns_cnt = prog->insns_cnt;
7390 load_attr.expected_attach_type = prog->expected_attach_type;
7392 if (obj->gen_loader) {
7393 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
7395 prog - obj->programs);
7396 *prog_fd = -1;
7407 if (prog->log_buf) {
7408 log_buf = prog->log_buf;
7409 log_buf_size = prog->log_size;
7411 } else if (obj->log_buf) {
7412 log_buf = obj->log_buf;
7413 log_buf_size = obj->log_size;
7419 ret = -ENOMEM;
7432 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
7435 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7436 prog->name, log_buf);
7439 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
7443 for (i = 0; i < obj->nr_maps; i++) {
7444 map = &prog->obj->maps[i];
7445 if (map->libbpf_type != LIBBPF_MAP_RODATA)
7451 prog->name, map->real_name, cp);
7471 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
7476 ret = -errno;
7478 /* post-process verifier log to improve error descriptions */
7482 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
7486 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7487 prog->name, log_buf);
7503 p = cur - 1;
7504 while (p - 1 >= buf && *(p - 1) != '\n')
7505 p--;
7513 /* size of the remaining log content to the right from the to-be-replaced part */
7514 size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
7519 * shift log contents by (patch_sz - orig_sz) bytes to the right
7520 * starting from after to-be-replaced part of the log.
7523 * shift log contents by (orig_sz - patch_sz) bytes to the left
7524 * starting from after to-be-replaced part of the log
7533 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
7535 } else if (patch_sz - orig_sz > buf_sz - log_sz) {
7537 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
7551 /* Expected log for failed and not properly guarded CO-RE relocation:
7552 * line1 -> 123: (85) call unknown#195896080
7553 * line2 -> invalid func unknown#195896080
7554 * line3 -> <anything else or end of buffer>
7557 * instruction index to find corresponding CO-RE relocation and
7559 * failed CO-RE relocation.
7573 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
7579 "%d: <invalid CO-RE relocation>\n"
7580 "failed to resolve CO-RE relocation %s%s\n",
7583 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7591 * line1 -> 123: (85) call unknown#2001000345
7592 * line2 -> invalid func unknown#2001000345
7593 * line3 -> <anything else or end of buffer>
7596 * "345" in "2001000345" is a map index in obj->maps to fetch map name.
7598 struct bpf_object *obj = prog->obj;
7606 map_idx -= POISON_LDIMM64_MAP_BASE;
7607 if (map_idx < 0 || map_idx >= obj->nr_maps)
7609 map = &obj->maps[map_idx];
7614 insn_idx, map->name);
7616 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7624 * line1 -> 123: (85) call unknown#2002000345
7625 * line2 -> invalid func unknown#2002000345
7626 * line3 -> <anything else or end of buffer>
7629 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name.
7631 struct bpf_object *obj = prog->obj;
7639 ext_idx -= POISON_CALL_KFUNC_BASE;
7640 if (ext_idx < 0 || ext_idx >= obj->nr_extern)
7642 ext = &obj->externs[ext_idx];
7647 insn_idx, ext->name);
7649 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7664 next_line = buf + log_sz - 1;
7676 /* failed CO-RE relocation case */
7704 struct bpf_object *obj = prog->obj;
7707 for (i = 0; i < prog->nr_reloc; i++) {
7708 struct reloc_desc *relo = &prog->reloc_desc[i];
7709 struct extern_desc *ext = &obj->externs[relo->ext_idx];
7712 switch (relo->type) {
7714 if (ext->type != EXT_KSYM)
7716 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ?
7718 bpf_gen__record_extern(obj->gen_loader, ext->name,
7719 ext->is_weak, !ext->ksym.type_id,
7720 true, kind, relo->insn_idx);
7723 bpf_gen__record_extern(obj->gen_loader, ext->name,
7724 ext->is_weak, false, false, BTF_KIND_FUNC,
7725 relo->insn_idx);
7729 .insn_off = relo->insn_idx * 8,
7730 .type_id = relo->core_relo->type_id,
7731 .access_str_off = relo->core_relo->access_str_off,
7732 .kind = relo->core_relo->kind,
7735 bpf_gen__record_relo_core(obj->gen_loader, &cr);
7752 for (i = 0; i < obj->nr_programs; i++) {
7753 prog = &obj->programs[i];
7759 for (i = 0; i < obj->nr_programs; i++) {
7760 prog = &obj->programs[i];
7763 if (!prog->autoload) {
7764 pr_debug("prog '%s': skipped loading\n", prog->name);
7767 prog->log_level |= log_level;
7769 if (obj->gen_loader)
7772 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7773 obj->license, obj->kern_version, &prog->fd);
7775 pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
7792 prog->sec_def = find_sec_def(prog->sec_name);
7793 if (!prog->sec_def) {
7796 prog->name, prog->sec_name);
7800 prog->type = prog->sec_def->prog_type;
7801 prog->expected_attach_type = prog->sec_def->expected_attach_type;
7806 if (prog->sec_def->prog_setup_fn) {
7807 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
7810 prog->name, err);
7834 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7839 return ERR_PTR(-EINVAL);
7844 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
7857 return ERR_PTR(-EINVAL);
7859 return ERR_PTR(-EINVAL);
7865 obj->log_buf = log_buf;
7866 obj->log_size = log_size;
7867 obj->log_level = log_level;
7872 err = -ENAMETOOLONG;
7875 obj->btf_custom_path = strdup(btf_tmp_path);
7876 if (!obj->btf_custom_path) {
7877 err = -ENOMEM;
7884 obj->kconfig = strdup(kconfig);
7885 if (!obj->kconfig) {
7886 err = -ENOMEM;
7914 return libbpf_err_ptr(-EINVAL);
7931 return libbpf_err_ptr(-EINVAL);
7941 return libbpf_err(-EINVAL);
7943 for (i = 0; i < obj->nr_maps; i++) {
7944 zclose(obj->maps[i].fd);
7945 if (obj->maps[i].st_ops)
7946 zfree(&obj->maps[i].st_ops->kern_vdata);
7949 for (i = 0; i < obj->nr_programs; i++)
7950 bpf_program__unload(&obj->programs[i]);
7963 m->def.map_flags &= ~BPF_F_MMAPABLE;
7978 err = -errno;
7990 err = -EINVAL;
8011 if (!ext || ext->type != EXT_KSYM)
8014 t = btf__type_by_id(obj->btf, ext->btf_id);
8018 if (ext->is_set && ext->ksym.addr != sym_addr) {
8020 sym_name, ext->ksym.addr, sym_addr);
8021 return -EINVAL;
8023 if (!ext->is_set) {
8024 ext->is_set = true;
8025 ext->ksym.addr = sym_addr;
8044 btf = obj->btf_vmlinux;
8048 if (id == -ENOENT) {
8053 for (i = 0; i < obj->btf_module_cnt; i++) {
8055 mod_btf = &obj->btf_modules[i];
8056 btf = mod_btf->btf;
8058 if (id != -ENOENT)
8063 return -ESRCH;
8080 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
8082 if (id == -ESRCH && ext->is_weak)
8085 ext->name);
8090 local_type_id = ext->ksym.type_id;
8094 targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
8095 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
8097 err = bpf_core_types_are_compat(obj->btf, local_type_id,
8103 local_type = btf__type_by_id(obj->btf, local_type_id);
8104 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
8105 targ_name = btf__name_by_offset(btf, targ_type->name_off);
8108 ext->name, local_type_id,
8111 return -EINVAL;
8114 ext->is_set = true;
8115 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8116 ext->ksym.kernel_btf_id = id;
8118 ext->name, id, btf_kind_str(targ_var), targ_var_name);
8132 local_func_proto_id = ext->ksym.type_id;
8134 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf,
8137 if (kfunc_id == -ESRCH && ext->is_weak)
8140 ext->name);
8145 kfunc_proto_id = kern_func->type;
8147 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
8150 if (ext->is_weak)
8154 ext->name, local_func_proto_id,
8155 mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id);
8156 return -EINVAL;
8160 if (mod_btf && !mod_btf->fd_array_idx) {
8161 /* insn->off is s16 */
8162 if (obj->fd_array_cnt == INT16_MAX) {
8164 ext->name, mod_btf->fd_array_idx);
8165 return -E2BIG;
8168 if (!obj->fd_array_cnt)
8169 obj->fd_array_cnt = 1;
8171 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
8172 obj->fd_array_cnt + 1);
8175 mod_btf->fd_array_idx = obj->fd_array_cnt;
8177 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
8180 ext->is_set = true;
8181 ext->ksym.kernel_btf_id = kfunc_id;
8182 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
8185 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call.
8186 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64.
8188 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8190 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id);
8201 for (i = 0; i < obj->nr_extern; i++) {
8202 ext = &obj->externs[i];
8203 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
8206 if (obj->gen_loader) {
8207 ext->is_set = true;
8208 ext->ksym.kernel_btf_obj_fd = 0;
8209 ext->ksym.kernel_btf_id = 0;
8212 t = btf__type_by_id(obj->btf, ext->btf_id);
8232 if (obj->nr_extern == 0)
8235 if (obj->kconfig_map_idx >= 0)
8236 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
8238 for (i = 0; i < obj->nr_extern; i++) {
8239 ext = &obj->externs[i];
8241 if (ext->type == EXT_KSYM) {
8242 if (ext->ksym.type_id)
8247 } else if (ext->type == EXT_KCFG) {
8248 void *ext_ptr = kcfg_data + ext->kcfg.data_off;
8252 if (str_has_pfx(ext->name, "CONFIG_")) {
8258 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
8261 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
8262 return -EINVAL;
8264 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
8266 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
8268 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
8276 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
8277 return -EINVAL;
8284 ext->name, (long long)value);
8286 pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
8287 return -EINVAL;
8293 return -EINVAL;
8295 for (i = 0; i < obj->nr_extern; i++) {
8296 ext = &obj->externs[i];
8297 if (ext->type == EXT_KCFG && !ext->is_set) {
8306 return -EINVAL;
8311 return -EINVAL;
8316 return -EINVAL;
8318 for (i = 0; i < obj->nr_extern; i++) {
8319 ext = &obj->externs[i];
8321 if (!ext->is_set && !ext->is_weak) {
8322 pr_warn("extern '%s' (strong): not resolved\n", ext->name);
8323 return -ESRCH;
8324 } else if (!ext->is_set) {
8326 ext->name);
8338 st_ops = map->st_ops;
8339 for (i = 0; i < btf_vlen(st_ops->type); i++) {
8340 struct bpf_program *prog = st_ops->progs[i];
8348 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
8357 for (i = 0; i < obj->nr_maps; i++)
8358 if (bpf_map__is_struct_ops(&obj->maps[i]))
8359 bpf_map_prepare_vdata(&obj->maps[i]);
8369 return libbpf_err(-EINVAL);
8371 if (obj->loaded) {
8372 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
8373 return libbpf_err(-EINVAL);
8376 if (obj->gen_loader)
8377 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
8381 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
8386 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
8391 if (obj->gen_loader) {
8393 if (obj->btf)
8394 btf__set_fd(obj->btf, -1);
8395 for (i = 0; i < obj->nr_maps; i++)
8396 obj->maps[i].fd = -1;
8398 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
8402 zfree(&obj->fd_array);
8405 for (i = 0; i < obj->btf_module_cnt; i++) {
8406 close(obj->btf_modules[i].fd);
8407 btf__free(obj->btf_modules[i].btf);
8408 free(obj->btf_modules[i].name);
8410 free(obj->btf_modules);
8413 btf__free(obj->btf_vmlinux);
8414 obj->btf_vmlinux = NULL;
8416 obj->loaded = true; /* doesn't matter if successfully or not */
8423 /* unpin any maps that were auto-pinned during load */
8424 for (i = 0; i < obj->nr_maps; i++)
8425 if (obj->maps[i].pinned && !obj->maps[i].reused)
8426 bpf_map__unpin(&obj->maps[i], NULL);
8429 pr_warn("failed to load object '%s'\n", obj->path);
8446 return -ENOMEM;
8450 err = -errno;
8454 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8468 return -EINVAL;
8472 return -ENOMEM;
8478 err = -errno;
8484 err = -EINVAL;
8495 if (prog->fd < 0) {
8496 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
8497 return libbpf_err(-EINVAL);
8508 if (bpf_obj_pin(prog->fd, path)) {
8509 err = -errno;
8511 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
8515 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
8523 if (prog->fd < 0) {
8524 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
8525 return libbpf_err(-EINVAL);
8534 return libbpf_err(-errno);
8536 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
8547 return libbpf_err(-EINVAL);
8550 if (map->pin_path) {
8551 if (path && strcmp(path, map->pin_path)) {
8553 bpf_map__name(map), map->pin_path, path);
8554 return libbpf_err(-EINVAL);
8555 } else if (map->pinned) {
8556 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
8557 bpf_map__name(map), map->pin_path);
8564 return libbpf_err(-EINVAL);
8565 } else if (map->pinned) {
8567 return libbpf_err(-EEXIST);
8570 map->pin_path = strdup(path);
8571 if (!map->pin_path) {
8572 err = -errno;
8577 err = make_parent_dir(map->pin_path);
8581 err = check_path(map->pin_path);
8585 if (bpf_obj_pin(map->fd, map->pin_path)) {
8586 err = -errno;
8590 map->pinned = true;
8591 pr_debug("pinned map '%s'\n", map->pin_path);
8596 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8607 return libbpf_err(-EINVAL);
8610 if (map->pin_path) {
8611 if (path && strcmp(path, map->pin_path)) {
8613 bpf_map__name(map), map->pin_path, path);
8614 return libbpf_err(-EINVAL);
8616 path = map->pin_path;
8620 return libbpf_err(-EINVAL);
8629 return libbpf_err(-errno);
8631 map->pinned = false;
8644 return libbpf_err(-errno);
8647 free(map->pin_path);
8648 map->pin_path = new;
8657 return map->pin_path;
8662 return map->pinned;
8681 return libbpf_err(-ENOENT);
8683 if (!obj->loaded) {
8685 return libbpf_err(-ENOENT);
8692 if (!map->autocreate)
8701 } else if (!map->pin_path) {
8714 if (!map->pin_path)
8729 return libbpf_err(-ENOENT);
8741 } else if (!map->pin_path) {
8760 return libbpf_err(-ENOENT);
8762 if (!obj->loaded) {
8764 return libbpf_err(-ENOENT);
8768 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8781 if (pathname_concat(buf, sizeof(buf), path, prog->name))
8796 return libbpf_err(-ENOENT);
8801 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8847 if (map->inner_map) {
8848 bpf_map__destroy(map->inner_map);
8849 zfree(&map->inner_map);
8852 zfree(&map->init_slots);
8853 map->init_slots_sz = 0;
8855 if (map->mmaped) {
8858 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
8859 munmap(map->mmaped, mmap_sz);
8860 map->mmaped = NULL;
8863 if (map->st_ops) {
8864 zfree(&map->st_ops->data);
8865 zfree(&map->st_ops->progs);
8866 zfree(&map->st_ops->kern_func_off);
8867 zfree(&map->st_ops);
8870 zfree(&map->name);
8871 zfree(&map->real_name);
8872 zfree(&map->pin_path);
8874 if (map->fd >= 0)
8875 zclose(map->fd);
8885 usdt_manager_free(obj->usdt_man);
8886 obj->usdt_man = NULL;
8888 bpf_gen__free(obj->gen_loader);
8891 btf__free(obj->btf);
8892 btf__free(obj->btf_vmlinux);
8893 btf_ext__free(obj->btf_ext);
8895 for (i = 0; i < obj->nr_maps; i++)
8896 bpf_map__destroy(&obj->maps[i]);
8898 zfree(&obj->btf_custom_path);
8899 zfree(&obj->kconfig);
8901 for (i = 0; i < obj->nr_extern; i++)
8902 zfree(&obj->externs[i].essent_name);
8904 zfree(&obj->externs);
8905 obj->nr_extern = 0;
8907 zfree(&obj->maps);
8908 obj->nr_maps = 0;
8910 if (obj->programs && obj->nr_programs) {
8911 for (i = 0; i < obj->nr_programs; i++)
8912 bpf_program__exit(&obj->programs[i]);
8914 zfree(&obj->programs);
8921 return obj ? obj->name : libbpf_err_ptr(-EINVAL);
8926 return obj ? obj->kern_version : 0;
8931 return obj ? obj->btf : NULL;
8936 return obj->btf ? btf__fd(obj->btf) : -1;
8941 if (obj->loaded)
8942 return libbpf_err(-EINVAL);
8944 obj->kern_version = kern_version;
8954 return -EFAULT;
8956 return -EINVAL;
8959 return -ENOMEM;
8960 gen->opts = opts;
8961 obj->gen_loader = gen;
8969 size_t nr_programs = obj->nr_programs;
8977 return forward ? &obj->programs[0] :
8978 &obj->programs[nr_programs - 1];
8980 if (p->obj != obj) {
8985 idx = (p - obj->programs) + (forward ? 1 : -1);
8986 if (idx >= obj->nr_programs || idx < 0)
8988 return &obj->programs[idx];
9017 prog->prog_ifindex = ifindex;
9022 return prog->name;
9027 return prog->sec_name;
9032 return prog->autoload;
9037 if (prog->obj->loaded)
9038 return libbpf_err(-EINVAL);
9040 prog->autoload = autoload;
9046 return prog->autoattach;
9051 prog->autoattach = autoattach;
9056 return prog->insns;
9061 return prog->insns_cnt;
9069 if (prog->obj->loaded)
9070 return -EBUSY;
9072 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
9075 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
9076 return -ENOMEM;
9080 prog->insns = insns;
9081 prog->insns_cnt = new_insn_cnt;
9088 return libbpf_err(-EINVAL);
9090 if (prog->fd < 0)
9091 return libbpf_err(-ENOENT);
9093 return prog->fd;
9101 return prog->type;
9112 if (prog->obj->loaded)
9113 return libbpf_err(-EBUSY);
9116 if (prog->type == type)
9119 prog->type = type;
9123 * fallback handler, which by definition is program type-agnostic and
9124 * is a catch-all custom handler, optionally set by the application,
9127 if (prog->sec_def != &custom_fallback_def)
9128 prog->sec_def = NULL;
9137 return prog->expected_attach_type;
9143 if (prog->obj->loaded)
9144 return libbpf_err(-EBUSY);
9146 prog->expected_attach_type = type;
9152 return prog->prog_flags;
9157 if (prog->obj->loaded)
9158 return libbpf_err(-EBUSY);
9160 prog->prog_flags = flags;
9166 return prog->log_level;
9171 if (prog->obj->loaded)
9172 return libbpf_err(-EBUSY);
9174 prog->log_level = log_level;
9180 *log_size = prog->log_size;
9181 return prog->log_buf;
9187 return -EINVAL;
9188 if (prog->log_size > UINT_MAX)
9189 return -EINVAL;
9190 if (prog->obj->loaded)
9191 return -EBUSY;
9193 prog->log_buf = log_buf;
9194 prog->log_size = log_size;
9318 SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT),
9333 return libbpf_err(-EINVAL);
9336 return libbpf_err(-E2BIG);
9342 return libbpf_err(-ENOMEM);
9348 return libbpf_err(-EBUSY);
9353 sec_def->sec = sec ? strdup(sec) : NULL;
9354 if (sec && !sec_def->sec)
9355 return libbpf_err(-ENOMEM);
9357 sec_def->prog_type = prog_type;
9358 sec_def->expected_attach_type = exp_attach_type;
9359 sec_def->cookie = OPTS_GET(opts, cookie, 0);
9361 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
9362 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
9363 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
9365 sec_def->handler_id = ++last_custom_sec_def_handler_id;
9372 return sec_def->handler_id;
9381 return libbpf_err(-EINVAL);
9395 return libbpf_err(-ENOENT);
9399 custom_sec_defs[i - 1] = custom_sec_defs[i];
9400 custom_sec_def_cnt--;
9416 size_t len = strlen(sec_def->sec);
9419 if (sec_def->sec[len - 1] == '/') {
9420 if (str_has_pfx(sec_name, sec_def->sec))
9426 * well-formed SEC("type/extras") with proper '/' separator
9428 if (sec_def->sec[len - 1] == '+') {
9429 len--;
9431 if (strncmp(sec_name, sec_def->sec, len) != 0)
9439 return strcmp(sec_name, sec_def->sec) == 0;
9484 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9487 if (!(sec_def->cookie & SEC_ATTACHABLE))
9509 return libbpf_err(-EINVAL);
9513 *prog_type = sec_def->prog_type;
9514 *expected_attach_type = sec_def->expected_attach_type;
9525 return libbpf_err(-ESRCH);
9567 for (i = 0; i < obj->nr_maps; i++) {
9568 map = &obj->maps[i];
9571 if (map->sec_idx == sec_idx &&
9572 map->sec_offset <= offset &&
9573 offset - map->sec_offset < map->def.value_size)
9580 /* Collect the reloc from ELF and populate the st_ops->progs[] */
9597 btf = obj->btf;
9598 nrels = shdr->sh_size / shdr->sh_entsize;
9603 return -LIBBPF_ERRNO__FORMAT;
9606 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
9609 (size_t)ELF64_R_SYM(rel->r_info));
9610 return -LIBBPF_ERRNO__FORMAT;
9613 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
9614 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset);
9616 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
9617 (size_t)rel->r_offset);
9618 return -EINVAL;
9621 moff = rel->r_offset - map->sec_offset;
9622 shdr_idx = sym->st_shndx;
9623 st_ops = map->st_ops;
9624 …pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %…
9625 map->name,
9626 (long long)(rel->r_info >> 32),
9627 (long long)sym->st_value,
9628 shdr_idx, (size_t)rel->r_offset,
9629 map->sec_offset, sym->st_name, name);
9632 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
9633 map->name, (size_t)rel->r_offset, shdr_idx);
9634 return -LIBBPF_ERRNO__RELOC;
9636 if (sym->st_value % BPF_INSN_SZ) {
9638 map->name, (unsigned long long)sym->st_value);
9639 return -LIBBPF_ERRNO__FORMAT;
9641 insn_idx = sym->st_value / BPF_INSN_SZ;
9643 member = find_member_by_offset(st_ops->type, moff * 8);
9646 map->name, moff);
9647 return -EINVAL;
9649 member_idx = member - btf_members(st_ops->type);
9650 name = btf__name_by_offset(btf, member->name_off);
9652 if (!resolve_func_ptr(btf, member->type, NULL)) {
9654 map->name, name);
9655 return -EINVAL;
9661 map->name, shdr_idx, name);
9662 return -EINVAL;
9666 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
9668 map->name, prog->name);
9669 return -EINVAL;
9675 if (!prog->attach_btf_id) {
9676 prog->attach_btf_id = st_ops->type_id;
9677 prog->expected_attach_type = member_idx;
9680 /* struct_ops BPF prog can be re-used between multiple
9685 if (prog->attach_btf_id != st_ops->type_id ||
9686 prog->expected_attach_type != member_idx) {
9688 map->name, prog->name, prog->sec_name, prog->type,
9689 prog->attach_btf_id, prog->expected_attach_type, name);
9690 return -EINVAL;
9693 st_ops->progs[member_idx] = prog;
9740 return -ENAMETOOLONG;
9790 err = -EINVAL;
9817 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
9823 if (ret != -ENOENT)
9830 for (i = 0; i < obj->btf_module_cnt; i++) {
9831 const struct module_btf *mod = &obj->btf_modules[i];
9833 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
9835 *btf_obj_fd = mod->fd;
9839 if (ret == -ENOENT)
9845 return -ESRCH;
9851 enum bpf_attach_type attach_type = prog->expected_attach_type;
9852 __u32 attach_prog_fd = prog->attach_prog_fd;
9856 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
9858 pr_warn("prog '%s': attach program FD is not set\n", prog->name);
9859 return -EINVAL;
9864 prog->name, attach_prog_fd, attach_name, err);
9873 if (prog->obj->gen_loader) {
9874 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
9878 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9882 prog->name, attach_name, err);
9895 return libbpf_err(-EINVAL);
9906 return libbpf_err(-EINVAL);
9909 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9910 return libbpf_err(-EINVAL);
9911 if (!(sec_def->cookie & SEC_ATTACHABLE))
9912 return libbpf_err(-EINVAL);
9914 *attach_type = sec_def->expected_attach_type;
9920 return map ? map->fd : libbpf_err(-EINVAL);
9926 * their user-visible name differs from kernel-visible name. Users see
9931 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
9933 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
9944 return map->real_name;
9946 return map->name;
9951 return map->def.type;
9956 if (map->fd >= 0)
9957 return libbpf_err(-EBUSY);
9958 map->def.type = type;
9964 return map->def.map_flags;
9969 if (map->fd >= 0)
9970 return libbpf_err(-EBUSY);
9971 map->def.map_flags = flags;
9977 return map->map_extra;
9982 if (map->fd >= 0)
9983 return libbpf_err(-EBUSY);
9984 map->map_extra = map_extra;
9990 return map->numa_node;
9995 if (map->fd >= 0)
9996 return libbpf_err(-EBUSY);
9997 map->numa_node = numa_node;
10003 return map->def.key_size;
10008 if (map->fd >= 0)
10009 return libbpf_err(-EBUSY);
10010 map->def.key_size = size;
10016 return map->def.value_size;
10030 btf = bpf_object__btf(map->obj);
10032 return -ENOENT;
10039 return -EINVAL;
10047 return -EINVAL;
10051 var = &btf_var_secinfos(datasec_type)[vlen - 1];
10052 var_type = btf_type_by_id(btf, var->type);
10053 array_type = skip_mods_and_typedefs(btf, var_type->type, NULL);
10057 return -EINVAL;
10062 element_sz = btf__resolve_size(btf, array->type);
10063 if (element_sz <= 0 || (size - var->offset) % element_sz != 0) {
10066 return -EINVAL;
10070 nr_elements = (size - var->offset) / element_sz;
10071 new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements);
10078 datasec_type = btf_type_by_id(btf, map->btf_value_type_id);
10079 var = &btf_var_secinfos(datasec_type)[vlen - 1];
10080 var_type = btf_type_by_id(btf, var->type);
10083 datasec_type->size = size;
10084 var->size = size - var->offset;
10085 var_type->type = new_array_id;
10092 if (map->fd >= 0)
10093 return libbpf_err(-EBUSY);
10095 if (map->mmaped) {
10099 mmap_old_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
10100 mmap_new_sz = bpf_map_mmap_sz(size, map->def.max_entries);
10103 pr_warn("map '%s': failed to resize memory-mapped region: %d\n",
10108 if (err && err != -ENOENT) {
10111 map->btf_value_type_id = 0;
10112 map->btf_key_type_id = 0;
10116 map->def.value_size = size;
10122 return map ? map->btf_key_type_id : 0;
10127 return map ? map->btf_value_type_id : 0;
10133 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
10134 size != map->def.value_size || map->fd >= 0)
10135 return libbpf_err(-EINVAL);
10137 memcpy(map->mmaped, data, size);
10143 if (!map->mmaped)
10145 *psize = map->def.value_size;
10146 return map->mmaped;
10151 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
10156 return map->map_ifindex;
10161 if (map->fd >= 0)
10162 return libbpf_err(-EBUSY);
10163 map->map_ifindex = ifindex;
10169 if (!bpf_map_type__is_map_in_map(map->def.type)) {
10171 return libbpf_err(-EINVAL);
10173 if (map->inner_map_fd != -1) {
10175 return libbpf_err(-EINVAL);
10177 if (map->inner_map) {
10178 bpf_map__destroy(map->inner_map);
10179 zfree(&map->inner_map);
10181 map->inner_map_fd = fd;
10191 if (!obj || !obj->maps)
10194 s = obj->maps;
10195 e = obj->maps + obj->nr_maps;
10203 idx = (m - obj->maps) + i;
10204 if (idx >= obj->nr_maps || idx < 0)
10206 return &obj->maps[idx];
10213 return obj->maps;
10222 if (!obj->nr_maps)
10224 return obj->maps + obj->nr_maps - 1;
10227 return __bpf_map__iter(next, obj, -1);
10241 if (pos->real_name && strcmp(pos->real_name, name) == 0)
10247 if (strcmp(pos->real_name, name) == 0)
10251 if (strcmp(pos->name, name) == 0)
10266 if (map->fd <= 0)
10267 return -ENOENT;
10269 if (map->def.key_size != key_sz) {
10271 map->name, key_sz, map->def.key_size);
10272 return -EINVAL;
10278 switch (map->def.type) {
10284 size_t elem_sz = roundup(map->def.value_size, 8);
10287 … pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",
10288 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
10289 return -EINVAL;
10294 if (map->def.value_size != value_sz) {
10296 map->name, value_sz, map->def.value_size);
10297 return -EINVAL;
10314 return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
10327 return bpf_map_update_elem(map->fd, key, value, flags);
10339 return bpf_map_delete_elem_flags(map->fd, key, flags);
10352 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
10364 return bpf_map_get_next_key(map->fd, cur_key, next_key);
10373 errno = -PTR_ERR(ptr);
10380 return -errno;
10404 link->disconnected = true;
10414 if (!link->disconnected && link->detach)
10415 err = link->detach(link);
10416 if (link->pin_path)
10417 free(link->pin_path);
10418 if (link->dealloc)
10419 link->dealloc(link);
10428 return link->fd;
10433 return link->pin_path;
10438 return libbpf_err_errno(close(link->fd));
10448 fd = -errno;
10456 return libbpf_err_ptr(-ENOMEM);
10458 link->detach = &bpf_link__detach_fd;
10459 link->fd = fd;
10461 link->pin_path = strdup(path);
10462 if (!link->pin_path) {
10464 return libbpf_err_ptr(-ENOMEM);
10472 return bpf_link_detach(link->fd) ? -errno : 0;
10479 if (link->pin_path)
10480 return libbpf_err(-EBUSY);
10488 link->pin_path = strdup(path);
10489 if (!link->pin_path)
10490 return libbpf_err(-ENOMEM);
10492 if (bpf_obj_pin(link->fd, link->pin_path)) {
10493 err = -errno;
10494 zfree(&link->pin_path);
10498 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
10506 if (!link->pin_path)
10507 return libbpf_err(-EINVAL);
10509 err = unlink(link->pin_path);
10511 return -errno;
10513 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
10514 zfree(&link->pin_path);
10535 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
10536 err = -errno;
10538 if (perf_link->perf_event_fd != link->fd)
10539 close(perf_link->perf_event_fd);
10540 close(link->fd);
10543 if (perf_link->legacy_probe_name) {
10544 if (perf_link->legacy_is_kprobe) {
10545 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
10546 perf_link->legacy_is_retprobe);
10548 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
10549 perf_link->legacy_is_retprobe);
10560 free(perf_link->legacy_probe_name);
10569 int prog_fd, link_fd = -1, err;
10573 return libbpf_err_ptr(-EINVAL);
10577 prog->name, pfd);
10578 return libbpf_err_ptr(-EINVAL);
10583 prog->name);
10584 return libbpf_err_ptr(-EINVAL);
10589 return libbpf_err_ptr(-ENOMEM);
10590 link->link.detach = &bpf_link_perf_detach;
10591 link->link.dealloc = &bpf_link_perf_dealloc;
10592 link->perf_event_fd = pfd;
10595 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {
10601 err = -errno;
10603 prog->name, pfd,
10607 link->link.fd = link_fd;
10610 pr_warn("prog '%s': user context value is not supported\n", prog->name);
10611 err = -EOPNOTSUPP;
10616 err = -errno;
10618 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10619 if (err == -EPROTO)
10621 prog->name, pfd);
10624 link->link.fd = pfd;
10627 err = -errno;
10629 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10633 return &link->link;
10647 * this function is expected to parse integer in the range of [0, 2^31-1] from
10659 err = -errno;
10666 err = err == EOF ? -EIO : -errno;
10716 return -EINVAL;
10748 pid < 0 ? -1 : pid /* pid */,
10749 pid == -1 ? 0 : -1 /* cpu */,
10750 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10751 return pfd >= 0 ? pfd : -errno;
10765 return -EINVAL;
10769 return -errno;
10772 err = -errno;
10783 static int has_debugfs = -1;
10845 return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
10889 pid < 0 ? -1 : pid, /* pid */
10890 pid == -1 ? 0 : -1, /* cpu */
10891 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10893 err = -errno;
10979 return libbpf_err_ptr(-EINVAL);
10994 return libbpf_err_ptr(-ENOTSUP);
10998 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
10999 return libbpf_err_ptr(-ENOTSUP);
11004 return libbpf_err_ptr(-EINVAL);
11010 -1 /* pid */, 0 /* ref_ctr_off */);
11019 return libbpf_err_ptr(-ENOMEM);
11022 offset, -1 /* pid */);
11025 err = -errno;
11027 prog->name, retprobe ? "kretprobe" : "kprobe",
11037 prog->name, retprobe ? "kretprobe" : "kprobe",
11045 perf_link->legacy_probe_name = legacy_probe;
11046 perf_link->legacy_is_kprobe = true;
11047 perf_link->legacy_is_retprobe = retprobe;
11079 return libbpf_err_ptr(-EINVAL);
11081 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
11148 struct kprobe_multi_resolve *res = data->res;
11151 if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp))
11154 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1);
11158 res->addrs[res->cnt++] = (unsigned long)sym_addr;
11174 err = -errno;
11188 err = -EINVAL;
11192 if (!glob_match(sym_name, res->pattern))
11201 err = -errno;
11210 err = -ENOENT;
11222 if (res->cnt == 0)
11223 err = -ENOENT;
11236 return access(tracefs_available_filter_functions_addrs(), R_OK) != -1;
11249 err = -errno;
11262 err = -EINVAL;
11266 if (!glob_match(sym_name, res->pattern))
11269 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap,
11270 sizeof(*res->addrs), res->cnt + 1);
11274 res->addrs[res->cnt++] = (unsigned long)sym_addr;
11277 if (res->cnt == 0)
11278 err = -ENOENT;
11304 return libbpf_err_ptr(-EINVAL);
11312 return libbpf_err_ptr(-EINVAL);
11314 return libbpf_err_ptr(-EINVAL);
11316 return libbpf_err_ptr(-EINVAL);
11318 return libbpf_err_ptr(-EINVAL);
11341 err = -ENOMEM;
11344 link->detach = &bpf_link__detach_fd;
11349 err = -errno;
11351 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11354 link->fd = link_fd;
11374 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
11375 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
11378 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
11380 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
11382 func_name = prog->sec_name + sizeof("kprobe/") - 1;
11384 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
11387 return -EINVAL;
11392 return -EINVAL;
11408 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
11409 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
11412 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
11414 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
11416 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
11419 return *link ? 0 : -errno;
11431 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
11432 if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
11433 strcmp(prog->sec_name, "kretprobe.multi") == 0)
11436 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
11438 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
11440 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
11442 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
11445 return -EINVAL;
11457 int n, ret = -EINVAL;
11461 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
11465 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
11470 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
11474 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
11475 prog->sec_name);
11509 return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
11550 pid < 0 ? -1 : pid, /* pid */
11551 pid == -1 ? 0 : -1, /* cpu */
11552 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11554 err = -errno;
11600 ret = -LIBBPF_ERRNO__FORMAT;
11616 elf_errmsg(-1));
11617 ret = -LIBBPF_ERRNO__LIBELF;
11641 * Based on https://packages.debian.org/sid/libc6.
11647 return "/lib/x86_64-linux-gnu";
11649 return "/lib/i386-linux-gnu";
11651 return "/lib/s390x-linux-gnu";
11653 return "/lib/s390-linux-gnu";
11655 return "/lib/arm-linux-gnueabi";
11657 return "/lib/arm-linux-gnueabihf";
11659 return "/lib/aarch64-linux-gnu";
11661 return "/lib/mips64el-linux-gnuabi64";
11663 return "/lib/mipsel-linux-gnu";
11665 return "/lib/powerpc64le-linux-gnu";
11667 return "/lib/sparc64-linux-gnu";
11669 return "/lib/riscv64-linux-gnu";
11704 seg_len = next_path ? next_path - s : strlen(s);
11715 return -ENOENT;
11737 return libbpf_err_ptr(-EINVAL);
11753 * - syms and offsets are mutually exclusive
11754 * - ref_ctr_offsets and cookies are optional
11760 return libbpf_err_ptr(-EINVAL);
11762 return libbpf_err_ptr(-EINVAL);
11766 return libbpf_err_ptr(-EINVAL);
11769 return libbpf_err_ptr(-EINVAL);
11777 prog->name, path, err);
11809 err = -ENOMEM;
11812 link->detach = &bpf_link__detach_fd;
11817 err = -errno;
11818 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
11819 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11822 link->fd = link_fd;
11849 return libbpf_err_ptr(-EINVAL);
11857 return libbpf_err_ptr(-EINVAL);
11864 min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1)));
11871 prog->name, binary_path, err);
11900 return libbpf_err_ptr(-ENOTSUP);
11904 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
11905 return libbpf_err_ptr(-ENOTSUP);
11910 return libbpf_err_ptr(-EINVAL);
11920 return libbpf_err_ptr(-EINVAL);
11927 return libbpf_err_ptr(-ENOMEM);
11933 err = -errno;
11935 prog->name, retprobe ? "uretprobe" : "uprobe",
11946 prog->name, retprobe ? "uretprobe" : "uprobe",
11954 perf_link->legacy_probe_name = legacy_probe;
11955 perf_link->legacy_is_kprobe = false;
11956 perf_link->legacy_is_retprobe = retprobe;
11968 /* Format of u[ret]probe section definition supporting auto-attach:
11975 * specified (and auto-attach is not possible) or the above format is specified for
11976 * auto-attach.
11982 int n, c, ret = -EINVAL;
11987 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
11991 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
11996 prog->name, prog->sec_name);
12014 prog->name);
12018 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
12022 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
12023 prog->sec_name);
12050 struct bpf_object *obj = prog->obj;
12056 return libbpf_err_ptr(-EINVAL);
12060 prog->name);
12061 return libbpf_err_ptr(-EINVAL);
12065 return libbpf_err_ptr(-EINVAL);
12071 prog->name, binary_path, err);
12080 if (IS_ERR(obj->usdt_man))
12081 return libbpf_ptr(obj->usdt_man);
12082 if (!obj->usdt_man) {
12083 obj->usdt_man = usdt_manager_new(obj);
12084 if (IS_ERR(obj->usdt_man))
12085 return libbpf_ptr(obj->usdt_man);
12089 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
12107 /* no auto-attach for just SEC("usdt") */
12116 err = -EINVAL;
12118 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
12138 return -errno;
12142 return -E2BIG;
12168 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
12169 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
12171 err = -errno;
12191 return libbpf_err_ptr(-EINVAL);
12198 prog->name, tp_category, tp_name,
12207 prog->name, tp_category, tp_name,
12227 /* no auto-attach for SEC("tp") or SEC("tracepoint") */
12228 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
12231 sec_name = strdup(prog->sec_name);
12233 return -ENOMEM;
12236 if (str_has_pfx(prog->sec_name, "tp/"))
12237 tp_cat = sec_name + sizeof("tp/") - 1;
12239 tp_cat = sec_name + sizeof("tracepoint/") - 1;
12243 return -EINVAL;
12262 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12263 return libbpf_err_ptr(-EINVAL);
12268 return libbpf_err_ptr(-ENOMEM);
12269 link->detach = &bpf_link__detach_fd;
12273 pfd = -errno;
12276 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12279 link->fd = pfd;
12299 if (!str_has_pfx(prog->sec_name, prefixes[i]))
12303 /* no auto-attach case of, e.g., SEC("raw_tp") */
12304 if (prog->sec_name[pfx_len] == '\0')
12307 if (prog->sec_name[pfx_len] != '/')
12310 tp_name = prog->sec_name + pfx_len + 1;
12316 prog->name, prog->sec_name);
12317 return -EINVAL;
12334 return libbpf_err_ptr(-EINVAL);
12338 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12339 return libbpf_err_ptr(-EINVAL);
12344 return libbpf_err_ptr(-ENOMEM);
12345 link->detach = &bpf_link__detach_fd;
12351 pfd = -errno;
12354 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12357 link->fd = pfd;
12401 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12402 return libbpf_err_ptr(-EINVAL);
12407 return libbpf_err_ptr(-ENOMEM);
12408 link->detach = &bpf_link__detach_fd;
12413 link_fd = -errno;
12416 prog->name, target_name,
12420 link->fd = link_fd;
12451 return libbpf_err_ptr(-EINVAL);
12456 /* validate we don't have unexpected combinations of non-zero fields */
12459 prog->name);
12460 return libbpf_err_ptr(-EINVAL);
12464 prog->name);
12465 return libbpf_err_ptr(-EINVAL);
12486 return libbpf_err_ptr(-EINVAL);
12491 /* validate we don't have unexpected combinations of non-zero fields */
12494 prog->name);
12495 return libbpf_err_ptr(-EINVAL);
12499 prog->name);
12500 return libbpf_err_ptr(-EINVAL);
12519 prog->name);
12520 return libbpf_err_ptr(-EINVAL);
12523 if (prog->type != BPF_PROG_TYPE_EXT) {
12525 prog->name);
12526 return libbpf_err_ptr(-EINVAL);
12559 return libbpf_err_ptr(-EINVAL);
12566 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12567 return libbpf_err_ptr(-EINVAL);
12572 return libbpf_err_ptr(-ENOMEM);
12573 link->detach = &bpf_link__detach_fd;
12578 link_fd = -errno;
12581 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12584 link->fd = link_fd;
12602 return libbpf_err_ptr(-EINVAL);
12606 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12607 return libbpf_err_ptr(-EINVAL);
12612 return libbpf_err_ptr(-ENOMEM);
12614 link->detach = &bpf_link__detach_fd;
12625 link_fd = -errno;
12628 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12631 link->fd = link_fd;
12641 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
12642 return libbpf_err_ptr(-EOPNOTSUPP);
12644 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
12648 /* When calling bpf_program__attach() explicitly, auto-attach support
12654 return libbpf_err_ptr(-EOPNOTSUPP);
12671 if (st_link->map_fd < 0)
12673 return bpf_map_delete_elem(link->fd, &zero);
12675 return close(link->fd);
12684 if (!bpf_map__is_struct_ops(map) || map->fd == -1)
12685 return libbpf_err_ptr(-EINVAL);
12689 return libbpf_err_ptr(-EINVAL);
12692 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
12698 if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) {
12703 link->link.detach = bpf_link__detach_struct_ops;
12705 if (!(map->def.map_flags & BPF_F_LINK)) {
12707 link->link.fd = map->fd;
12708 link->map_fd = -1;
12709 return &link->link;
12712 fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL);
12718 link->link.fd = fd;
12719 link->map_fd = map->fd;
12721 return &link->link;
12733 if (!bpf_map__is_struct_ops(map) || map->fd < 0)
12734 return -EINVAL;
12738 if (st_ops_link->map_fd < 0)
12739 return -EINVAL;
12741 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
12747 if (err && err != -EBUSY)
12750 err = bpf_link_update(link->fd, map->fd, NULL);
12754 st_ops_link->map_fd = map->fd;
12769 __u64 data_tail = header->data_tail;
12776 ehdr = base + (data_tail & (mmap_size - 1));
12777 ehdr_size = ehdr->size;
12781 size_t len_first = base + mmap_size - copy_start;
12782 size_t len_secnd = ehdr_size - len_first;
12816 /* sample_cb and lost_cb are higher-level common-case callbacks */
12855 if (cpu_buf->base &&
12856 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
12857 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
12858 if (cpu_buf->fd >= 0) {
12859 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
12860 close(cpu_buf->fd);
12862 free(cpu_buf->buf);
12872 if (pb->cpu_bufs) {
12873 for (i = 0; i < pb->cpu_cnt; i++) {
12874 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
12879 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
12882 free(pb->cpu_bufs);
12884 if (pb->epoll_fd >= 0)
12885 close(pb->epoll_fd);
12886 free(pb->events);
12900 return ERR_PTR(-ENOMEM);
12902 cpu_buf->pb = pb;
12903 cpu_buf->cpu = cpu;
12904 cpu_buf->map_key = map_key;
12906 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
12907 -1, PERF_FLAG_FD_CLOEXEC);
12908 if (cpu_buf->fd < 0) {
12909 err = -errno;
12915 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
12917 cpu_buf->fd, 0);
12918 if (cpu_buf->base == MAP_FAILED) {
12919 cpu_buf->base = NULL;
12920 err = -errno;
12926 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
12927 err = -errno;
12955 return libbpf_err_ptr(-EINVAL);
12985 return libbpf_err_ptr(-EINVAL);
12988 return libbpf_err_ptr(-EINVAL);
13011 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {
13014 return ERR_PTR(-EINVAL);
13017 /* best-effort sanity checks */
13022 err = -errno;
13024 * -EBADFD, -EFAULT, or -E2BIG on real error
13026 if (err != -EINVAL) {
13037 return ERR_PTR(-EINVAL);
13043 return ERR_PTR(-ENOMEM);
13045 pb->event_cb = p->event_cb;
13046 pb->sample_cb = p->sample_cb;
13047 pb->lost_cb = p->lost_cb;
13048 pb->ctx = p->ctx;
13050 pb->page_size = getpagesize();
13051 pb->mmap_size = pb->page_size * page_cnt;
13052 pb->map_fd = map_fd;
13054 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
13055 if (pb->epoll_fd < 0) {
13056 err = -errno;
13062 if (p->cpu_cnt > 0) {
13063 pb->cpu_cnt = p->cpu_cnt;
13065 pb->cpu_cnt = libbpf_num_possible_cpus();
13066 if (pb->cpu_cnt < 0) {
13067 err = pb->cpu_cnt;
13070 if (map.max_entries && map.max_entries < pb->cpu_cnt)
13071 pb->cpu_cnt = map.max_entries;
13074 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
13075 if (!pb->events) {
13076 err = -ENOMEM;
13080 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
13081 if (!pb->cpu_bufs) {
13082 err = -ENOMEM;
13093 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
13097 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
13098 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
13103 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
13106 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
13112 pb->cpu_bufs[j] = cpu_buf;
13114 err = bpf_map_update_elem(pb->map_fd, &map_key,
13115 &cpu_buf->fd, 0);
13117 err = -errno;
13118 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
13119 cpu, map_key, cpu_buf->fd,
13124 pb->events[j].events = EPOLLIN;
13125 pb->events[j].data.ptr = cpu_buf;
13126 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
13127 &pb->events[j]) < 0) {
13128 err = -errno;
13130 cpu, cpu_buf->fd,
13136 pb->cpu_cnt = j;
13165 struct perf_buffer *pb = cpu_buf->pb;
13169 if (pb->event_cb)
13170 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
13172 switch (e->type) {
13176 if (pb->sample_cb)
13177 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
13183 if (pb->lost_cb)
13184 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
13188 pr_warn("unknown perf sample type %d\n", e->type);
13199 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
13200 pb->page_size, &cpu_buf->buf,
13201 &cpu_buf->buf_size,
13210 return pb->epoll_fd;
13217 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
13219 return -errno;
13222 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
13238 return pb->cpu_cnt;
13250 if (buf_idx >= pb->cpu_cnt)
13251 return libbpf_err(-EINVAL);
13253 cpu_buf = pb->cpu_bufs[buf_idx];
13255 return libbpf_err(-ENOENT);
13257 return cpu_buf->fd;
13264 if (buf_idx >= pb->cpu_cnt)
13265 return libbpf_err(-EINVAL);
13267 cpu_buf = pb->cpu_bufs[buf_idx];
13269 return libbpf_err(-ENOENT);
13271 *buf = cpu_buf->base;
13272 *buf_size = pb->mmap_size;
13281 * - 0 on success;
13282 * - <0 on failure.
13288 if (buf_idx >= pb->cpu_cnt)
13289 return libbpf_err(-EINVAL);
13291 cpu_buf = pb->cpu_bufs[buf_idx];
13293 return libbpf_err(-ENOENT);
13302 for (i = 0; i < pb->cpu_cnt; i++) {
13303 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
13324 return libbpf_err(-EINVAL);
13326 if (prog->obj->loaded)
13327 return libbpf_err(-EINVAL);
13333 prog->attach_prog_fd = attach_prog_fd;
13344 return libbpf_err(-EINVAL);
13347 err = bpf_object__load_vmlinux_btf(prog->obj, true);
13350 err = find_kernel_btf_id(prog->obj, attach_func_name,
13351 prog->expected_attach_type,
13357 prog->attach_btf_id = btf_id;
13358 prog->attach_btf_obj_fd = btf_obj_fd;
13359 prog->attach_prog_fd = attach_prog_fd;
13365 int err = 0, n, len, start, end = -1;
13371 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
13377 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
13380 err = -EINVAL;
13388 err = -EINVAL;
13393 err = -ENOMEM;
13397 memset(tmp + *mask_sz, 0, start - *mask_sz);
13398 memset(tmp + start, 1, end - start + 1);
13404 return -EINVAL;
13420 err = -errno;
13427 err = len ? -errno : -EINVAL;
13433 return -E2BIG;
13480 return -ESRCH;
13483 /* externs shouldn't be pre-setup from user code */
13484 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
13485 *mmaped = (*map)->mmaped;
13503 return -ESRCH;
13513 .object_name = s->name,
13518 /* Attempt to preserve opts->object_name, unless overriden by user
13526 if (!opts->object_name)
13527 skel_opts.object_name = s->name;
13530 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
13534 s->name, err);
13538 *s->obj = obj;
13539 err = populate_skeleton_maps(obj, s->maps, s->map_cnt);
13541 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);
13545 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt);
13547 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);
13565 if (!s->obj)
13566 return libbpf_err(-EINVAL);
13568 btf = bpf_object__btf(s->obj);
13571 bpf_object__name(s->obj));
13572 return libbpf_err(-errno);
13575 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt);
13581 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt);
13587 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
13588 var_skel = &s->vars[var_idx];
13589 map = *var_skel->map;
13597 return libbpf_err(-EINVAL);
13603 var_type = btf__type_by_id(btf, var->type);
13604 var_name = btf__name_by_offset(btf, var_type->name_off);
13605 if (strcmp(var_name, var_skel->name) == 0) {
13606 *var_skel->addr = map->mmaped + var->offset;
13618 free(s->maps);
13619 free(s->progs);
13620 free(s->vars);
13628 err = bpf_object__load(*s->obj);
13630 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
13634 for (i = 0; i < s->map_cnt; i++) {
13635 struct bpf_map *map = *s->maps[i].map;
13636 size_t mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
13638 void **mmaped = s->maps[i].mmaped;
13643 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
13648 if (map->def.map_flags & BPF_F_RDONLY_PROG)
13653 /* Remap anonymous mmap()-ed "map initialization image" as
13654 * a BPF map-backed mmap()-ed memory, but preserving the same
13663 *mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map_fd, 0);
13665 err = -errno;
13667 pr_warn("failed to re-mmap() map '%s': %d\n",
13680 for (i = 0; i < s->prog_cnt; i++) {
13681 struct bpf_program *prog = *s->progs[i].prog;
13682 struct bpf_link **link = s->progs[i].link;
13684 if (!prog->autoload || !prog->autoattach)
13687 /* auto-attaching not supported for this program */
13688 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
13691 /* if user already set the link manually, don't attempt auto-attach */
13695 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
13697 pr_warn("prog '%s': failed to auto-attach: %d\n",
13702 /* It's possible that for some SEC() definitions auto-attach
13707 * auto-attached. But if not, it shouldn't trigger skeleton's
13721 for (i = 0; i < s->prog_cnt; i++) {
13722 struct bpf_link **link = s->progs[i].link;
13734 if (s->progs)
13736 if (s->obj)
13737 bpf_object__close(*s->obj);
13738 free(s->maps);
13739 free(s->progs);