Lines Matching +full:libc6 +full:- +full:dev +full:- +full:i386
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
71 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
271 if (err != -EPERM || geteuid() != 0) in pr_perm_msg()
288 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", in pr_perm_msg()
304 fd = -1; \
315 /* as of v1.0 libbpf_set_strict_mode() is a no-op */ in libbpf_set_strict_mode()
361 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
380 /* BPF program support non-linear XDP buffer */
418 * program. For the entry-point (main) BPF program, this is always
419 * zero. For a sub-program, this gets reset before each of main BPF
421 * whether sub-program was already appended to the main program, and
433 * entry-point BPF programs this includes the size of main program
434 * itself plus all the used sub-programs, appended at the end
485 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
591 /* BTF fd index to be patched in for insn->off, this is
592 * 0 for vmlinux BTF, index in obj->fd_array for module
692 /* Path to the custom BTF to be used for BPF CO-RE relocations as an
696 /* vmlinux BTF override for CO-RE relocations */
748 zclose(prog->fd); in bpf_program__unload()
750 zfree(&prog->func_info); in bpf_program__unload()
751 zfree(&prog->line_info); in bpf_program__unload()
760 zfree(&prog->name); in bpf_program__exit()
761 zfree(&prog->sec_name); in bpf_program__exit()
762 zfree(&prog->insns); in bpf_program__exit()
763 zfree(&prog->reloc_desc); in bpf_program__exit()
765 prog->nr_reloc = 0; in bpf_program__exit()
766 prog->insns_cnt = 0; in bpf_program__exit()
767 prog->sec_idx = -1; in bpf_program__exit()
772 return BPF_CLASS(insn->code) == BPF_JMP && in insn_is_subprog_call()
773 BPF_OP(insn->code) == BPF_CALL && in insn_is_subprog_call()
774 BPF_SRC(insn->code) == BPF_K && in insn_is_subprog_call()
775 insn->src_reg == BPF_PSEUDO_CALL && in insn_is_subprog_call()
776 insn->dst_reg == 0 && in insn_is_subprog_call()
777 insn->off == 0; in insn_is_subprog_call()
782 return insn->code == (BPF_JMP | BPF_CALL); in is_call_insn()
787 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; in insn_is_pseudo_func()
798 return -EINVAL; in bpf_object__init_prog()
802 prog->obj = obj; in bpf_object__init_prog()
804 prog->sec_idx = sec_idx; in bpf_object__init_prog()
805 prog->sec_insn_off = sec_off / BPF_INSN_SZ; in bpf_object__init_prog()
806 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; in bpf_object__init_prog()
808 prog->insns_cnt = prog->sec_insn_cnt; in bpf_object__init_prog()
810 prog->type = BPF_PROG_TYPE_UNSPEC; in bpf_object__init_prog()
811 prog->fd = -1; in bpf_object__init_prog()
812 prog->exception_cb_idx = -1; in bpf_object__init_prog()
819 prog->autoload = false; in bpf_object__init_prog()
823 prog->autoload = true; in bpf_object__init_prog()
826 prog->autoattach = true; in bpf_object__init_prog()
829 prog->log_level = obj->log_level; in bpf_object__init_prog()
831 prog->sec_name = strdup(sec_name); in bpf_object__init_prog()
832 if (!prog->sec_name) in bpf_object__init_prog()
835 prog->name = strdup(name); in bpf_object__init_prog()
836 if (!prog->name) in bpf_object__init_prog()
839 prog->insns = malloc(insn_data_sz); in bpf_object__init_prog()
840 if (!prog->insns) in bpf_object__init_prog()
842 memcpy(prog->insns, insn_data, insn_data_sz); in bpf_object__init_prog()
848 return -ENOMEM; in bpf_object__init_prog()
855 Elf_Data *symbols = obj->efile.symbols; in bpf_object__add_programs()
857 void *data = sec_data->d_buf; in bpf_object__add_programs()
858 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; in bpf_object__add_programs()
863 progs = obj->programs; in bpf_object__add_programs()
864 nr_progs = obj->nr_programs; in bpf_object__add_programs()
865 nr_syms = symbols->d_size / sizeof(Elf64_Sym); in bpf_object__add_programs()
870 if (sym->st_shndx != sec_idx) in bpf_object__add_programs()
872 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) in bpf_object__add_programs()
875 prog_sz = sym->st_size; in bpf_object__add_programs()
876 sec_off = sym->st_value; in bpf_object__add_programs()
878 name = elf_sym_str(obj, sym->st_name); in bpf_object__add_programs()
882 return -LIBBPF_ERRNO__FORMAT; in bpf_object__add_programs()
888 return -LIBBPF_ERRNO__FORMAT; in bpf_object__add_programs()
891 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { in bpf_object__add_programs()
893 return -ENOTSUP; in bpf_object__add_programs()
902 * In this case the original obj->programs in bpf_object__add_programs()
908 return -ENOMEM; in bpf_object__add_programs()
910 obj->programs = progs; in bpf_object__add_programs()
919 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL) in bpf_object__add_programs()
920 prog->sym_global = true; in bpf_object__add_programs()
927 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN in bpf_object__add_programs()
928 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) in bpf_object__add_programs()
929 prog->mark_btf_static = true; in bpf_object__add_programs()
932 obj->nr_programs = nr_progs; in bpf_object__add_programs()
960 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) in find_member_by_name()
1012 if (kern_data_member->type == kern_type_id) in find_struct_ops_kern_types()
1018 return -EINVAL; in find_struct_ops_kern_types()
1032 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; in bpf_map__is_struct_ops()
1048 st_ops = map->st_ops; in bpf_map__init_kern_struct_ops()
1049 type = st_ops->type; in bpf_map__init_kern_struct_ops()
1050 tname = st_ops->tname; in bpf_map__init_kern_struct_ops()
1059 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); in bpf_map__init_kern_struct_ops()
1061 map->def.value_size = kern_vtype->size; in bpf_map__init_kern_struct_ops()
1062 map->btf_vmlinux_value_type_id = kern_vtype_id; in bpf_map__init_kern_struct_ops()
1064 st_ops->kern_vdata = calloc(1, kern_vtype->size); in bpf_map__init_kern_struct_ops()
1065 if (!st_ops->kern_vdata) in bpf_map__init_kern_struct_ops()
1066 return -ENOMEM; in bpf_map__init_kern_struct_ops()
1068 data = st_ops->data; in bpf_map__init_kern_struct_ops()
1069 kern_data_off = kern_data_member->offset / 8; in bpf_map__init_kern_struct_ops()
1070 kern_data = st_ops->kern_vdata + kern_data_off; in bpf_map__init_kern_struct_ops()
1082 mname = btf__name_by_offset(btf, member->name_off); in bpf_map__init_kern_struct_ops()
1086 map->name, mname); in bpf_map__init_kern_struct_ops()
1087 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1090 kern_member_idx = kern_member - btf_members(kern_type); in bpf_map__init_kern_struct_ops()
1094 map->name, mname); in bpf_map__init_kern_struct_ops()
1095 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1098 moff = member->offset / 8; in bpf_map__init_kern_struct_ops()
1099 kern_moff = kern_member->offset / 8; in bpf_map__init_kern_struct_ops()
1104 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); in bpf_map__init_kern_struct_ops()
1105 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, in bpf_map__init_kern_struct_ops()
1107 if (BTF_INFO_KIND(mtype->info) != in bpf_map__init_kern_struct_ops()
1108 BTF_INFO_KIND(kern_mtype->info)) { in bpf_map__init_kern_struct_ops()
1110 map->name, mname, BTF_INFO_KIND(mtype->info), in bpf_map__init_kern_struct_ops()
1111 BTF_INFO_KIND(kern_mtype->info)); in bpf_map__init_kern_struct_ops()
1112 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1118 prog = st_ops->progs[i]; in bpf_map__init_kern_struct_ops()
1123 kern_mtype->type, in bpf_map__init_kern_struct_ops()
1126 /* mtype->type must be a func_proto which was in bpf_map__init_kern_struct_ops()
1132 map->name, mname); in bpf_map__init_kern_struct_ops()
1133 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1136 prog->attach_btf_id = kern_type_id; in bpf_map__init_kern_struct_ops()
1137 prog->expected_attach_type = kern_member_idx; in bpf_map__init_kern_struct_ops()
1139 st_ops->kern_func_off[i] = kern_data_off + kern_moff; in bpf_map__init_kern_struct_ops()
1142 map->name, mname, prog->name, moff, in bpf_map__init_kern_struct_ops()
1152 map->name, mname, (ssize_t)msize, in bpf_map__init_kern_struct_ops()
1154 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1158 map->name, mname, (unsigned int)msize, in bpf_map__init_kern_struct_ops()
1172 for (i = 0; i < obj->nr_maps; i++) { in bpf_object__init_kern_struct_ops_maps()
1173 map = &obj->maps[i]; in bpf_object__init_kern_struct_ops_maps()
1178 err = bpf_map__init_kern_struct_ops(map, obj->btf, in bpf_object__init_kern_struct_ops_maps()
1179 obj->btf_vmlinux); in bpf_object__init_kern_struct_ops_maps()
1199 if (shndx == -1) in init_struct_ops_maps()
1202 btf = obj->btf; in init_struct_ops_maps()
1208 return -EINVAL; in init_struct_ops_maps()
1214 type = btf__type_by_id(obj->btf, vsi->type); in init_struct_ops_maps()
1215 var_name = btf__name_by_offset(obj->btf, type->name_off); in init_struct_ops_maps()
1217 type_id = btf__resolve_type(obj->btf, vsi->type); in init_struct_ops_maps()
1220 vsi->type, sec_name); in init_struct_ops_maps()
1221 return -EINVAL; in init_struct_ops_maps()
1224 type = btf__type_by_id(obj->btf, type_id); in init_struct_ops_maps()
1225 tname = btf__name_by_offset(obj->btf, type->name_off); in init_struct_ops_maps()
1228 return -ENOTSUP; in init_struct_ops_maps()
1232 return -EINVAL; in init_struct_ops_maps()
1239 map->sec_idx = shndx; in init_struct_ops_maps()
1240 map->sec_offset = vsi->offset; in init_struct_ops_maps()
1241 map->name = strdup(var_name); in init_struct_ops_maps()
1242 if (!map->name) in init_struct_ops_maps()
1243 return -ENOMEM; in init_struct_ops_maps()
1245 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; in init_struct_ops_maps()
1246 map->def.key_size = sizeof(int); in init_struct_ops_maps()
1247 map->def.value_size = type->size; in init_struct_ops_maps()
1248 map->def.max_entries = 1; in init_struct_ops_maps()
1249 map->def.map_flags = map_flags; in init_struct_ops_maps()
1251 map->st_ops = calloc(1, sizeof(*map->st_ops)); in init_struct_ops_maps()
1252 if (!map->st_ops) in init_struct_ops_maps()
1253 return -ENOMEM; in init_struct_ops_maps()
1254 st_ops = map->st_ops; in init_struct_ops_maps()
1255 st_ops->data = malloc(type->size); in init_struct_ops_maps()
1256 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); in init_struct_ops_maps()
1257 st_ops->kern_func_off = malloc(btf_vlen(type) * in init_struct_ops_maps()
1258 sizeof(*st_ops->kern_func_off)); in init_struct_ops_maps()
1259 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) in init_struct_ops_maps()
1260 return -ENOMEM; in init_struct_ops_maps()
1262 if (vsi->offset + type->size > data->d_size) { in init_struct_ops_maps()
1265 return -EINVAL; in init_struct_ops_maps()
1268 memcpy(st_ops->data, in init_struct_ops_maps()
1269 data->d_buf + vsi->offset, in init_struct_ops_maps()
1270 type->size); in init_struct_ops_maps()
1271 st_ops->tname = tname; in init_struct_ops_maps()
1272 st_ops->type = type; in init_struct_ops_maps()
1273 st_ops->type_id = type_id; in init_struct_ops_maps()
1276 tname, type_id, var_name, vsi->offset); in init_struct_ops_maps()
1286 err = init_struct_ops_maps(obj, STRUCT_OPS_SEC, obj->efile.st_ops_shndx, in bpf_object_init_struct_ops()
1287 obj->efile.st_ops_data, 0); in bpf_object_init_struct_ops()
1289 obj->efile.st_ops_link_shndx, in bpf_object_init_struct_ops()
1290 obj->efile.st_ops_link_data, in bpf_object_init_struct_ops()
1306 return ERR_PTR(-ENOMEM); in bpf_object__new()
1309 strcpy(obj->path, path); in bpf_object__new()
1311 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); in bpf_object__new()
1314 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); in bpf_object__new()
1315 end = strchr(obj->name, '.'); in bpf_object__new()
1320 obj->efile.fd = -1; in bpf_object__new()
1327 obj->efile.obj_buf = obj_buf; in bpf_object__new()
1328 obj->efile.obj_buf_sz = obj_buf_sz; in bpf_object__new()
1329 obj->efile.btf_maps_shndx = -1; in bpf_object__new()
1330 obj->efile.st_ops_shndx = -1; in bpf_object__new()
1331 obj->efile.st_ops_link_shndx = -1; in bpf_object__new()
1332 obj->kconfig_map_idx = -1; in bpf_object__new()
1334 obj->kern_version = get_kernel_version(); in bpf_object__new()
1335 obj->loaded = false; in bpf_object__new()
1342 if (!obj->efile.elf) in bpf_object__elf_finish()
1345 elf_end(obj->efile.elf); in bpf_object__elf_finish()
1347 if (obj->efile.shstring) { in bpf_object__elf_finish()
1348 elfio_string_section_accessor_delete(obj->efile.shstring); in bpf_object__elf_finish()
1350 if (obj->efile.strstring) { in bpf_object__elf_finish()
1351 elfio_string_section_accessor_delete(obj->efile.strstring); in bpf_object__elf_finish()
1353 elfio_delete(obj->efile.elf); in bpf_object__elf_finish()
1355 obj->efile.elf = NULL; in bpf_object__elf_finish()
1356 obj->efile.symbols = NULL; in bpf_object__elf_finish()
1357 obj->efile.st_ops_data = NULL; in bpf_object__elf_finish()
1358 obj->efile.st_ops_link_data = NULL; in bpf_object__elf_finish()
1360 zfree(&obj->efile.secs); in bpf_object__elf_finish()
1361 obj->efile.sec_cnt = 0; in bpf_object__elf_finish()
1362 zclose(obj->efile.fd); in bpf_object__elf_finish()
1363 obj->efile.obj_buf = NULL; in bpf_object__elf_finish()
1364 obj->efile.obj_buf_sz = 0; in bpf_object__elf_finish()
1377 if (obj->efile.elf) {
1379 return -LIBBPF_ERRNO__LIBELF;
1382 if (obj->efile.obj_buf_sz > 0) {
1385 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1390 ftruncate(fdm, obj->efile.obj_buf_sz);
1391 write(fdm, (char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1396 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1397 if (obj->efile.fd < 0) {
1400 err = -errno;
1402 pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1406 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1411 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1412 err = -LIBBPF_ERRNO__LIBELF;
1416 obj->efile.elf = elf;
1419 err = -LIBBPF_ERRNO__FORMAT;
1420 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1428 err = -LIBBPF_ERRNO__FORMAT;
1429 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1433 obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1435 obj->efile.ehdr = ehdr = (Elf64_Ehdr*)obj->efile.obj_buf;
1437 if (!obj->efile.ehdr) {
1438 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1439 err = -LIBBPF_ERRNO__FORMAT;
1444 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1446 obj->path, elf_errmsg(-1));
1447 err = -LIBBPF_ERRNO__FORMAT;
1452 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1454 obj->path, elf_errmsg(-1));
1455 err = -LIBBPF_ERRNO__FORMAT;
1459 obj->efile.shstrndx = elfio_get_section_name_str_index(elf);
1462 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1463 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1464 err = -LIBBPF_ERRNO__FORMAT;
1477 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1480 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1485 pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1486 return -LIBBPF_ERRNO__ENDIAN;
1493 pr_warn("invalid license section in %s\n", obj->path);
1494 return -LIBBPF_ERRNO__FORMAT;
1496 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1499 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1500 pr_debug("license of %s is %s\n", obj->path, obj->license);
1510 pr_warn("invalid kver section in %s\n", obj->path);
1511 return -LIBBPF_ERRNO__FORMAT;
1514 obj->kern_version = kver;
1515 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1535 return -EINVAL;
1545 *size = data->d_size;
1549 return -ENOENT;
1554 Elf_Data *symbols = obj->efile.symbols;
1558 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1561 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1564 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1565 ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1568 sname = elf_sym_str(obj, sym->st_name);
1571 return ERR_PTR(-EIO);
1577 return ERR_PTR(-ENOENT);
1585 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1586 sizeof(*obj->maps), obj->nr_maps + 1);
1590 map = &obj->maps[obj->nr_maps++];
1591 map->obj = obj;
1592 map->fd = -1;
1593 map->inner_map_fd = -1;
1594 map->autocreate = true;
1613 if (!map->mmaped)
1614 return -EINVAL;
1619 mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1621 return -errno;
1623 memcpy(mmaped, map->mmaped, min(old_sz, new_sz));
1624 munmap(map->mmaped, old_sz);
1625 map->mmaped = mmaped;
1659 * '.rodata.abracad' kernel and user-visible name.
1668 sfx_len = BPF_OBJ_NAME_LEN - 1;
1674 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1676 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1701 if (!map->btf_value_type_id)
1704 t = btf__type_by_id(obj->btf, map->btf_value_type_id);
1710 vt = btf__type_by_id(obj->btf, vsi->type);
1714 if (btf_var(vt)->linkage != BTF_VAR_STATIC)
1734 map->libbpf_type = type;
1735 map->sec_idx = sec_idx;
1736 map->sec_offset = 0;
1737 map->real_name = strdup(real_name);
1738 map->name = internal_map_name(obj, real_name);
1739 if (!map->real_name || !map->name) {
1740 zfree(&map->real_name);
1741 zfree(&map->name);
1742 return -ENOMEM;
1745 def = &map->def;
1746 def->type = BPF_MAP_TYPE_ARRAY;
1747 def->key_size = sizeof(int);
1748 def->value_size = data_sz;
1749 def->max_entries = 1;
1750 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1757 def->map_flags |= BPF_F_MMAPABLE;
1760 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1762 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
1763 map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1764 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1765 if (map->mmaped == MAP_FAILED) {
1766 err = -errno;
1767 map->mmaped = NULL;
1769 map->name, err);
1770 zfree(&map->real_name);
1771 zfree(&map->name);
1776 memcpy(map->mmaped, data, data_sz);
1778 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1789 * Populate obj->maps with libbpf internal maps.
1791 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
1792 sec_desc = &obj->efile.secs[sec_idx];
1795 if (!sec_desc->data || sec_desc->data->d_size == 0)
1798 switch (sec_desc->sec_type) {
1807 sec_desc->data->d_buf,
1808 sec_desc->data->d_size);
1811 obj->has_rodata = true;
1819 sec_desc->data->d_buf,
1820 sec_desc->data->d_size);
1831 sec_desc->data->d_size);
1849 for (i = 0; i < obj->nr_extern; i++) {
1850 if (strcmp(obj->externs[i].name, name) == 0)
1851 return &obj->externs[i];
1859 switch (ext->kcfg.type) {
1863 ext->name, value);
1864 return -EINVAL;
1884 ext->name, value);
1885 return -EINVAL;
1887 ext->is_set = true;
1896 if (ext->kcfg.type != KCFG_CHAR_ARR) {
1898 ext->name, value);
1899 return -EINVAL;
1903 if (len < 2 || value[len - 1] != '"') {
1905 ext->name, value);
1906 return -EINVAL;
1910 len -= 2;
1911 if (len >= ext->kcfg.sz) {
1913 ext->name, value, len, ext->kcfg.sz - 1);
1914 len = ext->kcfg.sz - 1;
1918 ext->is_set = true;
1930 err = -errno;
1936 return -EINVAL;
1943 int bit_sz = ext->kcfg.sz * 8;
1945 if (ext->kcfg.sz == 8)
1948 /* Validate that value stored in u64 fits in integer of `ext->sz`
1953 * -2^(Y-1) <= X <= 2^(Y-1) - 1
1954 * 0 <= X + 2^(Y-1) <= 2^Y - 1
1955 * 0 <= X + 2^(Y-1) < 2^Y
1957 * For unsigned target integer, check that all the (64 - Y) bits are
1960 if (ext->kcfg.is_signed)
1961 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1969 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
1970 ext->kcfg.type != KCFG_BOOL) {
1972 ext->name, (unsigned long long)value);
1973 return -EINVAL;
1975 if (ext->kcfg.type == KCFG_BOOL && value > 1) {
1977 ext->name, (unsigned long long)value);
1978 return -EINVAL;
1983 ext->name, (unsigned long long)value, ext->kcfg.sz);
1984 return -ERANGE;
1986 switch (ext->kcfg.sz) {
2000 return -EINVAL;
2002 ext->is_set = true;
2021 return -EINVAL;
2026 if (buf[len - 1] == '\n')
2027 buf[len - 1] = '\0';
2033 return -EINVAL;
2037 if (!ext || ext->is_set)
2040 ext_val = data + ext->kcfg.data_off;
2054 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
2057 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
2058 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
2059 return -EINVAL;
2066 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
2078 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
2080 return -EINVAL;
2082 return -ENAMETOOLONG;
2091 return -ENOENT;
2117 err = -errno;
2118 pr_warn("failed to open in-memory Kconfig: %d\n", err);
2125 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
2141 for (i = 0; i < obj->nr_extern; i++) {
2142 ext = &obj->externs[i];
2143 if (ext->type == EXT_KCFG)
2150 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
2152 ".kconfig", obj->efile.symbols_shndx,
2157 obj->kconfig_map_idx = obj->nr_maps - 1;
2172 *res_id = t->type;
2173 t = btf__type_by_id(btf, t->type);
2188 t = skip_mods_and_typedefs(btf, t->type, res_id);
2235 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2236 const char *name = btf__name_by_offset(btf, m->name_off);
2246 arr_t = btf__type_by_id(btf, t->type);
2249 map_name, name, t->type);
2258 *res = arr_info->nelems;
2268 return -EINVAL;
2270 return -ENAMETOOLONG;
2309 const char *name = btf__name_by_offset(btf, m->name_off);
2313 return -EINVAL;
2316 if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2317 return -EINVAL;
2318 map_def->parts |= MAP_DEF_MAP_TYPE;
2320 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2321 return -EINVAL;
2322 map_def->parts |= MAP_DEF_MAX_ENTRIES;
2324 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2325 return -EINVAL;
2326 map_def->parts |= MAP_DEF_MAP_FLAGS;
2328 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2329 return -EINVAL;
2330 map_def->parts |= MAP_DEF_NUMA_NODE;
2335 return -EINVAL;
2336 if (map_def->key_size && map_def->key_size != sz) {
2338 map_name, map_def->key_size, sz);
2339 return -EINVAL;
2341 map_def->key_size = sz;
2342 map_def->parts |= MAP_DEF_KEY_SIZE;
2346 t = btf__type_by_id(btf, m->type);
2349 map_name, m->type);
2350 return -EINVAL;
2355 return -EINVAL;
2357 sz = btf__resolve_size(btf, t->type);
2360 map_name, t->type, (ssize_t)sz);
2363 if (map_def->key_size && map_def->key_size != sz) {
2365 map_name, map_def->key_size, (ssize_t)sz);
2366 return -EINVAL;
2368 map_def->key_size = sz;
2369 map_def->key_type_id = t->type;
2370 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2375 return -EINVAL;
2376 if (map_def->value_size && map_def->value_size != sz) {
2378 map_name, map_def->value_size, sz);
2379 return -EINVAL;
2381 map_def->value_size = sz;
2382 map_def->parts |= MAP_DEF_VALUE_SIZE;
2386 t = btf__type_by_id(btf, m->type);
2389 map_name, m->type);
2390 return -EINVAL;
2395 return -EINVAL;
2397 sz = btf__resolve_size(btf, t->type);
2400 map_name, t->type, (ssize_t)sz);
2403 if (map_def->value_size && map_def->value_size != sz) {
2405 map_name, map_def->value_size, (ssize_t)sz);
2406 return -EINVAL;
2408 map_def->value_size = sz;
2409 map_def->value_type_id = t->type;
2410 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2413 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2414 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2415 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2420 pr_warn("map '%s': multi-level inner maps not supported.\n",
2422 return -ENOTSUP;
2424 if (i != vlen - 1) {
2427 return -EINVAL;
2430 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2432 return -ENOTSUP;
2434 if (map_def->value_size && map_def->value_size != 4) {
2436 map_name, map_def->value_size);
2437 return -EINVAL;
2439 map_def->value_size = 4;
2440 t = btf__type_by_id(btf, m->type);
2443 map_name, desc, m->type);
2444 return -EINVAL;
2446 if (!btf_is_array(t) || btf_array(t)->nelems) {
2447 pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2449 return -EINVAL;
2451 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2455 return -EINVAL;
2457 t = skip_mods_and_typedefs(btf, t->type, NULL);
2460 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2462 return -EINVAL;
2467 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2469 return -EINVAL;
2477 map_def->parts |= MAP_DEF_INNER_MAP;
2483 return -EINVAL;
2486 return -EINVAL;
2490 return -EINVAL;
2492 map_def->pinning = val;
2493 map_def->parts |= MAP_DEF_PINNING;
2498 return -EINVAL;
2499 map_def->map_extra = map_extra;
2500 map_def->parts |= MAP_DEF_MAP_EXTRA;
2504 return -ENOTSUP;
2510 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2512 return -EINVAL;
2527 * a power-of-2 multiple of kernel's page size. If user diligently
2534 * user-set size to satisfy both user size request and kernel
2543 * very close to UINT_MAX but is not a power-of-2 multiple of
2551 return map->def.type == BPF_MAP_TYPE_RINGBUF ||
2552 map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2557 map->def.type = def->map_type;
2558 map->def.key_size = def->key_size;
2559 map->def.value_size = def->value_size;
2560 map->def.max_entries = def->max_entries;
2561 map->def.map_flags = def->map_flags;
2562 map->map_extra = def->map_extra;
2564 map->numa_node = def->numa_node;
2565 map->btf_key_type_id = def->key_type_id;
2566 map->btf_value_type_id = def->value_type_id;
2568 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2570 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2572 if (def->parts & MAP_DEF_MAP_TYPE)
2573 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2575 if (def->parts & MAP_DEF_KEY_TYPE)
2577 map->name, def->key_type_id, def->key_size);
2578 else if (def->parts & MAP_DEF_KEY_SIZE)
2579 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2581 if (def->parts & MAP_DEF_VALUE_TYPE)
2583 map->name, def->value_type_id, def->value_size);
2584 else if (def->parts & MAP_DEF_VALUE_SIZE)
2585 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2587 if (def->parts & MAP_DEF_MAX_ENTRIES)
2588 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2589 if (def->parts & MAP_DEF_MAP_FLAGS)
2590 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2591 if (def->parts & MAP_DEF_MAP_EXTRA)
2592 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2593 (unsigned long long)def->map_extra);
2594 if (def->parts & MAP_DEF_PINNING)
2595 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2596 if (def->parts & MAP_DEF_NUMA_NODE)
2597 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2599 if (def->parts & MAP_DEF_INNER_MAP)
2600 pr_debug("map '%s': found inner map definition.\n", map->name);
2628 var = btf__type_by_id(obj->btf, vi->type);
2630 map_name = btf__name_by_offset(obj->btf, var->name_off);
2634 return -EINVAL;
2636 if ((__u64)vi->offset + vi->size > data->d_size) {
2638 return -EINVAL;
2643 return -EINVAL;
2645 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2647 map_name, btf_var_linkage_str(var_extra->linkage));
2648 return -EOPNOTSUPP;
2651 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2655 return -EINVAL;
2657 if (def->size > vi->size) {
2659 return -EINVAL;
2665 map->name = strdup(map_name);
2666 if (!map->name) {
2668 return -ENOMEM;
2670 map->libbpf_type = LIBBPF_MAP_UNSPEC;
2671 map->def.type = BPF_MAP_TYPE_UNSPEC;
2672 map->sec_idx = sec_idx;
2673 map->sec_offset = vi->offset;
2674 map->btf_var_idx = var_idx;
2676 map_name, map->sec_idx, map->sec_offset);
2678 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2687 pr_warn("map '%s': couldn't build pin path.\n", map->name);
2693 map->inner_map = calloc(1, sizeof(*map->inner_map));
2694 if (!map->inner_map)
2695 return -ENOMEM;
2696 map->inner_map->fd = -1;
2697 map->inner_map->sec_idx = sec_idx;
2698 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2699 if (!map->inner_map->name)
2700 return -ENOMEM;
2701 sprintf(map->inner_map->name, "%s.inner", map_name);
2703 fill_map_from_def(map->inner_map, &inner_def);
2725 if (obj->efile.btf_maps_shndx < 0)
2728 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2733 data = elf_sec_data_by_idx(obj, obj->efile.btf_maps_shndx, &realdata);
2737 MAPS_ELF_SEC, obj->path);
2738 return -EINVAL;
2741 nr_types = btf__type_cnt(obj->btf);
2743 t = btf__type_by_id(obj->btf, i);
2746 name = btf__name_by_offset(obj->btf, t->name_off);
2749 obj->efile.btf_maps_sec_btf_id = i;
2756 return -ENOENT;
2762 obj->efile.btf_maps_shndx,
2802 return sh->sh_flags & SHF_EXECINSTR;
2837 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2843 t->size = 1;
2852 name = (char *)btf__name_by_offset(btf, t->name_off);
2860 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2863 m->offset = v->offset * 8;
2864 m->type = v->type;
2866 vt = (void *)btf__type_by_id(btf, v->type);
2867 m->name_off = vt->name_off;
2872 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2873 t->size = sizeof(__u32); /* kernel enforced */
2876 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2879 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2881 /* replace FLOAT with an equally-sized empty STRUCT;
2885 t->name_off = 0;
2886 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
2889 t->name_off = 0;
2890 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
2893 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
2908 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
2910 m->type = enum64_placeholder_id;
2911 m->offset = 0;
2921 return obj->efile.btf_maps_shndx >= 0 ||
2922 obj->efile.st_ops_shndx >= 0 ||
2923 obj->efile.st_ops_link_shndx >= 0 ||
2924 obj->nr_extern > 0;
2929 return obj->efile.st_ops_shndx >= 0 || obj->efile.st_ops_link_shndx >= 0;
2936 int err = -ENOENT;
2939 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2940 err = libbpf_get_error(obj->btf);
2942 obj->btf = NULL;
2946 /* enforce 8-byte pointers for BPF-targeted BTFs */
2947 btf__set_pointer_size(obj->btf, 8);
2953 if (!obj->btf) {
2958 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
2959 err = libbpf_get_error(obj->btf_ext);
2963 obj->btf_ext = NULL;
2968 ext_segs[0] = &obj->btf_ext->func_info;
2969 ext_segs[1] = &obj->btf_ext->line_info;
2970 ext_segs[2] = &obj->btf_ext->core_relo_info;
2981 if (seg->sec_cnt == 0)
2984 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
2985 if (!seg->sec_idxs) {
2986 err = -ENOMEM;
2997 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
3005 pelfio_t elf = obj->efile.elf;
3011 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
3013 seg->sec_idxs[sec_num - 1] = elfio_section_get_index(sec_obj);
3031 return a->offset - b->offset;
3038 const char *sec_name = btf__name_by_offset(btf, t->name_off);
3045 return -ENOENT;
3048 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and
3060 * to be optional. But the STV_HIDDEN handling is non-optional for any
3061 * non-extern DATASEC, so the variable fixup loop below handles both
3062 * functions at the same time, paying the cost of BTF VAR <-> ELF
3065 if (t->size == 0) {
3070 return -ENOENT;
3073 t->size = size;
3083 t_var = btf__type_by_id(btf, vsi->type);
3085 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);
3086 return -EINVAL;
3090 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)
3093 var_name = btf__name_by_offset(btf, t_var->name_off);
3097 return -ENOENT;
3104 return -ENOENT;
3108 vsi->offset = sym->st_value;
3117 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
3118 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)
3119 var->linkage = BTF_VAR_STATIC;
3131 if (!obj->btf)
3134 n = btf__type_cnt(obj->btf);
3136 struct btf_type *t = btf_type_by_id(obj->btf, i);
3144 err = btf_fixup_datasec(obj, obj->btf, t);
3155 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3156 prog->type == BPF_PROG_TYPE_LSM)
3162 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3173 /* CO-RE relocations need kernel BTF, only when btf_custom_path
3176 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
3180 for (i = 0; i < obj->nr_extern; i++) {
3183 ext = &obj->externs[i];
3184 if (ext->type == EXT_KSYM && ext->ksym.type_id)
3189 if (!prog->autoload)
3203 if (obj->btf_vmlinux || obj->gen_loader)
3209 obj->btf_vmlinux = btf__load_vmlinux_btf();
3210 err = libbpf_get_error(obj->btf_vmlinux);
3213 obj->btf_vmlinux = NULL;
3221 struct btf *kern_btf = obj->btf;
3225 if (!obj->btf)
3230 err = -EOPNOTSUPP;
3245 for (i = 0; i < obj->nr_programs; i++) {
3246 struct bpf_program *prog = &obj->programs[i];
3251 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3254 n = btf__type_cnt(obj->btf);
3256 t = btf_type_by_id(obj->btf, j);
3260 name = btf__str_by_offset(obj->btf, t->name_off);
3261 if (strcmp(name, prog->name) != 0)
3264 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3271 for (i = 0; i < obj->nr_programs; i++) {
3272 struct bpf_program *prog = &obj->programs[i];
3277 n = btf__type_cnt(obj->btf);
3283 t = btf_type_by_id(obj->btf, j);
3284 if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1)
3287 name = btf__str_by_offset(obj->btf, t->name_off);
3291 t = btf_type_by_id(obj->btf, t->type);
3294 prog->name);
3295 return -EINVAL;
3297 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)))
3303 if (prog->exception_cb_idx >= 0) {
3304 prog->exception_cb_idx = -1;
3311 prog->name);
3312 return -EINVAL;
3315 for (k = 0; k < obj->nr_programs; k++) {
3316 struct bpf_program *subprog = &obj->programs[k];
3320 if (strcmp(name, subprog->name))
3322 /* Enforce non-hidden, as from verifier point of
3326 if (!subprog->sym_global || subprog->mark_btf_static) {
3327 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n",
3328 prog->name, subprog->name);
3329 return -EINVAL;
3332 if (prog->exception_cb_idx >= 0) {
3334 prog->name, subprog->name);
3335 return -EINVAL;
3337 prog->exception_cb_idx = k;
3341 if (prog->exception_cb_idx >= 0)
3343 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name);
3344 return -ENOENT;
3355 raw_data = btf__raw_data(obj->btf, &sz);
3361 /* enforce 8-byte pointers for BPF-targeted BTFs */
3362 btf__set_pointer_size(obj->btf, 8);
3368 if (obj->gen_loader) {
3373 return -ENOMEM;
3374 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3376 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3381 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3382 obj->log_level ? 1 : 0);
3387 btf__set_fd(obj->btf, btf__fd(kern_btf));
3388 btf__set_fd(kern_btf, -1);
3408 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3410 name = elfio_string_get_string(obj->efile.strstring, off);
3414 off, obj->path, elf_errmsg(-1));
3425 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3427 name = elfio_string_get_string(obj->efile.shstring, off);
3432 off, obj->path, elf_errmsg(-1));
3444 scn = elf_getscn(obj->efile.elf, idx);
3447 idx, obj->path, elf_errmsg(-1));
3456 Elf *elf = obj->efile.elf;
3482 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3501 name = elf_sec_str(obj, sh->sh_name);
3504 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3513 psection_t psection = elfio_get_section_by_index(obj->efile.elf, idx);
3515 sheader->sh_name = elfio_section_get_name_string_offset(psection);
3516 sheader->sh_type = elfio_section_get_type(psection);
3517 sheader->sh_flags = elfio_section_get_flags(psection);
3518 sheader->sh_addr = elfio_section_get_address(psection);
3519 sheader->sh_offset = elfio_section_get_offset(psection);
3520 sheader->sh_size = elfio_section_get_size(psection);
3521 sheader->sh_link = elfio_section_get_link(psection);
3522 sheader->sh_info = elfio_section_get_info(psection);
3523 sheader->sh_addralign = elfio_section_get_addr_align(psection);
3524 sheader->sh_entsize = elfio_section_get_entry_size(psection);
3539 idx, obj->path, elf_errmsg(-1));
3559 obj->path, elf_errmsg(-1));
3568 pelfio_t elf = obj->efile.elf;
3570 data->d_buf = (void*)elfio_section_get_data(psection_name);
3571 data->d_size = elfio_section_get_size(psection_name);
3578 pelfio_t elf = obj->efile.elf;
3580 data->d_buf = (void*)elfio_section_get_data(psection_index);
3581 data->d_size = elfio_section_get_size(psection_index);
3589 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3592 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3597 if (idx >= data->d_size / sizeof(Elf64_Rel))
3600 return (Elf64_Rel *)data->d_buf + idx;
3612 if (hdr->sh_type == SHT_STRTAB)
3616 if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3620 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3629 name += sizeof(".rel") - 1;
3648 if (a->sec_idx != b->sec_idx)
3649 return a->sec_idx < b->sec_idx ? -1 : 1;
3652 return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3659 Elf *elf = obj->efile.elf;
3661 pelfio_t elf = obj->efile.elf;
3677 /* ELF section indices are 0-based, but sec #0 is special "invalid"
3683 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3685 obj->path, elf_errmsg(-1));
3686 return -LIBBPF_ERRNO__FORMAT;
3689 obj->efile.sec_cnt = elfio_get_sections_num(elf);
3691 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3692 if (!obj->efile.secs)
3693 return -ENOMEM;
3709 return -LIBBPF_ERRNO__FORMAT;
3711 if (sh->sh_type == SHT_SYMTAB) {
3712 if (obj->efile.symbols) {
3713 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3714 return -LIBBPF_ERRNO__FORMAT;
3722 return -LIBBPF_ERRNO__FORMAT;
3728 obj->efile.symbols = data;
3730 obj->efile.realsymbols.d_buf = data->d_buf;
3731 obj->efile.realsymbols.d_size = data->d_size;
3732 obj->efile.symbols = &(obj->efile.realsymbols);
3736 obj->efile.symbols_shndx = idx;
3738 obj->efile.symbols_shndx = i;
3740 obj->efile.strtabidx = sh->sh_link;
3748 psection_t psection = elfio_get_section_by_index(elf, obj->efile.strtabidx);
3750 return -LIBBPF_ERRNO__FORMAT;
3753 psection = elfio_get_section_by_index(elf, obj->efile.shstrndx);
3755 return -LIBBPF_ERRNO__FORMAT;
3759 return -LIBBPF_ERRNO__FORMAT;
3760 obj->efile.strstring = strstring;
3761 obj->efile.shstring = shstring;
3764 if (!obj->efile.symbols) {
3766 obj->path);
3767 return -ENOENT;
3784 sec_desc = &obj->efile.secs[idx];
3793 return -LIBBPF_ERRNO__FORMAT;
3795 name = elf_sec_str(obj, sh->sh_name);
3797 return -LIBBPF_ERRNO__FORMAT;
3805 data = elf_sec_data_by_idx(obj, i, &sec_desc->realdata);
3808 return -LIBBPF_ERRNO__FORMAT;
3811 idx, name, (unsigned long)data->d_size,
3812 (int)sh->sh_link, (unsigned long)sh->sh_flags,
3813 (int)sh->sh_type);
3816 err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3820 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3825 return -ENOTSUP;
3827 obj->efile.btf_maps_shndx = idx;
3829 if (sh->sh_type != SHT_PROGBITS)
3830 return -LIBBPF_ERRNO__FORMAT;
3833 if (sh->sh_type != SHT_PROGBITS)
3834 return -LIBBPF_ERRNO__FORMAT;
3836 } else if (sh->sh_type == SHT_SYMTAB) {
3838 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
3839 if (sh->sh_flags & SHF_EXECINSTR) {
3841 obj->efile.text_shndx = idx;
3847 sec_desc->sec_type = SEC_DATA;
3849 sec_desc->shdr = sh;
3850 sec_desc->data = data;
3852 sec_desc->psection = ptmpsection;
3853 sec_desc->realdata.d_buf = data->d_buf;
3854 sec_desc->realdata.d_size = data->d_size;
3855 sec_desc->data = &(sec_desc->realdata);
3859 sec_desc->sec_type = SEC_RODATA;
3861 sec_desc->shdr = sh;
3862 sec_desc->data = data;
3864 sec_desc->psection = ptmpsection;
3865 sec_desc->realdata.d_buf = data->d_buf;
3866 sec_desc->realdata.d_size = data->d_size;
3867 sec_desc->data = &(sec_desc->realdata);
3872 obj->efile.st_ops_data = data;
3874 obj->efile.realst_ops_data.d_buf = data->d_buf;
3875 obj->efile.realst_ops_data.d_size = data->d_size;
3876 obj->efile.st_ops_data = &(obj->efile.realst_ops_data);
3878 obj->efile.st_ops_shndx = idx;
3881 obj->efile.st_ops_link_data = data;
3883 obj->efile.realst_ops_link_data.d_buf = data->d_buf;
3884 obj->efile.realst_ops_link_data.d_size = data->d_size;
3885 obj->efile.st_ops_link_data = &(obj->efile.realst_ops_link_data);
3887 obj->efile.st_ops_link_shndx = idx;
3892 } else if (sh->sh_type == SHT_REL) {
3893 int targ_sec_idx = sh->sh_info; /* points to other section */
3895 if (sh->sh_entsize != sizeof(Elf64_Rel) ||
3896 targ_sec_idx >= obj->efile.sec_cnt)
3897 return -LIBBPF_ERRNO__FORMAT;
3916 sec_desc->sec_type = SEC_RELO;
3918 sec_desc->shdr = sh;
3920 sec_desc->psection = ptmpsection;
3922 sec_desc->data = data;
3923 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||
3925 sec_desc->sec_type = SEC_BSS;
3927 sec_desc->shdr = sh;
3929 sec_desc->psection = ptmpsection;
3931 sec_desc->data = data;
3934 (size_t)sh->sh_size);
3938 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3939 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3940 return -LIBBPF_ERRNO__FORMAT;
3943 /* sort BPF programs by section name and in-section instruction offset
3946 if (obj->nr_programs)
3947 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3954 int bind = ELF64_ST_BIND(sym->st_info);
3956 return sym->st_shndx == SHN_UNDEF &&
3958 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
3963 int bind = ELF64_ST_BIND(sym->st_info);
3964 int type = ELF64_ST_TYPE(sym->st_info);
3967 if (sym->st_shndx != text_shndx)
3985 return -ESRCH;
3994 tname = btf__name_by_offset(btf, t->name_off);
3999 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
4000 return -EINVAL;
4003 return -EINVAL;
4008 return -ENOENT;
4017 return -ESRCH;
4028 if (vs->type == ext_btf_id)
4033 return -ENOENT;
4043 name = btf__name_by_offset(btf, t->name_off);
4052 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
4055 if (t->size == 1)
4057 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
4062 if (t->size != 4)
4072 if (btf_array(t)->nelems == 0)
4074 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
4087 if (a->type != b->type)
4088 return a->type < b->type ? -1 : 1;
4090 if (a->type == EXT_KCFG) {
4092 if (a->kcfg.align != b->kcfg.align)
4093 return a->kcfg.align > b->kcfg.align ? -1 : 1;
4095 if (a->kcfg.sz != b->kcfg.sz)
4096 return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
4100 return strcmp(a->name, b->name);
4138 vt = btf__type_by_id(btf, vs->type);
4172 if (!obj->efile.symbols)
4176 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
4180 sh = elf_sec_hdr_by_idx(obj, obj->efile.symbols_shndx, sh);
4183 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
4184 return -LIBBPF_ERRNO__FORMAT;
4186 dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
4190 n = sh->sh_size / sh->sh_entsize;
4197 return -LIBBPF_ERRNO__FORMAT;
4200 ext_name = elf_sym_str(obj, sym->st_name);
4204 ext = obj->externs;
4205 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
4207 return -ENOMEM;
4208 obj->externs = ext;
4209 ext = &ext[obj->nr_extern];
4211 obj->nr_extern++;
4213 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
4214 if (ext->btf_id <= 0) {
4216 ext_name, ext->btf_id);
4217 return ext->btf_id;
4219 t = btf__type_by_id(obj->btf, ext->btf_id);
4220 ext->name = btf__name_by_offset(obj->btf, t->name_off);
4221 ext->sym_idx = i;
4222 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
4224 ext_essent_len = bpf_core_essential_name_len(ext->name);
4225 ext->essent_name = NULL;
4226 if (ext_essent_len != strlen(ext->name)) {
4227 ext->essent_name = strndup(ext->name, ext_essent_len);
4228 if (!ext->essent_name)
4229 return -ENOMEM;
4232 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
4233 if (ext->sec_btf_id <= 0) {
4235 ext_name, ext->btf_id, ext->sec_btf_id);
4236 return ext->sec_btf_id;
4238 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
4239 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
4244 ext->name, KCONFIG_SEC);
4245 return -ENOTSUP;
4248 ext->type = EXT_KCFG;
4249 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
4250 if (ext->kcfg.sz <= 0) {
4252 ext_name, ext->kcfg.sz);
4253 return ext->kcfg.sz;
4255 ext->kcfg.align = btf__align_of(obj->btf, t->type);
4256 if (ext->kcfg.align <= 0) {
4258 ext_name, ext->kcfg.align);
4259 return -EINVAL;
4261 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
4262 &ext->kcfg.is_signed);
4263 if (ext->kcfg.type == KCFG_UNKNOWN) {
4265 return -ENOTSUP;
4269 ext->type = EXT_KSYM;
4270 skip_mods_and_typedefs(obj->btf, t->type,
4271 &ext->ksym.type_id);
4274 return -ENOTSUP;
4277 pr_debug("collected %d externs total\n", obj->nr_extern);
4279 if (!obj->nr_extern)
4283 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
4287 * pretending that each extern is a 8-byte variable
4290 /* find existing 4-byte integer type in BTF to use for fake
4293 int int_btf_id = find_int_btf_id(obj->btf);
4295 * will be used to replace the vs->type and
4301 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
4302 for (i = 0; i < obj->nr_extern; i++) {
4303 ext = &obj->externs[i];
4304 if (ext->type != EXT_KSYM)
4307 i, ext->sym_idx, ext->name);
4316 vt = (void *)btf__type_by_id(obj->btf, vs->type);
4317 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
4322 return -ESRCH;
4329 func_proto = btf__type_by_id(obj->btf,
4330 vt->type);
4338 dummy_var->name_off;
4339 vs->type = dummy_var_btf_id;
4340 vt->info &= ~0xffff;
4341 vt->info |= BTF_FUNC_GLOBAL;
4343 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4344 vt->type = int_btf_id;
4346 vs->offset = off;
4347 vs->size = sizeof(int);
4349 sec->size = off;
4356 for (i = 0; i < obj->nr_extern; i++) {
4357 ext = &obj->externs[i];
4358 if (ext->type != EXT_KCFG)
4361 ext->kcfg.data_off = roundup(off, ext->kcfg.align);
4362 off = ext->kcfg.data_off + ext->kcfg.sz;
4364 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
4366 sec->size = off;
4371 t = btf__type_by_id(obj->btf, vs->type);
4372 ext_name = btf__name_by_offset(obj->btf, t->name_off);
4377 return -ESRCH;
4379 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4380 vs->offset = ext->kcfg.data_off;
4388 return prog->sec_idx == obj->efile.text_shndx;
4400 if (!strcmp(prog->name, name))
4409 switch (obj->efile.secs[shndx].sec_type) {
4422 return shndx == obj->efile.btf_maps_shndx;
4428 if (shndx == obj->efile.symbols_shndx)
4431 switch (obj->efile.secs[shndx].sec_type) {
4448 struct bpf_insn *insn = &prog->insns[insn_idx];
4449 size_t map_idx, nr_maps = prog->obj->nr_maps;
4450 struct bpf_object *obj = prog->obj;
4451 __u32 shdr_idx = sym->st_shndx;
4458 prog->name, sym_name, insn_idx, insn->code);
4459 return -LIBBPF_ERRNO__RELOC;
4463 int sym_idx = ELF64_R_SYM(rel->r_info);
4464 int i, n = obj->nr_extern;
4468 ext = &obj->externs[i];
4469 if (ext->sym_idx == sym_idx)
4474 prog->name, sym_name, sym_idx);
4475 return -LIBBPF_ERRNO__RELOC;
4478 prog->name, i, ext->name, ext->sym_idx, insn_idx);
4479 if (insn->code == (BPF_JMP | BPF_CALL))
4480 reloc_desc->type = RELO_EXTERN_CALL;
4482 reloc_desc->type = RELO_EXTERN_LD64;
4483 reloc_desc->insn_idx = insn_idx;
4484 reloc_desc->ext_idx = i;
4488 /* sub-program call relocation */
4490 if (insn->src_reg != BPF_PSEUDO_CALL) {
4491 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4492 return -LIBBPF_ERRNO__RELOC;
4495 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
4502 prog->name, sym_name, sym_sec_name);
4503 return -LIBBPF_ERRNO__RELOC;
4505 if (sym->st_value % BPF_INSN_SZ) {
4507 prog->name, sym_name, (size_t)sym->st_value);
4508 return -LIBBPF_ERRNO__RELOC;
4510 reloc_desc->type = RELO_CALL;
4511 reloc_desc->insn_idx = insn_idx;
4512 reloc_desc->sym_off = sym->st_value;
4518 prog->name, sym_name, shdr_idx);
4519 return -LIBBPF_ERRNO__RELOC;
4523 if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4524 /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4525 * local_func: sym->st_value = 0, insn->imm = offset in the section.
4527 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4529 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4530 return -LIBBPF_ERRNO__RELOC;
4533 reloc_desc->type = RELO_SUBPROG_ADDR;
4534 reloc_desc->insn_idx = insn_idx;
4535 reloc_desc->sym_off = sym->st_value;
4549 prog->name, sym_name, sym_sec_name);
4550 return -LIBBPF_ERRNO__RELOC;
4553 map = &obj->maps[map_idx];
4554 if (map->libbpf_type != type ||
4555 map->sec_idx != sym->st_shndx ||
4556 map->sec_offset != sym->st_value)
4559 prog->name, map_idx, map->name, map->sec_idx,
4560 map->sec_offset, insn_idx);
4565 prog->name, sym_sec_name, (size_t)sym->st_value);
4566 return -LIBBPF_ERRNO__RELOC;
4568 reloc_desc->type = RELO_LD64;
4569 reloc_desc->insn_idx = insn_idx;
4570 reloc_desc->map_idx = map_idx;
4571 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4578 prog->name, sym_sec_name);
4579 return -LIBBPF_ERRNO__RELOC;
4582 map = &obj->maps[map_idx];
4583 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4586 prog->name, map_idx, map->name, map->sec_idx,
4587 map->sec_offset, insn_idx);
4592 prog->name, sym_sec_name);
4593 return -LIBBPF_ERRNO__RELOC;
4596 reloc_desc->type = RELO_DATA;
4597 reloc_desc->insn_idx = insn_idx;
4598 reloc_desc->map_idx = map_idx;
4599 reloc_desc->sym_off = sym->st_value;
4605 return insn_idx >= prog->sec_insn_off &&
4606 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4612 int l = 0, r = obj->nr_programs - 1, m;
4615 if (!obj->nr_programs)
4619 m = l + (r - l + 1) / 2;
4620 prog = &obj->programs[m];
4622 if (prog->sec_idx < sec_idx ||
4623 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4626 r = m - 1;
4631 prog = &obj->programs[l];
4632 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4641 size_t sec_idx = shdr->sh_info, sym_idx;
4654 if (sec_idx >= obj->efile.sec_cnt)
4655 return -EINVAL;
4661 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4664 return -EINVAL;
4669 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4672 return -EINVAL;
4677 nrels = shdr->sh_size / shdr->sh_entsize;
4683 return -LIBBPF_ERRNO__FORMAT;
4686 sym_idx = ELF64_R_SYM(rel->r_info);
4691 return -LIBBPF_ERRNO__FORMAT;
4694 if (sym->st_shndx >= obj->efile.sec_cnt) {
4696 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4697 return -LIBBPF_ERRNO__FORMAT;
4700 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4702 relo_sec_name, (size_t)rel->r_offset, i);
4703 return -LIBBPF_ERRNO__FORMAT;
4706 insn_idx = rel->r_offset / BPF_INSN_SZ;
4713 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4715 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4717 sym_name = elf_sec_name_by_idx(obj, sym->st_shndx);
4720 sym_name = elf_sym_str(obj, sym->st_name);
4733 relos = libbpf_reallocarray(prog->reloc_desc,
4734 prog->nr_reloc + 1, sizeof(*relos));
4736 return -ENOMEM;
4737 prog->reloc_desc = relos;
4740 insn_idx -= prog->sec_insn_off;
4741 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4746 prog->nr_reloc++;
4755 if (!obj->btf)
4756 return -ENOENT;
4758 /* if it's BTF-defined map, we don't need to search for type IDs.
4762 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4770 return -ENOENT;
4772 id = btf__find_by_name(obj->btf, map->real_name);
4776 map->btf_key_type_id = 0;
4777 map->btf_value_type_id = id;
4793 err = -errno;
4801 info->type = val;
4803 info->key_size = val;
4805 info->value_size = val;
4807 info->max_entries = val;
4809 info->map_flags = val;
4819 return map->autocreate;
4824 if (map->obj->loaded)
4825 return libbpf_err(-EBUSY);
4827 map->autocreate = autocreate;
4846 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
4847 new_name = strdup(map->name);
4852 return libbpf_err(-errno);
4861 err = -errno;
4865 err = zclose(map->fd);
4867 err = -errno;
4870 free(map->name);
4872 map->fd = new_fd;
4873 map->name = new_name;
4874 map->def.type = info.type;
4875 map->def.key_size = info.key_size;
4876 map->def.value_size = info.value_size;
4877 map->def.max_entries = info.max_entries;
4878 map->def.map_flags = info.map_flags;
4879 map->btf_key_type_id = info.btf_key_type_id;
4880 map->btf_value_type_id = info.btf_value_type_id;
4881 map->reused = true;
4882 map->map_extra = info.map_extra;
4895 return map->def.max_entries;
4900 if (!bpf_map_type__is_map_in_map(map->def.type))
4903 return map->inner_map;
4908 if (map->obj->loaded)
4909 return libbpf_err(-EBUSY);
4911 map->def.max_entries = max_entries;
4913 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
4915 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
4930 if (obj->gen_loader)
4948 return -ret;
4997 ret = -errno;
5000 __func__, cp, -ret);
5100 BTF_TYPE_DECL_TAG_ENC(1, 2, -1),
5142 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
5154 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */
5177 ret = -errno;
5180 __func__, cp, -ret);
5238 return -errno;
5243 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL);
5244 err = -errno; /* close() can clobber errno */
5250 return link_fd < 0 && err == -EBADF;
5269 return -errno;
5271 /* Creating uprobe in '/' binary should fail with -EBADF. */
5276 link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts);
5277 err = -errno; /* close() can clobber errno */
5279 if (link_fd >= 0 || err != -EBADF) {
5285 /* Initial multi-uprobe support in kernel didn't handle PID filtering
5288 * we'll pretend multi-uprobes are not supported, if not.
5289 * Multi-uprobes are used in USDT attachment logic, and we need to be
5290 * conservative here, because multi-uprobe selection happens early at
5292 * attachment time, at which point it's too late to undo multi-uprobe
5295 * Creating uprobe with pid == -1 for (invalid) '/' binary will fail
5296 * early with -EINVAL on kernels with fixed PID filtering logic;
5297 * otherwise -ESRCH would be returned if passed correct binary path
5298 * (but we'll just get -BADF, of course).
5300 link_opts.uprobe_multi.pid = -1; /* invalid PID */
5305 link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts);
5306 err = -errno; /* close() can clobber errno */
5312 return link_fd < 0 && err == -EINVAL;
5400 "memcg-based memory accounting", probe_memcg_account,
5412 "BPF multi-uprobe link support", probe_uprobe_multi_link,
5421 if (obj && obj->gen_loader)
5427 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
5428 ret = feat->probe();
5430 WRITE_ONCE(feat->res, FEAT_SUPPORTED);
5432 WRITE_ONCE(feat->res, FEAT_MISSING);
5434 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
5435 WRITE_ONCE(feat->res, FEAT_MISSING);
5439 return READ_ONCE(feat->res) == FEAT_SUPPORTED;
5459 return (map_info.type == map->def.type &&
5460 map_info.key_size == map->def.key_size &&
5461 map_info.value_size == map->def.value_size &&
5462 map_info.max_entries == map->def.max_entries &&
5463 map_info.map_flags == map->def.map_flags &&
5464 map_info.map_extra == map->map_extra);
5473 pin_fd = bpf_obj_get(map->pin_path);
5475 err = -errno;
5476 if (err == -ENOENT) {
5478 map->pin_path);
5482 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
5484 map->pin_path, cp);
5490 map->pin_path);
5492 return -EINVAL;
5500 map->pinned = true;
5501 pr_debug("reused pinned map at '%s'\n", map->pin_path);
5509 enum libbpf_map_type map_type = map->libbpf_type;
5514 if (obj->gen_loader) {
5515 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
5516 map->mmaped, map->def.value_size);
5518 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
5521 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
5523 err = -errno;
5530 /* Freeze .rodata and .kconfig map as read-only from syscall side. */
5532 err = bpf_map_freeze(map->fd);
5534 err = -errno;
5536 pr_warn("map '%s': failed to freeze as read-only: %s\n",
5542 /* Remap anonymous mmap()-ed "map initialization image" as
5543 * a BPF map-backed mmap()-ed memory, but preserving the same
5551 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
5552 if (map->def.map_flags & BPF_F_MMAPABLE) {
5556 if (map->def.map_flags & BPF_F_RDONLY_PROG)
5560 mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map->fd, 0);
5562 err = -errno;
5563 pr_warn("map '%s': failed to re-mmap() contents: %d\n",
5567 map->mmaped = mmaped;
5568 } else if (map->mmaped) {
5569 munmap(map->mmaped, mmap_sz);
5570 map->mmaped = NULL;
5581 struct bpf_map_def *def = &map->def;
5586 map_name = map->name;
5587 create_attr.map_ifindex = map->map_ifindex;
5588 create_attr.map_flags = def->map_flags;
5589 create_attr.numa_node = map->numa_node;
5590 create_attr.map_extra = map->map_extra;
5593 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
5595 if (obj->btf && btf__fd(obj->btf) >= 0) {
5596 create_attr.btf_fd = btf__fd(obj->btf);
5597 create_attr.btf_key_type_id = map->btf_key_type_id;
5598 create_attr.btf_value_type_id = map->btf_value_type_id;
5601 if (bpf_map_type__is_map_in_map(def->type)) {
5602 if (map->inner_map) {
5603 err = bpf_object__create_map(obj, map->inner_map, true);
5606 map->name, err);
5609 map->inner_map_fd = bpf_map__fd(map->inner_map);
5611 if (map->inner_map_fd >= 0)
5612 create_attr.inner_map_fd = map->inner_map_fd;
5615 switch (def->type) {
5632 map->btf_key_type_id = 0;
5633 map->btf_value_type_id = 0;
5639 if (obj->gen_loader) {
5640 bpf_gen__map_create(obj->gen_loader, def->type, map_name,
5641 def->key_size, def->value_size, def->max_entries,
5642 &create_attr, is_inner ? -1 : map - obj->maps);
5644 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
5646 map->fd = 0;
5648 map->fd = bpf_map_create(def->type, map_name,
5649 def->key_size, def->value_size,
5650 def->max_entries, &create_attr);
5652 if (map->fd < 0 && (create_attr.btf_key_type_id ||
5656 err = -errno;
5659 map->name, cp, err);
5663 map->btf_key_type_id = 0;
5664 map->btf_value_type_id = 0;
5665 map->fd = bpf_map_create(def->type, map_name,
5666 def->key_size, def->value_size,
5667 def->max_entries, &create_attr);
5670 err = map->fd < 0 ? -errno : 0;
5672 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
5673 if (obj->gen_loader)
5674 map->inner_map->fd = -1;
5675 bpf_map__destroy(map->inner_map);
5676 zfree(&map->inner_map);
5688 for (i = 0; i < map->init_slots_sz; i++) {
5689 if (!map->init_slots[i])
5692 targ_map = map->init_slots[i];
5695 if (obj->gen_loader) {
5696 bpf_gen__populate_outer_map(obj->gen_loader,
5697 map - obj->maps, i,
5698 targ_map - obj->maps);
5700 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5703 err = -errno;
5705 map->name, i, targ_map->name, fd, err);
5709 map->name, i, targ_map->name, fd);
5712 zfree(&map->init_slots);
5713 map->init_slots_sz = 0;
5724 if (obj->gen_loader)
5725 return -ENOTSUP;
5727 for (i = 0; i < map->init_slots_sz; i++) {
5728 if (!map->init_slots[i])
5731 targ_prog = map->init_slots[i];
5734 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5736 err = -errno;
5738 map->name, i, targ_prog->name, fd, err);
5742 map->name, i, targ_prog->name, fd);
5745 zfree(&map->init_slots);
5746 map->init_slots_sz = 0;
5756 for (i = 0; i < obj->nr_maps; i++) {
5757 map = &obj->maps[i];
5759 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
5764 zclose(map->fd);
5773 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
5779 map->name, nr_cpus);
5782 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
5783 map->def.max_entries = nr_cpus;
5798 for (i = 0; i < obj->nr_maps; i++) {
5799 map = &obj->maps[i];
5803 * loading, if we detect that at least one of the to-be-loaded
5808 * but also it allows to have CO-RE applications that use
5810 * If those global variable-using programs are not loaded at
5816 map->autocreate = false;
5818 if (!map->autocreate) {
5819 pr_debug("map '%s': skipped auto-creating...\n", map->name);
5829 if (map->pin_path) {
5833 map->name);
5836 if (retried && map->fd < 0) {
5838 map->name);
5839 err = -ENOENT;
5844 if (map->fd >= 0) {
5846 map->name, map->fd);
5853 map->name, map->fd);
5858 zclose(map->fd);
5863 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
5866 zclose(map->fd);
5872 if (map->pin_path && !map->pinned) {
5875 zclose(map->fd);
5876 if (!retried && err == -EEXIST) {
5880 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
5881 map->name, map->pin_path, err);
5891 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
5894 zclose(obj->maps[j].fd);
5908 * underscore is ignored by BPF CO-RE relocation during relocation matching.
5915 for (i = n - 5; i >= 0; i--) {
5927 free(cands->cands);
5944 local_t = btf__type_by_id(local_cand->btf, local_cand->id);
5945 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
5953 targ_name = btf__name_by_offset(targ_btf, t->name_off);
5964 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
5965 local_cand->id, btf_kind_str(local_t),
5968 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
5969 sizeof(*cands->cands));
5971 return -ENOMEM;
5973 cand = &new_cands[cands->len];
5974 cand->btf = targ_btf;
5975 cand->id = i;
5977 cands->cands = new_cands;
5978 cands->len++;
5992 if (obj->btf_modules_loaded)
5995 if (obj->gen_loader)
5999 obj->btf_modules_loaded = true;
6014 err = -errno;
6023 err = -errno;
6035 err = -errno;
6040 /* ignore non-module BTFs */
6046 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
6054 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
6055 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
6059 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
6061 mod_btf->btf = btf;
6062 mod_btf->id = id;
6063 mod_btf->fd = fd;
6064 mod_btf->name = strdup(name);
6065 if (!mod_btf->name) {
6066 err = -ENOMEM;
6094 return ERR_PTR(-EINVAL);
6096 local_name = btf__name_by_offset(local_btf, local_t->name_off);
6098 return ERR_PTR(-EINVAL);
6103 return ERR_PTR(-ENOMEM);
6106 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
6112 if (cands->len)
6116 if (obj->btf_vmlinux_override)
6124 for (i = 0; i < obj->btf_module_cnt; i++) {
6126 obj->btf_modules[i].btf,
6127 obj->btf_modules[i].name,
6128 btf__type_cnt(obj->btf_vmlinux),
6141 * type-based CO-RE relocations and follow slightly different rules than
6142 * field-based relocations. This function assumes that root types were already
6143 * checked for name match. Beyond that initial root-level name check, names
6145 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
6148 * - for ENUMs, the size is ignored;
6149 * - for INT, size and signedness are ignored;
6150 * - for ARRAY, dimensionality is ignored, element types are checked for
6152 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
6153 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
6154 * - FUNC_PROTOs are compatible if they have compatible signature: same
6157 * more experience with using BPF CO-RE relocations.
6186 relos = libbpf_reallocarray(prog->reloc_desc,
6187 prog->nr_reloc + 1, sizeof(*relos));
6189 return -ENOMEM;
6190 relo = &relos[prog->nr_reloc];
6191 relo->type = RELO_CORE;
6192 relo->insn_idx = insn_idx;
6193 relo->core_relo = core_relo;
6194 prog->reloc_desc = relos;
6195 prog->nr_reloc++;
6204 for (i = 0; i < prog->nr_reloc; i++) {
6205 relo = &prog->reloc_desc[i];
6206 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)
6209 return relo->core_relo;
6224 const char *prog_name = prog->name;
6227 __u32 local_id = relo->type_id;
6232 return -EINVAL;
6234 local_name = btf__name_by_offset(local_btf, local_type->name_off);
6236 return -EINVAL;
6238 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
6240 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
6272 if (obj->btf_ext->core_relo_info.len == 0)
6276 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
6277 err = libbpf_get_error(obj->btf_vmlinux_override);
6290 seg = &obj->btf_ext->core_relo_info;
6293 sec_idx = seg->sec_idxs[sec_num];
6296 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6298 err = -EINVAL;
6302 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);
6305 if (rec->insn_off % BPF_INSN_SZ)
6306 return -EINVAL;
6307 insn_idx = rec->insn_off / BPF_INSN_SZ;
6314 * This is similar to what x86-64 linker does for relocations.
6318 …pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subpro…
6322 /* no need to apply CO-RE relocation if the program is
6325 if (!prog->autoload)
6329 * program's frame of reference; (sub-)program code is not yet
6330 * relocated, so it's enough to just subtract in-section offset
6332 insn_idx = insn_idx - prog->sec_insn_off;
6333 if (insn_idx >= prog->insns_cnt)
6334 return -EINVAL;
6335 insn = &prog->insns[insn_idx];
6340 prog->name, i, err);
6344 if (prog->obj->gen_loader)
6347 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
6350 prog->name, i, err);
6354 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
6357 prog->name, i, insn_idx, err);
6364 /* obj->btf_vmlinux and module BTFs are freed after object load */
6365 btf__free(obj->btf_vmlinux_override);
6366 obj->btf_vmlinux_override = NULL;
6370 bpf_core_free_cands(entry->pvalue);
6388 prog->name, relo_idx, insn_idx, map_idx, map->name);
6392 insn->code = BPF_JMP | BPF_CALL;
6393 insn->dst_reg = 0;
6394 insn->src_reg = 0;
6395 insn->off = 0;
6399 * where lower 123 is map index into obj->maps[] array
6401 insn->imm = POISON_LDIMM64_MAP_BASE + map_idx;
6416 prog->name, relo_idx, insn_idx, ext->name);
6419 insn->code = BPF_JMP | BPF_CALL;
6420 insn->dst_reg = 0;
6421 insn->src_reg = 0;
6422 insn->off = 0;
6426 * where lower 123 is extern index into obj->externs[] array
6428 insn->imm = POISON_CALL_KFUNC_BASE + ext_idx;
6432 * - map references;
6433 * - global variable references;
6434 * - extern references.
6441 for (i = 0; i < prog->nr_reloc; i++) {
6442 struct reloc_desc *relo = &prog->reloc_desc[i];
6443 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6447 switch (relo->type) {
6449 map = &obj->maps[relo->map_idx];
6450 if (obj->gen_loader) {
6452 insn[0].imm = relo->map_idx;
6453 } else if (map->autocreate) {
6455 insn[0].imm = map->fd;
6457 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6458 relo->map_idx, map);
6462 map = &obj->maps[relo->map_idx];
6463 insn[1].imm = insn[0].imm + relo->sym_off;
6464 if (obj->gen_loader) {
6466 insn[0].imm = relo->map_idx;
6467 } else if (map->autocreate) {
6469 insn[0].imm = map->fd;
6471 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6472 relo->map_idx, map);
6476 ext = &obj->externs[relo->ext_idx];
6477 if (ext->type == EXT_KCFG) {
6478 if (obj->gen_loader) {
6480 insn[0].imm = obj->kconfig_map_idx;
6483 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6485 insn[1].imm = ext->kcfg.data_off;
6487 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
6489 insn[0].imm = ext->ksym.kernel_btf_id;
6490 insn[1].imm = ext->ksym.kernel_btf_obj_fd;
6492 insn[0].imm = (__u32)ext->ksym.addr;
6493 insn[1].imm = ext->ksym.addr >> 32;
6498 ext = &obj->externs[relo->ext_idx];
6500 if (ext->is_set) {
6501 insn[0].imm = ext->ksym.kernel_btf_id;
6502 insn[0].off = ext->ksym.btf_fd_idx;
6504 poison_kfunc_call(prog, i, relo->insn_idx, insn,
6505 relo->ext_idx, ext);
6511 prog->name, i);
6512 return -EINVAL;
6524 prog->name, i, relo->type);
6525 return -EINVAL;
6546 sec_idx = ext_info->sec_idxs[sec_num];
6548 if (prog->sec_idx != sec_idx)
6554 if (insn_off < prog->sec_insn_off)
6556 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6561 copy_end = rec + ext_info->rec_size;
6565 return -ENOENT;
6567 /* append func/line info of a given (sub-)program to the main
6570 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
6571 new_sz = old_sz + (copy_end - copy_start);
6574 return -ENOMEM;
6576 *prog_rec_cnt = new_sz / ext_info->rec_size;
6577 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6579 /* Kernel instruction offsets are in units of 8-byte
6585 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6588 for (; rec < rec_end; rec += ext_info->rec_size) {
6593 *prog_rec_sz = ext_info->rec_size;
6597 return -ENOENT;
6610 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
6616 if (main_prog != prog && !main_prog->func_info)
6619 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6620 &main_prog->func_info,
6621 &main_prog->func_info_cnt,
6622 &main_prog->func_info_rec_size);
6624 if (err != -ENOENT) {
6626 prog->name, err);
6629 if (main_prog->func_info) {
6634 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6639 prog->name);
6644 if (main_prog != prog && !main_prog->line_info)
6647 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6648 &main_prog->line_info,
6649 &main_prog->line_info_cnt,
6650 &main_prog->line_info_rec_size);
6652 if (err != -ENOENT) {
6654 prog->name, err);
6657 if (main_prog->line_info) {
6662 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6667 prog->name);
6677 if (insn_idx == relo->insn_idx)
6679 return insn_idx < relo->insn_idx ? -1 : 1;
6684 if (!prog->nr_reloc)
6686 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6687 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6692 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
6698 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
6704 return -ENOMEM;
6705 if (subprog->nr_reloc)
6706 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
6707 sizeof(*relos) * subprog->nr_reloc);
6709 for (i = main_prog->nr_reloc; i < new_cnt; i++)
6710 relos[i].insn_idx += subprog->sub_insn_off;
6714 main_prog->reloc_desc = relos;
6715 main_prog->nr_reloc = new_cnt;
6727 subprog->sub_insn_off = main_prog->insns_cnt;
6729 new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6730 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6732 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6733 return -ENOMEM;
6735 main_prog->insns = insns;
6736 main_prog->insns_cnt = new_cnt;
6738 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6739 subprog->insns_cnt * sizeof(*insns));
6741 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6742 main_prog->name, subprog->insns_cnt, subprog->name);
6765 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6766 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6771 if (relo && relo->type == RELO_EXTERN_CALL)
6776 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6778 prog->name, insn_idx, relo->type);
6779 return -LIBBPF_ERRNO__RELOC;
6782 /* sub-program instruction index is a combination of
6785 * call always has imm = -1, but for static functions
6786 * relocation is against STT_SECTION and insn->imm
6789 * for subprog addr relocation, the relo->sym_off + insn->imm is
6792 if (relo->type == RELO_CALL)
6793 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6795 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6802 prog->name, insn_idx);
6803 return -LIBBPF_ERRNO__RELOC;
6808 * offset necessary, insns->imm is relative to
6811 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6814 /* we enforce that sub-programs should be in .text section */
6815 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6817 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6818 prog->name);
6819 return -LIBBPF_ERRNO__RELOC;
6825 * - append it at the end of main program's instructions blog;
6826 * - process is recursively, while current program is put on hold;
6827 * - if that subprogram calls some other not yet processes
6832 if (subprog->sub_insn_off == 0) {
6841 /* main_prog->insns memory could have been re-allocated, so
6844 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6851 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6854 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6861 * Relocate sub-program calls.
6863 * Algorithm operates as follows. Each entry-point BPF program (referred to as
6864 * main prog) is processed separately. For each subprog (non-entry functions,
6873 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6889 * subprog->sub_insn_off as zero at all times and won't be appended to current
6898 * +--------+ +-------+
6900 * +--+---+ +--+-+-+ +---+--+
6902 * +--+---+ +------+ +---+--+
6905 * +---+-------+ +------+----+
6907 * +-----------+ +-----------+
6912 * +-----------+------+
6914 * +-----------+------+
6919 * +-----------+------+------+
6921 * +-----------+------+------+
6930 * +-----------+------+
6932 * +-----------+------+
6935 * +-----------+------+------+
6937 * +-----------+------+------+
6950 for (i = 0; i < obj->nr_programs; i++) {
6951 subprog = &obj->programs[i];
6955 subprog->sub_insn_off = 0;
6972 for (i = 0; i < obj->nr_programs; i++) {
6973 prog = &obj->programs[i];
6974 zfree(&prog->reloc_desc);
6975 prog->nr_reloc = 0;
6984 if (a->insn_idx != b->insn_idx)
6985 return a->insn_idx < b->insn_idx ? -1 : 1;
6988 if (a->type != b->type)
6989 return a->type < b->type ? -1 : 1;
6998 for (i = 0; i < obj->nr_programs; i++) {
6999 struct bpf_program *p = &obj->programs[i];
7001 if (!p->nr_reloc)
7004 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
7015 if (obj->btf_ext) {
7018 pr_warn("failed to perform CO-RE relocations: %d\n",
7025 /* Before relocating calls pre-process relocations and mark
7032 for (i = 0; i < obj->nr_programs; i++) {
7033 prog = &obj->programs[i];
7034 for (j = 0; j < prog->nr_reloc; j++) {
7035 struct reloc_desc *relo = &prog->reloc_desc[j];
7036 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
7039 if (relo->type == RELO_SUBPROG_ADDR)
7051 for (i = 0; i < obj->nr_programs; i++) {
7052 prog = &obj->programs[i];
7053 /* sub-program's sub-calls are relocated within the context of
7058 if (!prog->autoload)
7064 prog->name, err);
7069 if (prog->exception_cb_idx >= 0) {
7070 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx];
7077 if (subprog->sub_insn_off == 0) {
7088 for (i = 0; i < obj->nr_programs; i++) {
7089 prog = &obj->programs[i];
7092 if (!prog->autoload)
7097 prog->name, err);
7125 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
7126 return -EINVAL;
7127 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
7129 return -EINVAL;
7131 nrels = shdr->sh_size / shdr->sh_entsize;
7136 return -LIBBPF_ERRNO__FORMAT;
7139 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
7142 i, (size_t)ELF64_R_SYM(rel->r_info));
7143 return -LIBBPF_ERRNO__FORMAT;
7145 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
7147 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",
7148 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
7149 (size_t)rel->r_offset, sym->st_name, name);
7151 for (j = 0; j < obj->nr_maps; j++) {
7152 map = &obj->maps[j];
7153 if (map->sec_idx != obj->efile.btf_maps_shndx)
7156 vi = btf_var_secinfos(sec) + map->btf_var_idx;
7157 if (vi->offset <= rel->r_offset &&
7158 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
7161 if (j == obj->nr_maps) {
7162 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
7163 i, name, (size_t)rel->r_offset);
7164 return -EINVAL;
7167 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
7168 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
7171 if (sym->st_shndx != obj->efile.btf_maps_shndx) {
7172 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
7174 return -LIBBPF_ERRNO__RELOC;
7176 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
7177 map->def.key_size != sizeof(int)) {
7178 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
7179 i, map->name, sizeof(int));
7180 return -EINVAL;
7186 return -ESRCH;
7193 return -ESRCH;
7195 if (targ_prog->sec_idx != sym->st_shndx ||
7196 targ_prog->sec_insn_off * 8 != sym->st_value ||
7198 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
7200 return -LIBBPF_ERRNO__RELOC;
7203 return -EINVAL;
7206 var = btf__type_by_id(obj->btf, vi->type);
7207 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
7209 return -EINVAL;
7210 member = btf_members(def) + btf_vlen(def) - 1;
7211 mname = btf__name_by_offset(obj->btf, member->name_off);
7213 return -EINVAL;
7215 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
7216 if (rel->r_offset - vi->offset < moff)
7217 return -EINVAL;
7219 moff = rel->r_offset - vi->offset - moff;
7224 return -EINVAL;
7226 if (moff >= map->init_slots_sz) {
7228 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
7230 return -ENOMEM;
7231 map->init_slots = tmp;
7232 memset(map->init_slots + map->init_slots_sz, 0,
7233 (new_sz - map->init_slots_sz) * host_ptr_sz);
7234 map->init_slots_sz = new_sz;
7236 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
7239 i, map->name, moff, type, name);
7249 for (i = 0; i < obj->efile.sec_cnt; i++) {
7250 struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
7256 if (sec_desc->sec_type != SEC_RELO)
7260 shdr = sec_desc->shdr;
7264 data = sec_desc->data;
7265 idx = shdr->sh_info;
7267 if (shdr->sh_type != SHT_REL) {
7269 return -LIBBPF_ERRNO__INTERNAL;
7272 if (idx == obj->efile.st_ops_shndx || idx == obj->efile.st_ops_link_shndx)
7274 else if (idx == obj->efile.btf_maps_shndx)
7288 if (BPF_CLASS(insn->code) == BPF_JMP &&
7289 BPF_OP(insn->code) == BPF_CALL &&
7290 BPF_SRC(insn->code) == BPF_K &&
7291 insn->src_reg == 0 &&
7292 insn->dst_reg == 0) {
7293 *func_id = insn->imm;
7301 struct bpf_insn *insn = prog->insns;
7305 if (obj->gen_loader)
7308 for (i = 0; i < prog->insns_cnt; i++, insn++) {
7320 insn->imm = BPF_FUNC_probe_read;
7325 insn->imm = BPF_FUNC_probe_read_str;
7337 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
7344 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
7345 opts->expected_attach_type = 0;
7348 opts->prog_flags |= BPF_F_SLEEPABLE;
7350 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
7351 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
7354 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) {
7359 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7360 opts->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7363 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
7367 attach_name = strchr(prog->sec_name, '/');
7378 …pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\…
7379 prog->name);
7380 return -EINVAL;
7389 prog->attach_btf_obj_fd = btf_obj_fd;
7390 prog->attach_btf_id = btf_type_id;
7393 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
7397 opts->attach_btf_obj_fd = btf_obj_fd;
7398 opts->attach_btf_id = btf_type_id;
7416 __u32 log_level = prog->log_level;
7418 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
7424 prog->name, prog->sec_name);
7425 return -EINVAL;
7429 return -EINVAL;
7432 prog_name = prog->name;
7433 load_attr.attach_prog_fd = prog->attach_prog_fd;
7434 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
7435 load_attr.attach_btf_id = prog->attach_btf_id;
7437 load_attr.prog_ifindex = prog->prog_ifindex;
7438 load_attr.expected_attach_type = prog->expected_attach_type;
7444 load_attr.func_info = prog->func_info;
7445 load_attr.func_info_rec_size = prog->func_info_rec_size;
7446 load_attr.func_info_cnt = prog->func_info_cnt;
7447 load_attr.line_info = prog->line_info;
7448 load_attr.line_info_rec_size = prog->line_info_rec_size;
7449 load_attr.line_info_cnt = prog->line_info_cnt;
7452 load_attr.prog_flags = prog->prog_flags;
7453 load_attr.fd_array = obj->fd_array;
7456 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
7457 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
7460 prog->name, err);
7463 insns = prog->insns;
7464 insns_cnt = prog->insns_cnt;
7467 if (obj->gen_loader) {
7468 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
7470 prog - obj->programs);
7471 *prog_fd = -1;
7482 if (prog->log_buf) {
7483 log_buf = prog->log_buf;
7484 log_buf_size = prog->log_size;
7486 } else if (obj->log_buf) {
7487 log_buf = obj->log_buf;
7488 log_buf_size = obj->log_size;
7494 ret = -ENOMEM;
7507 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
7510 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7511 prog->name, log_buf);
7514 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
7518 for (i = 0; i < obj->nr_maps; i++) {
7519 map = &prog->obj->maps[i];
7520 if (map->libbpf_type != LIBBPF_MAP_RODATA)
7526 prog->name, map->real_name, cp);
7546 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
7551 ret = -errno;
7553 /* post-process verifier log to improve error descriptions */
7557 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
7561 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7562 prog->name, log_buf);
7578 p = cur - 1;
7579 while (p - 1 >= buf && *(p - 1) != '\n')
7580 p--;
7588 /* size of the remaining log content to the right from the to-be-replaced part */
7589 size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
7594 * shift log contents by (patch_sz - orig_sz) bytes to the right
7595 * starting from after to-be-replaced part of the log.
7598 * shift log contents by (orig_sz - patch_sz) bytes to the left
7599 * starting from after to-be-replaced part of the log
7608 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
7610 } else if (patch_sz - orig_sz > buf_sz - log_sz) {
7612 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
7626 /* Expected log for failed and not properly guarded CO-RE relocation:
7627 * line1 -> 123: (85) call unknown#195896080
7628 * line2 -> invalid func unknown#195896080
7629 * line3 -> <anything else or end of buffer>
7632 * instruction index to find corresponding CO-RE relocation and
7634 * failed CO-RE relocation.
7648 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
7654 "%d: <invalid CO-RE relocation>\n"
7655 "failed to resolve CO-RE relocation %s%s\n",
7658 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7666 * line1 -> 123: (85) call unknown#2001000345
7667 * line2 -> invalid func unknown#2001000345
7668 * line3 -> <anything else or end of buffer>
7671 * "345" in "2001000345" is a map index in obj->maps to fetch map name.
7673 struct bpf_object *obj = prog->obj;
7681 map_idx -= POISON_LDIMM64_MAP_BASE;
7682 if (map_idx < 0 || map_idx >= obj->nr_maps)
7684 map = &obj->maps[map_idx];
7689 insn_idx, map->name);
7691 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7699 * line1 -> 123: (85) call unknown#2002000345
7700 * line2 -> invalid func unknown#2002000345
7701 * line3 -> <anything else or end of buffer>
7704 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name.
7706 struct bpf_object *obj = prog->obj;
7714 ext_idx -= POISON_CALL_KFUNC_BASE;
7715 if (ext_idx < 0 || ext_idx >= obj->nr_extern)
7717 ext = &obj->externs[ext_idx];
7722 insn_idx, ext->name);
7724 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7739 next_line = buf + log_sz - 1;
7751 /* failed CO-RE relocation case */
7779 struct bpf_object *obj = prog->obj;
7782 for (i = 0; i < prog->nr_reloc; i++) {
7783 struct reloc_desc *relo = &prog->reloc_desc[i];
7784 struct extern_desc *ext = &obj->externs[relo->ext_idx];
7787 switch (relo->type) {
7789 if (ext->type != EXT_KSYM)
7791 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ?
7793 bpf_gen__record_extern(obj->gen_loader, ext->name,
7794 ext->is_weak, !ext->ksym.type_id,
7795 true, kind, relo->insn_idx);
7798 bpf_gen__record_extern(obj->gen_loader, ext->name,
7799 ext->is_weak, false, false, BTF_KIND_FUNC,
7800 relo->insn_idx);
7804 .insn_off = relo->insn_idx * 8,
7805 .type_id = relo->core_relo->type_id,
7806 .access_str_off = relo->core_relo->access_str_off,
7807 .kind = relo->core_relo->kind,
7810 bpf_gen__record_relo_core(obj->gen_loader, &cr);
7827 for (i = 0; i < obj->nr_programs; i++) {
7828 prog = &obj->programs[i];
7834 for (i = 0; i < obj->nr_programs; i++) {
7835 prog = &obj->programs[i];
7838 if (!prog->autoload) {
7839 pr_debug("prog '%s': skipped loading\n", prog->name);
7842 prog->log_level |= log_level;
7844 if (obj->gen_loader)
7847 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7848 obj->license, obj->kern_version, &prog->fd);
7850 pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
7867 prog->sec_def = find_sec_def(prog->sec_name);
7868 if (!prog->sec_def) {
7871 prog->name, prog->sec_name);
7875 prog->type = prog->sec_def->prog_type;
7876 prog->expected_attach_type = prog->sec_def->expected_attach_type;
7881 if (prog->sec_def->prog_setup_fn) {
7882 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
7885 prog->name, err);
7906 return ERR_PTR(-EINVAL);
7911 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7916 return ERR_PTR(-EINVAL);
7930 return ERR_PTR(-EINVAL);
7932 return ERR_PTR(-EINVAL);
7938 obj->log_buf = log_buf;
7939 obj->log_size = log_size;
7940 obj->log_level = log_level;
7945 err = -ENAMETOOLONG;
7948 obj->btf_custom_path = strdup(btf_tmp_path);
7949 if (!obj->btf_custom_path) {
7950 err = -ENOMEM;
7957 obj->kconfig = strdup(kconfig);
7958 if (!obj->kconfig) {
7959 err = -ENOMEM;
7987 return libbpf_err_ptr(-EINVAL);
8004 return libbpf_err_ptr(-EINVAL);
8007 snprintf(tmp_name, sizeof(tmp_name), "%lx-%zx", (unsigned long)obj_buf, obj_buf_sz);
8017 return libbpf_err(-EINVAL);
8019 for (i = 0; i < obj->nr_maps; i++) {
8020 zclose(obj->maps[i].fd);
8021 if (obj->maps[i].st_ops)
8022 zfree(&obj->maps[i].st_ops->kern_vdata);
8025 for (i = 0; i < obj->nr_programs; i++)
8026 bpf_program__unload(&obj->programs[i]);
8039 m->def.map_flags &= ~BPF_F_MMAPABLE;
8054 err = -errno;
8066 err = -EINVAL;
8087 if (!ext || ext->type != EXT_KSYM)
8090 t = btf__type_by_id(obj->btf, ext->btf_id);
8094 if (ext->is_set && ext->ksym.addr != sym_addr) {
8096 sym_name, ext->ksym.addr, sym_addr);
8097 return -EINVAL;
8099 if (!ext->is_set) {
8100 ext->is_set = true;
8101 ext->ksym.addr = sym_addr;
8120 btf = obj->btf_vmlinux;
8124 if (id == -ENOENT) {
8129 for (i = 0; i < obj->btf_module_cnt; i++) {
8131 mod_btf = &obj->btf_modules[i];
8132 btf = mod_btf->btf;
8134 if (id != -ENOENT)
8139 return -ESRCH;
8156 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
8158 if (id == -ESRCH && ext->is_weak)
8161 ext->name);
8166 local_type_id = ext->ksym.type_id;
8170 targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
8171 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
8173 err = bpf_core_types_are_compat(obj->btf, local_type_id,
8179 local_type = btf__type_by_id(obj->btf, local_type_id);
8180 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
8181 targ_name = btf__name_by_offset(btf, targ_type->name_off);
8184 ext->name, local_type_id,
8187 return -EINVAL;
8190 ext->is_set = true;
8191 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8192 ext->ksym.kernel_btf_id = id;
8194 ext->name, id, btf_kind_str(targ_var), targ_var_name);
8208 local_func_proto_id = ext->ksym.type_id;
8210 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf,
8213 if (kfunc_id == -ESRCH && ext->is_weak)
8216 ext->name);
8221 kfunc_proto_id = kern_func->type;
8223 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
8226 if (ext->is_weak)
8230 ext->name, local_func_proto_id,
8231 mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id);
8232 return -EINVAL;
8236 if (mod_btf && !mod_btf->fd_array_idx) {
8237 /* insn->off is s16 */
8238 if (obj->fd_array_cnt == INT16_MAX) {
8240 ext->name, mod_btf->fd_array_idx);
8241 return -E2BIG;
8244 if (!obj->fd_array_cnt)
8245 obj->fd_array_cnt = 1;
8247 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
8248 obj->fd_array_cnt + 1);
8251 mod_btf->fd_array_idx = obj->fd_array_cnt;
8253 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
8256 ext->is_set = true;
8257 ext->ksym.kernel_btf_id = kfunc_id;
8258 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
8261 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call.
8262 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64.
8264 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8266 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id);
8277 for (i = 0; i < obj->nr_extern; i++) {
8278 ext = &obj->externs[i];
8279 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
8282 if (obj->gen_loader) {
8283 ext->is_set = true;
8284 ext->ksym.kernel_btf_obj_fd = 0;
8285 ext->ksym.kernel_btf_id = 0;
8288 t = btf__type_by_id(obj->btf, ext->btf_id);
8308 if (obj->nr_extern == 0)
8311 if (obj->kconfig_map_idx >= 0)
8312 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
8314 for (i = 0; i < obj->nr_extern; i++) {
8315 ext = &obj->externs[i];
8317 if (ext->type == EXT_KSYM) {
8318 if (ext->ksym.type_id)
8323 } else if (ext->type == EXT_KCFG) {
8324 void *ext_ptr = kcfg_data + ext->kcfg.data_off;
8328 if (str_has_pfx(ext->name, "CONFIG_")) {
8334 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
8337 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
8338 return -EINVAL;
8340 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
8342 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
8344 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
8352 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
8353 return -EINVAL;
8360 ext->name, (long long)value);
8362 pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
8363 return -EINVAL;
8369 return -EINVAL;
8371 for (i = 0; i < obj->nr_extern; i++) {
8372 ext = &obj->externs[i];
8373 if (ext->type == EXT_KCFG && !ext->is_set) {
8382 return -EINVAL;
8387 return -EINVAL;
8392 return -EINVAL;
8394 for (i = 0; i < obj->nr_extern; i++) {
8395 ext = &obj->externs[i];
8397 if (!ext->is_set && !ext->is_weak) {
8398 pr_warn("extern '%s' (strong): not resolved\n", ext->name);
8399 return -ESRCH;
8400 } else if (!ext->is_set) {
8402 ext->name);
8414 st_ops = map->st_ops;
8415 for (i = 0; i < btf_vlen(st_ops->type); i++) {
8416 struct bpf_program *prog = st_ops->progs[i];
8424 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
8433 for (i = 0; i < obj->nr_maps; i++)
8434 if (bpf_map__is_struct_ops(&obj->maps[i]))
8435 bpf_map_prepare_vdata(&obj->maps[i]);
8445 return libbpf_err(-EINVAL);
8447 if (obj->loaded) {
8448 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
8449 return libbpf_err(-EINVAL);
8452 if (obj->gen_loader)
8453 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
8457 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
8462 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
8467 if (obj->gen_loader) {
8469 if (obj->btf)
8470 btf__set_fd(obj->btf, -1);
8471 for (i = 0; i < obj->nr_maps; i++)
8472 obj->maps[i].fd = -1;
8474 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
8478 zfree(&obj->fd_array);
8481 for (i = 0; i < obj->btf_module_cnt; i++) {
8482 close(obj->btf_modules[i].fd);
8483 btf__free(obj->btf_modules[i].btf);
8484 free(obj->btf_modules[i].name);
8486 free(obj->btf_modules);
8489 btf__free(obj->btf_vmlinux);
8490 obj->btf_vmlinux = NULL;
8492 obj->loaded = true; /* doesn't matter if successfully or not */
8499 /* unpin any maps that were auto-pinned during load */
8500 for (i = 0; i < obj->nr_maps; i++)
8501 if (obj->maps[i].pinned && !obj->maps[i].reused)
8502 bpf_map__unpin(&obj->maps[i], NULL);
8505 pr_warn("failed to load object '%s'\n", obj->path);
8522 return -ENOMEM;
8526 err = -errno;
8530 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8544 return -EINVAL;
8548 return -ENOMEM;
8554 err = -errno;
8560 err = -EINVAL;
8571 if (prog->fd < 0) {
8572 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
8573 return libbpf_err(-EINVAL);
8584 if (bpf_obj_pin(prog->fd, path)) {
8585 err = -errno;
8587 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
8591 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
8599 if (prog->fd < 0) {
8600 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
8601 return libbpf_err(-EINVAL);
8610 return libbpf_err(-errno);
8612 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
8623 return libbpf_err(-EINVAL);
8626 if (map->pin_path) {
8627 if (path && strcmp(path, map->pin_path)) {
8629 bpf_map__name(map), map->pin_path, path);
8630 return libbpf_err(-EINVAL);
8631 } else if (map->pinned) {
8632 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
8633 bpf_map__name(map), map->pin_path);
8640 return libbpf_err(-EINVAL);
8641 } else if (map->pinned) {
8643 return libbpf_err(-EEXIST);
8646 map->pin_path = strdup(path);
8647 if (!map->pin_path) {
8648 err = -errno;
8653 err = make_parent_dir(map->pin_path);
8657 err = check_path(map->pin_path);
8661 if (bpf_obj_pin(map->fd, map->pin_path)) {
8662 err = -errno;
8666 map->pinned = true;
8667 pr_debug("pinned map '%s'\n", map->pin_path);
8672 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8683 return libbpf_err(-EINVAL);
8686 if (map->pin_path) {
8687 if (path && strcmp(path, map->pin_path)) {
8689 bpf_map__name(map), map->pin_path, path);
8690 return libbpf_err(-EINVAL);
8692 path = map->pin_path;
8696 return libbpf_err(-EINVAL);
8705 return libbpf_err(-errno);
8707 map->pinned = false;
8720 return libbpf_err(-errno);
8723 free(map->pin_path);
8724 map->pin_path = new;
8733 return map->pin_path;
8738 return map->pinned;
8757 return libbpf_err(-ENOENT);
8759 if (!obj->loaded) {
8761 return libbpf_err(-ENOENT);
8768 if (!map->autocreate)
8777 } else if (!map->pin_path) {
8790 if (!map->pin_path)
8805 return libbpf_err(-ENOENT);
8817 } else if (!map->pin_path) {
8836 return libbpf_err(-ENOENT);
8838 if (!obj->loaded) {
8840 return libbpf_err(-ENOENT);
8844 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8857 if (pathname_concat(buf, sizeof(buf), path, prog->name))
8872 return libbpf_err(-ENOENT);
8877 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8923 if (map->inner_map) {
8924 bpf_map__destroy(map->inner_map);
8925 zfree(&map->inner_map);
8928 zfree(&map->init_slots);
8929 map->init_slots_sz = 0;
8931 if (map->mmaped) {
8934 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
8935 munmap(map->mmaped, mmap_sz);
8936 map->mmaped = NULL;
8939 if (map->st_ops) {
8940 zfree(&map->st_ops->data);
8941 zfree(&map->st_ops->progs);
8942 zfree(&map->st_ops->kern_func_off);
8943 zfree(&map->st_ops);
8946 zfree(&map->name);
8947 zfree(&map->real_name);
8948 zfree(&map->pin_path);
8950 if (map->fd >= 0)
8951 zclose(map->fd);
8961 usdt_manager_free(obj->usdt_man);
8962 obj->usdt_man = NULL;
8964 bpf_gen__free(obj->gen_loader);
8967 btf__free(obj->btf);
8968 btf__free(obj->btf_vmlinux);
8969 btf_ext__free(obj->btf_ext);
8971 for (i = 0; i < obj->nr_maps; i++)
8972 bpf_map__destroy(&obj->maps[i]);
8974 zfree(&obj->btf_custom_path);
8975 zfree(&obj->kconfig);
8977 for (i = 0; i < obj->nr_extern; i++)
8978 zfree(&obj->externs[i].essent_name);
8980 zfree(&obj->externs);
8981 obj->nr_extern = 0;
8983 zfree(&obj->maps);
8984 obj->nr_maps = 0;
8986 if (obj->programs && obj->nr_programs) {
8987 for (i = 0; i < obj->nr_programs; i++)
8988 bpf_program__exit(&obj->programs[i]);
8990 zfree(&obj->programs);
8997 return obj ? obj->name : libbpf_err_ptr(-EINVAL);
9002 return obj ? obj->kern_version : 0;
9007 return obj ? obj->btf : NULL;
9012 return obj->btf ? btf__fd(obj->btf) : -1;
9017 if (obj->loaded)
9018 return libbpf_err(-EINVAL);
9020 obj->kern_version = kern_version;
9030 return -EFAULT;
9032 return -EINVAL;
9035 return -ENOMEM;
9036 gen->opts = opts;
9037 obj->gen_loader = gen;
9045 size_t nr_programs = obj->nr_programs;
9053 return forward ? &obj->programs[0] :
9054 &obj->programs[nr_programs - 1];
9056 if (p->obj != obj) {
9061 idx = (p - obj->programs) + (forward ? 1 : -1);
9062 if (idx >= obj->nr_programs || idx < 0)
9064 return &obj->programs[idx];
9093 prog->prog_ifindex = ifindex;
9098 return prog->name;
9103 return prog->sec_name;
9108 return prog->autoload;
9113 if (prog->obj->loaded)
9114 return libbpf_err(-EINVAL);
9116 prog->autoload = autoload;
9122 return prog->autoattach;
9127 prog->autoattach = autoattach;
9132 return prog->insns;
9137 return prog->insns_cnt;
9145 if (prog->obj->loaded)
9146 return -EBUSY;
9148 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
9151 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
9152 return -ENOMEM;
9156 prog->insns = insns;
9157 prog->insns_cnt = new_insn_cnt;
9164 return libbpf_err(-EINVAL);
9166 if (prog->fd < 0)
9167 return libbpf_err(-ENOENT);
9169 return prog->fd;
9177 return prog->type;
9188 if (prog->obj->loaded)
9189 return libbpf_err(-EBUSY);
9192 if (prog->type == type)
9195 prog->type = type;
9199 * fallback handler, which by definition is program type-agnostic and
9200 * is a catch-all custom handler, optionally set by the application,
9203 if (prog->sec_def != &custom_fallback_def)
9204 prog->sec_def = NULL;
9213 return prog->expected_attach_type;
9219 if (prog->obj->loaded)
9220 return libbpf_err(-EBUSY);
9222 prog->expected_attach_type = type;
9228 return prog->prog_flags;
9233 if (prog->obj->loaded)
9234 return libbpf_err(-EBUSY);
9236 prog->prog_flags = flags;
9242 return prog->log_level;
9247 if (prog->obj->loaded)
9248 return libbpf_err(-EBUSY);
9250 prog->log_level = log_level;
9256 *log_size = prog->log_size;
9257 return prog->log_buf;
9263 return -EINVAL;
9264 if (prog->log_size > UINT_MAX)
9265 return -EINVAL;
9266 if (prog->obj->loaded)
9267 return -EBUSY;
9269 prog->log_buf = log_buf;
9270 prog->log_size = log_size;
9394 SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT),
9409 return libbpf_err(-EINVAL);
9412 return libbpf_err(-E2BIG);
9418 return libbpf_err(-ENOMEM);
9424 return libbpf_err(-EBUSY);
9429 sec_def->sec = sec ? strdup(sec) : NULL;
9430 if (sec && !sec_def->sec)
9431 return libbpf_err(-ENOMEM);
9433 sec_def->prog_type = prog_type;
9434 sec_def->expected_attach_type = exp_attach_type;
9435 sec_def->cookie = OPTS_GET(opts, cookie, 0);
9437 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
9438 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
9439 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
9441 sec_def->handler_id = ++last_custom_sec_def_handler_id;
9448 return sec_def->handler_id;
9457 return libbpf_err(-EINVAL);
9471 return libbpf_err(-ENOENT);
9475 custom_sec_defs[i - 1] = custom_sec_defs[i];
9476 custom_sec_def_cnt--;
9492 size_t len = strlen(sec_def->sec);
9495 if (sec_def->sec[len - 1] == '/') {
9496 if (str_has_pfx(sec_name, sec_def->sec))
9502 * well-formed SEC("type/extras") with proper '/' separator
9504 if (sec_def->sec[len - 1] == '+') {
9505 len--;
9507 if (strncmp(sec_name, sec_def->sec, len) != 0)
9515 return strcmp(sec_name, sec_def->sec) == 0;
9560 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9563 if (!(sec_def->cookie & SEC_ATTACHABLE))
9585 return libbpf_err(-EINVAL);
9589 *prog_type = sec_def->prog_type;
9590 *expected_attach_type = sec_def->expected_attach_type;
9601 return libbpf_err(-ESRCH);
9643 for (i = 0; i < obj->nr_maps; i++) {
9644 map = &obj->maps[i];
9647 if (map->sec_idx == sec_idx &&
9648 map->sec_offset <= offset &&
9649 offset - map->sec_offset < map->def.value_size)
9656 /* Collect the reloc from ELF and populate the st_ops->progs[] */
9673 btf = obj->btf;
9674 nrels = shdr->sh_size / shdr->sh_entsize;
9679 return -LIBBPF_ERRNO__FORMAT;
9682 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
9685 (size_t)ELF64_R_SYM(rel->r_info));
9686 return -LIBBPF_ERRNO__FORMAT;
9689 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
9690 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset);
9692 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
9693 (size_t)rel->r_offset);
9694 return -EINVAL;
9697 moff = rel->r_offset - map->sec_offset;
9698 shdr_idx = sym->st_shndx;
9699 st_ops = map->st_ops;
9700 …pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %…
9701 map->name,
9702 (long long)(rel->r_info >> 32),
9703 (long long)sym->st_value,
9704 shdr_idx, (size_t)rel->r_offset,
9705 map->sec_offset, sym->st_name, name);
9708 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
9709 map->name, (size_t)rel->r_offset, shdr_idx);
9710 return -LIBBPF_ERRNO__RELOC;
9712 if (sym->st_value % BPF_INSN_SZ) {
9714 map->name, (unsigned long long)sym->st_value);
9715 return -LIBBPF_ERRNO__FORMAT;
9717 insn_idx = sym->st_value / BPF_INSN_SZ;
9719 member = find_member_by_offset(st_ops->type, moff * 8);
9722 map->name, moff);
9723 return -EINVAL;
9725 member_idx = member - btf_members(st_ops->type);
9726 name = btf__name_by_offset(btf, member->name_off);
9728 if (!resolve_func_ptr(btf, member->type, NULL)) {
9730 map->name, name);
9731 return -EINVAL;
9737 map->name, shdr_idx, name);
9738 return -EINVAL;
9742 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
9744 map->name, prog->name);
9745 return -EINVAL;
9751 if (!prog->attach_btf_id) {
9752 prog->attach_btf_id = st_ops->type_id;
9753 prog->expected_attach_type = member_idx;
9756 /* struct_ops BPF prog can be re-used between multiple
9761 if (prog->attach_btf_id != st_ops->type_id ||
9762 prog->expected_attach_type != member_idx) {
9764 map->name, prog->name, prog->sec_name, prog->type,
9765 prog->attach_btf_id, prog->expected_attach_type, name);
9766 return -EINVAL;
9769 st_ops->progs[member_idx] = prog;
9816 return -ENAMETOOLONG;
9866 err = -EINVAL;
9893 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
9899 if (ret != -ENOENT)
9906 for (i = 0; i < obj->btf_module_cnt; i++) {
9907 const struct module_btf *mod = &obj->btf_modules[i];
9909 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
9911 *btf_obj_fd = mod->fd;
9915 if (ret == -ENOENT)
9921 return -ESRCH;
9927 enum bpf_attach_type attach_type = prog->expected_attach_type;
9928 __u32 attach_prog_fd = prog->attach_prog_fd;
9932 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
9934 pr_warn("prog '%s': attach program FD is not set\n", prog->name);
9935 return -EINVAL;
9940 prog->name, attach_prog_fd, attach_name, err);
9949 if (prog->obj->gen_loader) {
9950 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
9954 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9958 prog->name, attach_name, err);
9971 return libbpf_err(-EINVAL);
9982 return libbpf_err(-EINVAL);
9985 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9986 return libbpf_err(-EINVAL);
9987 if (!(sec_def->cookie & SEC_ATTACHABLE))
9988 return libbpf_err(-EINVAL);
9990 *attach_type = sec_def->expected_attach_type;
9996 return map ? map->fd : libbpf_err(-EINVAL);
10002 * their user-visible name differs from kernel-visible name. Users see
10007 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
10009 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
10020 return map->real_name;
10022 return map->name;
10027 return map->def.type;
10032 if (map->fd >= 0)
10033 return libbpf_err(-EBUSY);
10034 map->def.type = type;
10040 return map->def.map_flags;
10045 if (map->fd >= 0)
10046 return libbpf_err(-EBUSY);
10047 map->def.map_flags = flags;
10053 return map->map_extra;
10058 if (map->fd >= 0)
10059 return libbpf_err(-EBUSY);
10060 map->map_extra = map_extra;
10066 return map->numa_node;
10071 if (map->fd >= 0)
10072 return libbpf_err(-EBUSY);
10073 map->numa_node = numa_node;
10079 return map->def.key_size;
10084 if (map->fd >= 0)
10085 return libbpf_err(-EBUSY);
10086 map->def.key_size = size;
10092 return map->def.value_size;
10106 btf = bpf_object__btf(map->obj);
10108 return -ENOENT;
10115 return -EINVAL;
10123 return -EINVAL;
10127 var = &btf_var_secinfos(datasec_type)[vlen - 1];
10128 var_type = btf_type_by_id(btf, var->type);
10129 array_type = skip_mods_and_typedefs(btf, var_type->type, NULL);
10133 return -EINVAL;
10138 element_sz = btf__resolve_size(btf, array->type);
10139 if (element_sz <= 0 || (size - var->offset) % element_sz != 0) {
10142 return -EINVAL;
10146 nr_elements = (size - var->offset) / element_sz;
10147 new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements);
10154 datasec_type = btf_type_by_id(btf, map->btf_value_type_id);
10155 var = &btf_var_secinfos(datasec_type)[vlen - 1];
10156 var_type = btf_type_by_id(btf, var->type);
10159 datasec_type->size = size;
10160 var->size = size - var->offset;
10161 var_type->type = new_array_id;
10168 if (map->fd >= 0)
10169 return libbpf_err(-EBUSY);
10171 if (map->mmaped) {
10175 mmap_old_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
10176 mmap_new_sz = bpf_map_mmap_sz(size, map->def.max_entries);
10179 pr_warn("map '%s': failed to resize memory-mapped region: %d\n",
10184 if (err && err != -ENOENT) {
10187 map->btf_value_type_id = 0;
10188 map->btf_key_type_id = 0;
10192 map->def.value_size = size;
10198 return map ? map->btf_key_type_id : 0;
10203 return map ? map->btf_value_type_id : 0;
10209 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
10210 size != map->def.value_size || map->fd >= 0)
10211 return libbpf_err(-EINVAL);
10213 memcpy(map->mmaped, data, size);
10219 if (!map->mmaped)
10221 *psize = map->def.value_size;
10222 return map->mmaped;
10227 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
10232 return map->map_ifindex;
10237 if (map->fd >= 0)
10238 return libbpf_err(-EBUSY);
10239 map->map_ifindex = ifindex;
10245 if (!bpf_map_type__is_map_in_map(map->def.type)) {
10247 return libbpf_err(-EINVAL);
10249 if (map->inner_map_fd != -1) {
10251 return libbpf_err(-EINVAL);
10253 if (map->inner_map) {
10254 bpf_map__destroy(map->inner_map);
10255 zfree(&map->inner_map);
10257 map->inner_map_fd = fd;
10267 if (!obj || !obj->maps)
10270 s = obj->maps;
10271 e = obj->maps + obj->nr_maps;
10279 idx = (m - obj->maps) + i;
10280 if (idx >= obj->nr_maps || idx < 0)
10282 return &obj->maps[idx];
10289 return obj->maps;
10298 if (!obj->nr_maps)
10300 return obj->maps + obj->nr_maps - 1;
10303 return __bpf_map__iter(next, obj, -1);
10317 if (pos->real_name && strcmp(pos->real_name, name) == 0)
10323 if (strcmp(pos->real_name, name) == 0)
10327 if (strcmp(pos->name, name) == 0)
10342 if (map->fd <= 0)
10343 return -ENOENT;
10345 if (map->def.key_size != key_sz) {
10347 map->name, key_sz, map->def.key_size);
10348 return -EINVAL;
10354 switch (map->def.type) {
10360 size_t elem_sz = roundup(map->def.value_size, 8);
10363 … pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",
10364 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
10365 return -EINVAL;
10370 if (map->def.value_size != value_sz) {
10372 map->name, value_sz, map->def.value_size);
10373 return -EINVAL;
10390 return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
10403 return bpf_map_update_elem(map->fd, key, value, flags);
10415 return bpf_map_delete_elem_flags(map->fd, key, flags);
10428 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
10440 return bpf_map_get_next_key(map->fd, cur_key, next_key);
10449 errno = -PTR_ERR(ptr);
10456 return -errno;
10480 link->disconnected = true;
10490 if (!link->disconnected && link->detach)
10491 err = link->detach(link);
10492 if (link->pin_path)
10493 free(link->pin_path);
10494 if (link->dealloc)
10495 link->dealloc(link);
10504 return link->fd;
10509 return link->pin_path;
10514 return libbpf_err_errno(close(link->fd));
10524 fd = -errno;
10532 return libbpf_err_ptr(-ENOMEM);
10534 link->detach = &bpf_link__detach_fd;
10535 link->fd = fd;
10537 link->pin_path = strdup(path);
10538 if (!link->pin_path) {
10540 return libbpf_err_ptr(-ENOMEM);
10548 return bpf_link_detach(link->fd) ? -errno : 0;
10555 if (link->pin_path)
10556 return libbpf_err(-EBUSY);
10564 link->pin_path = strdup(path);
10565 if (!link->pin_path)
10566 return libbpf_err(-ENOMEM);
10568 if (bpf_obj_pin(link->fd, link->pin_path)) {
10569 err = -errno;
10570 zfree(&link->pin_path);
10574 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
10582 if (!link->pin_path)
10583 return libbpf_err(-EINVAL);
10585 err = unlink(link->pin_path);
10587 return -errno;
10589 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
10590 zfree(&link->pin_path);
10611 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
10612 err = -errno;
10614 if (perf_link->perf_event_fd != link->fd)
10615 close(perf_link->perf_event_fd);
10616 close(link->fd);
10619 if (perf_link->legacy_probe_name) {
10620 if (perf_link->legacy_is_kprobe) {
10621 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
10622 perf_link->legacy_is_retprobe);
10624 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
10625 perf_link->legacy_is_retprobe);
10636 free(perf_link->legacy_probe_name);
10645 int prog_fd, link_fd = -1, err;
10649 return libbpf_err_ptr(-EINVAL);
10653 prog->name, pfd);
10654 return libbpf_err_ptr(-EINVAL);
10659 prog->name);
10660 return libbpf_err_ptr(-EINVAL);
10665 return libbpf_err_ptr(-ENOMEM);
10666 link->link.detach = &bpf_link_perf_detach;
10667 link->link.dealloc = &bpf_link_perf_dealloc;
10668 link->perf_event_fd = pfd;
10671 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {
10677 err = -errno;
10679 prog->name, pfd,
10683 link->link.fd = link_fd;
10686 pr_warn("prog '%s': user context value is not supported\n", prog->name);
10687 err = -EOPNOTSUPP;
10692 err = -errno;
10694 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10695 if (err == -EPROTO)
10697 prog->name, pfd);
10700 link->link.fd = pfd;
10703 err = -errno;
10705 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10709 return &link->link;
10723 * this function is expected to parse integer in the range of [0, 2^31-1] from
10735 err = -errno;
10742 err = err == EOF ? -EIO : -errno;
10792 return -EINVAL;
10824 pid < 0 ? -1 : pid /* pid */,
10825 pid == -1 ? 0 : -1 /* cpu */,
10826 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10827 return pfd >= 0 ? pfd : -errno;
10841 return -EINVAL;
10845 return -errno;
10848 err = -errno;
10859 static int has_debugfs = -1;
10921 return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
10965 pid < 0 ? -1 : pid, /* pid */
10966 pid == -1 ? 0 : -1, /* cpu */
10967 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10969 err = -errno;
11055 return libbpf_err_ptr(-EINVAL);
11070 return libbpf_err_ptr(-ENOTSUP);
11074 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
11075 return libbpf_err_ptr(-ENOTSUP);
11080 return libbpf_err_ptr(-EINVAL);
11086 -1 /* pid */, 0 /* ref_ctr_off */);
11095 return libbpf_err_ptr(-ENOMEM);
11098 offset, -1 /* pid */);
11101 err = -errno;
11103 prog->name, retprobe ? "kretprobe" : "kprobe",
11113 prog->name, retprobe ? "kretprobe" : "kprobe",
11121 perf_link->legacy_probe_name = legacy_probe;
11122 perf_link->legacy_is_kprobe = true;
11123 perf_link->legacy_is_retprobe = retprobe;
11155 return libbpf_err_ptr(-EINVAL);
11157 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
11224 struct kprobe_multi_resolve *res = data->res;
11227 if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp))
11230 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1);
11234 res->addrs[res->cnt++] = (unsigned long)sym_addr;
11250 err = -errno;
11264 err = -EINVAL;
11268 if (!glob_match(sym_name, res->pattern))
11277 err = -errno;
11286 err = -ENOENT;
11298 if (res->cnt == 0)
11299 err = -ENOENT;
11312 return access(tracefs_available_filter_functions_addrs(), R_OK) != -1;
11325 err = -errno;
11338 err = -EINVAL;
11342 if (!glob_match(sym_name, res->pattern))
11345 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap,
11346 sizeof(*res->addrs), res->cnt + 1);
11350 res->addrs[res->cnt++] = (unsigned long)sym_addr;
11353 if (res->cnt == 0)
11354 err = -ENOENT;
11380 return libbpf_err_ptr(-EINVAL);
11388 return libbpf_err_ptr(-EINVAL);
11390 return libbpf_err_ptr(-EINVAL);
11392 return libbpf_err_ptr(-EINVAL);
11394 return libbpf_err_ptr(-EINVAL);
11417 err = -ENOMEM;
11420 link->detach = &bpf_link__detach_fd;
11425 err = -errno;
11427 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11430 link->fd = link_fd;
11450 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
11451 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
11454 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
11456 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
11458 func_name = prog->sec_name + sizeof("kprobe/") - 1;
11460 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
11463 return -EINVAL;
11468 return -EINVAL;
11484 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
11485 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
11488 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
11490 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
11492 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
11495 return *link ? 0 : -errno;
11507 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
11508 if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
11509 strcmp(prog->sec_name, "kretprobe.multi") == 0)
11512 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
11514 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
11516 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
11518 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
11521 return -EINVAL;
11533 int n, ret = -EINVAL;
11537 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
11541 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
11546 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
11550 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
11551 prog->sec_name);
11585 return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
11626 pid < 0 ? -1 : pid, /* pid */
11627 pid == -1 ? 0 : -1, /* cpu */
11628 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11630 err = -errno;
11682 ret = -LIBBPF_ERRNO__FORMAT;
11698 elf_errmsg(-1));
11699 ret = -LIBBPF_ERRNO__LIBELF;
11723 * Based on https://packages.debian.org/sid/libc6.
11729 return "/lib/x86_64-linux-gnu";
11731 return "/lib/i386-linux-gnu";
11733 return "/lib/s390x-linux-gnu";
11735 return "/lib/s390-linux-gnu";
11737 return "/lib/arm-linux-gnueabi";
11739 return "/lib/arm-linux-gnueabihf";
11741 return "/lib/aarch64-linux-gnu";
11743 return "/lib/mips64el-linux-gnuabi64";
11745 return "/lib/mipsel-linux-gnu";
11747 return "/lib/powerpc64le-linux-gnu";
11749 return "/lib/sparc64-linux-gnu";
11751 return "/lib/riscv64-linux-gnu";
11786 seg_len = next_path ? next_path - s : strlen(s);
11797 return -ENOENT;
11819 return libbpf_err_ptr(-EINVAL);
11835 * - syms and offsets are mutually exclusive
11836 * - ref_ctr_offsets and cookies are optional
11842 return libbpf_err_ptr(-EINVAL);
11844 return libbpf_err_ptr(-EINVAL);
11848 return libbpf_err_ptr(-EINVAL);
11851 return libbpf_err_ptr(-EINVAL);
11859 prog->name, path, err);
11891 err = -ENOMEM;
11894 link->detach = &bpf_link__detach_fd;
11899 err = -errno;
11900 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
11901 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11904 link->fd = link_fd;
11931 return libbpf_err_ptr(-EINVAL);
11939 return libbpf_err_ptr(-EINVAL);
11946 min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1)));
11953 prog->name, binary_path, err);
11982 return libbpf_err_ptr(-ENOTSUP);
11986 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
11987 return libbpf_err_ptr(-ENOTSUP);
11992 return libbpf_err_ptr(-EINVAL);
12002 return libbpf_err_ptr(-EINVAL);
12009 return libbpf_err_ptr(-ENOMEM);
12015 err = -errno;
12017 prog->name, retprobe ? "uretprobe" : "uprobe",
12028 prog->name, retprobe ? "uretprobe" : "uprobe",
12036 perf_link->legacy_probe_name = legacy_probe;
12037 perf_link->legacy_is_kprobe = false;
12038 perf_link->legacy_is_retprobe = retprobe;
12050 /* Format of u[ret]probe section definition supporting auto-attach:
12057 * specified (and auto-attach is not possible) or the above format is specified for
12058 * auto-attach.
12064 int n, c, ret = -EINVAL;
12069 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
12073 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
12078 prog->name, prog->sec_name);
12096 prog->name);
12100 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
12104 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
12105 prog->sec_name);
12132 struct bpf_object *obj = prog->obj;
12138 return libbpf_err_ptr(-EINVAL);
12142 prog->name);
12143 return libbpf_err_ptr(-EINVAL);
12147 return libbpf_err_ptr(-EINVAL);
12153 prog->name, binary_path, err);
12162 if (IS_ERR(obj->usdt_man))
12163 return libbpf_ptr(obj->usdt_man);
12164 if (!obj->usdt_man) {
12165 obj->usdt_man = usdt_manager_new(obj);
12166 if (IS_ERR(obj->usdt_man))
12167 return libbpf_ptr(obj->usdt_man);
12171 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
12189 /* no auto-attach for just SEC("usdt") */
12198 err = -EINVAL;
12200 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
12220 return -errno;
12224 return -E2BIG;
12250 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
12251 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
12253 err = -errno;
12273 return libbpf_err_ptr(-EINVAL);
12280 prog->name, tp_category, tp_name,
12289 prog->name, tp_category, tp_name,
12309 /* no auto-attach for SEC("tp") or SEC("tracepoint") */
12310 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
12313 sec_name = strdup(prog->sec_name);
12315 return -ENOMEM;
12318 if (str_has_pfx(prog->sec_name, "tp/"))
12319 tp_cat = sec_name + sizeof("tp/") - 1;
12321 tp_cat = sec_name + sizeof("tracepoint/") - 1;
12325 return -EINVAL;
12344 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12345 return libbpf_err_ptr(-EINVAL);
12350 return libbpf_err_ptr(-ENOMEM);
12351 link->detach = &bpf_link__detach_fd;
12355 pfd = -errno;
12358 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12361 link->fd = pfd;
12381 if (!str_has_pfx(prog->sec_name, prefixes[i]))
12385 /* no auto-attach case of, e.g., SEC("raw_tp") */
12386 if (prog->sec_name[pfx_len] == '\0')
12389 if (prog->sec_name[pfx_len] != '/')
12392 tp_name = prog->sec_name + pfx_len + 1;
12398 prog->name, prog->sec_name);
12399 return -EINVAL;
12416 return libbpf_err_ptr(-EINVAL);
12420 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12421 return libbpf_err_ptr(-EINVAL);
12426 return libbpf_err_ptr(-ENOMEM);
12427 link->detach = &bpf_link__detach_fd;
12433 pfd = -errno;
12436 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12439 link->fd = pfd;
12483 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12484 return libbpf_err_ptr(-EINVAL);
12489 return libbpf_err_ptr(-ENOMEM);
12490 link->detach = &bpf_link__detach_fd;
12495 link_fd = -errno;
12498 prog->name, target_name,
12502 link->fd = link_fd;
12533 return libbpf_err_ptr(-EINVAL);
12538 /* validate we don't have unexpected combinations of non-zero fields */
12541 prog->name);
12542 return libbpf_err_ptr(-EINVAL);
12546 prog->name);
12547 return libbpf_err_ptr(-EINVAL);
12568 return libbpf_err_ptr(-EINVAL);
12573 /* validate we don't have unexpected combinations of non-zero fields */
12576 prog->name);
12577 return libbpf_err_ptr(-EINVAL);
12581 prog->name);
12582 return libbpf_err_ptr(-EINVAL);
12601 prog->name);
12602 return libbpf_err_ptr(-EINVAL);
12605 if (prog->type != BPF_PROG_TYPE_EXT) {
12607 prog->name);
12608 return libbpf_err_ptr(-EINVAL);
12641 return libbpf_err_ptr(-EINVAL);
12648 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12649 return libbpf_err_ptr(-EINVAL);
12654 return libbpf_err_ptr(-ENOMEM);
12655 link->detach = &bpf_link__detach_fd;
12660 link_fd = -errno;
12663 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12666 link->fd = link_fd;
12684 return libbpf_err_ptr(-EINVAL);
12688 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12689 return libbpf_err_ptr(-EINVAL);
12694 return libbpf_err_ptr(-ENOMEM);
12696 link->detach = &bpf_link__detach_fd;
12707 link_fd = -errno;
12710 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12713 link->fd = link_fd;
12723 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
12724 return libbpf_err_ptr(-EOPNOTSUPP);
12726 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
12730 /* When calling bpf_program__attach() explicitly, auto-attach support
12736 return libbpf_err_ptr(-EOPNOTSUPP);
12753 if (st_link->map_fd < 0)
12755 return bpf_map_delete_elem(link->fd, &zero);
12757 return close(link->fd);
12766 if (!bpf_map__is_struct_ops(map) || map->fd == -1)
12767 return libbpf_err_ptr(-EINVAL);
12771 return libbpf_err_ptr(-EINVAL);
12774 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
12780 if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) {
12785 link->link.detach = bpf_link__detach_struct_ops;
12787 if (!(map->def.map_flags & BPF_F_LINK)) {
12789 link->link.fd = map->fd;
12790 link->map_fd = -1;
12791 return &link->link;
12794 fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL);
12800 link->link.fd = fd;
12801 link->map_fd = map->fd;
12803 return &link->link;
12815 if (!bpf_map__is_struct_ops(map) || map->fd < 0)
12816 return -EINVAL;
12820 if (st_ops_link->map_fd < 0)
12821 return -EINVAL;
12823 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
12829 if (err && err != -EBUSY)
12832 err = bpf_link_update(link->fd, map->fd, NULL);
12836 st_ops_link->map_fd = map->fd;
12851 __u64 data_tail = header->data_tail;
12858 ehdr = base + (data_tail & (mmap_size - 1));
12859 ehdr_size = ehdr->size;
12863 size_t len_first = base + mmap_size - copy_start;
12864 size_t len_secnd = ehdr_size - len_first;
12898 /* sample_cb and lost_cb are higher-level common-case callbacks */
12937 if (cpu_buf->base &&
12938 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
12939 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
12940 if (cpu_buf->fd >= 0) {
12941 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
12942 close(cpu_buf->fd);
12944 free(cpu_buf->buf);
12954 if (pb->cpu_bufs) {
12955 for (i = 0; i < pb->cpu_cnt; i++) {
12956 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
12961 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
12964 free(pb->cpu_bufs);
12966 if (pb->epoll_fd >= 0)
12967 close(pb->epoll_fd);
12968 free(pb->events);
12982 return ERR_PTR(-ENOMEM);
12984 cpu_buf->pb = pb;
12985 cpu_buf->cpu = cpu;
12986 cpu_buf->map_key = map_key;
12988 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
12989 -1, PERF_FLAG_FD_CLOEXEC);
12990 if (cpu_buf->fd < 0) {
12991 err = -errno;
12997 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
12999 cpu_buf->fd, 0);
13000 if (cpu_buf->base == MAP_FAILED) {
13001 cpu_buf->base = NULL;
13002 err = -errno;
13008 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
13009 err = -errno;
13037 return libbpf_err_ptr(-EINVAL);
13067 return libbpf_err_ptr(-EINVAL);
13070 return libbpf_err_ptr(-EINVAL);
13093 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {
13096 return ERR_PTR(-EINVAL);
13099 /* best-effort sanity checks */
13104 err = -errno;
13106 * -EBADFD, -EFAULT, or -E2BIG on real error
13108 if (err != -EINVAL) {
13119 return ERR_PTR(-EINVAL);
13125 return ERR_PTR(-ENOMEM);
13127 pb->event_cb = p->event_cb;
13128 pb->sample_cb = p->sample_cb;
13129 pb->lost_cb = p->lost_cb;
13130 pb->ctx = p->ctx;
13132 pb->page_size = getpagesize();
13133 pb->mmap_size = pb->page_size * page_cnt;
13134 pb->map_fd = map_fd;
13136 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
13137 if (pb->epoll_fd < 0) {
13138 err = -errno;
13144 if (p->cpu_cnt > 0) {
13145 pb->cpu_cnt = p->cpu_cnt;
13147 pb->cpu_cnt = libbpf_num_possible_cpus();
13148 if (pb->cpu_cnt < 0) {
13149 err = pb->cpu_cnt;
13152 if (map.max_entries && map.max_entries < pb->cpu_cnt)
13153 pb->cpu_cnt = map.max_entries;
13156 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
13157 if (!pb->events) {
13158 err = -ENOMEM;
13162 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
13163 if (!pb->cpu_bufs) {
13164 err = -ENOMEM;
13175 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
13179 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
13180 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
13185 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
13188 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
13194 pb->cpu_bufs[j] = cpu_buf;
13196 err = bpf_map_update_elem(pb->map_fd, &map_key,
13197 &cpu_buf->fd, 0);
13199 err = -errno;
13200 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
13201 cpu, map_key, cpu_buf->fd,
13206 pb->events[j].events = EPOLLIN;
13207 pb->events[j].data.ptr = cpu_buf;
13208 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
13209 &pb->events[j]) < 0) {
13210 err = -errno;
13212 cpu, cpu_buf->fd,
13218 pb->cpu_cnt = j;
13247 struct perf_buffer *pb = cpu_buf->pb;
13251 if (pb->event_cb)
13252 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
13254 switch (e->type) {
13258 if (pb->sample_cb)
13259 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
13265 if (pb->lost_cb)
13266 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
13270 pr_warn("unknown perf sample type %d\n", e->type);
13281 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
13282 pb->page_size, &cpu_buf->buf,
13283 &cpu_buf->buf_size,
13292 return pb->epoll_fd;
13299 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
13301 return -errno;
13304 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
13320 return pb->cpu_cnt;
13332 if (buf_idx >= pb->cpu_cnt)
13333 return libbpf_err(-EINVAL);
13335 cpu_buf = pb->cpu_bufs[buf_idx];
13337 return libbpf_err(-ENOENT);
13339 return cpu_buf->fd;
13346 if (buf_idx >= pb->cpu_cnt)
13347 return libbpf_err(-EINVAL);
13349 cpu_buf = pb->cpu_bufs[buf_idx];
13351 return libbpf_err(-ENOENT);
13353 *buf = cpu_buf->base;
13354 *buf_size = pb->mmap_size;
13363 * - 0 on success;
13364 * - <0 on failure.
13370 if (buf_idx >= pb->cpu_cnt)
13371 return libbpf_err(-EINVAL);
13373 cpu_buf = pb->cpu_bufs[buf_idx];
13375 return libbpf_err(-ENOENT);
13384 for (i = 0; i < pb->cpu_cnt; i++) {
13385 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
13406 return libbpf_err(-EINVAL);
13408 if (prog->obj->loaded)
13409 return libbpf_err(-EINVAL);
13415 prog->attach_prog_fd = attach_prog_fd;
13426 return libbpf_err(-EINVAL);
13429 err = bpf_object__load_vmlinux_btf(prog->obj, true);
13432 err = find_kernel_btf_id(prog->obj, attach_func_name,
13433 prog->expected_attach_type,
13439 prog->attach_btf_id = btf_id;
13440 prog->attach_btf_obj_fd = btf_obj_fd;
13441 prog->attach_prog_fd = attach_prog_fd;
13447 int err = 0, n, len, start, end = -1;
13453 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
13459 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
13462 err = -EINVAL;
13470 err = -EINVAL;
13475 err = -ENOMEM;
13479 memset(tmp + *mask_sz, 0, start - *mask_sz);
13480 memset(tmp + start, 1, end - start + 1);
13486 return -EINVAL;
13502 err = -errno;
13509 err = len ? -errno : -EINVAL;
13515 return -E2BIG;
13556 struct bpf_map **map = map_skel->map;
13557 const char *name = map_skel->name;
13558 void **mmaped = map_skel->mmaped;
13563 return -ESRCH;
13566 /* externs shouldn't be pre-setup from user code */
13567 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
13568 *mmaped = (*map)->mmaped;
13581 struct bpf_program **prog = prog_skel->prog;
13582 const char *name = prog_skel->name;
13587 return -ESRCH;
13599 obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts);
13602 pr_warn("failed to initialize skeleton BPF object '%s': %d\n", s->name, err);
13606 *s->obj = obj;
13607 err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz);
13609 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);
13613 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz);
13615 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);
13633 if (!s->obj)
13634 return libbpf_err(-EINVAL);
13636 btf = bpf_object__btf(s->obj);
13639 bpf_object__name(s->obj));
13640 return libbpf_err(-errno);
13643 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz);
13649 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz);
13655 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
13656 var_skel = (void *)s->vars + var_idx * s->var_skel_sz;
13657 map = *var_skel->map;
13665 return libbpf_err(-EINVAL);
13671 var_type = btf__type_by_id(btf, var->type);
13672 var_name = btf__name_by_offset(btf, var_type->name_off);
13673 if (strcmp(var_name, var_skel->name) == 0) {
13674 *var_skel->addr = map->mmaped + var->offset;
13686 free(s->maps);
13687 free(s->progs);
13688 free(s->vars);
13696 err = bpf_object__load(*s->obj);
13698 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
13702 for (i = 0; i < s->map_cnt; i++) {
13703 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
13704 struct bpf_map *map = *map_skel->map;
13706 if (!map_skel->mmaped)
13709 *map_skel->mmaped = map->mmaped;
13719 for (i = 0; i < s->prog_cnt; i++) {
13720 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;
13721 struct bpf_program *prog = *prog_skel->prog;
13722 struct bpf_link **link = prog_skel->link;
13724 if (!prog->autoload || !prog->autoattach)
13727 /* auto-attaching not supported for this program */
13728 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
13731 /* if user already set the link manually, don't attempt auto-attach */
13735 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
13737 pr_warn("prog '%s': failed to auto-attach: %d\n",
13742 /* It's possible that for some SEC() definitions auto-attach
13747 * auto-attached. But if not, it shouldn't trigger skeleton's
13761 for (i = 0; i < s->prog_cnt; i++) {
13762 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;
13763 struct bpf_link **link = prog_skel->link;
13775 if (s->progs)
13777 if (s->obj)
13778 bpf_object__close(*s->obj);
13779 free(s->maps);
13780 free(s->progs);