Lines Matching +full:mips64el +full:- +full:unknown +full:- +full:linux +full:- +full:gnuabi64
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
29 #include <linux/err.h>
30 #include <linux/kernel.h>
31 #include <linux/bpf.h>
32 #include <linux/btf.h>
33 #include <linux/filter.h>
34 #include <linux/limits.h>
35 #include <linux/perf_event.h>
36 #include <linux/ring_buffer.h>
37 #include <linux/version.h>
71 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
271 if (err != -EPERM || geteuid() != 0) in pr_perm_msg()
288 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", in pr_perm_msg()
304 fd = -1; \
315 /* as of v1.0 libbpf_set_strict_mode() is a no-op */ in libbpf_set_strict_mode()
361 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
380 /* BPF program support non-linear XDP buffer */
400 * linux/filter.h.
418 * program. For the entry-point (main) BPF program, this is always
419 * zero. For a sub-program, this gets reset before each of main BPF
421 * whether sub-program was already appended to the main program, and
433 * entry-point BPF programs this includes the size of main program
434 * itself plus all the used sub-programs, appended at the end
485 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
591 /* BTF fd index to be patched in for insn->off, this is
592 * 0 for vmlinux BTF, index in obj->fd_array for module
692 /* Path to the custom BTF to be used for BPF CO-RE relocations as an
696 /* vmlinux BTF override for CO-RE relocations */
748 zclose(prog->fd); in bpf_program__unload()
750 zfree(&prog->func_info); in bpf_program__unload()
751 zfree(&prog->line_info); in bpf_program__unload()
760 zfree(&prog->name); in bpf_program__exit()
761 zfree(&prog->sec_name); in bpf_program__exit()
762 zfree(&prog->insns); in bpf_program__exit()
763 zfree(&prog->reloc_desc); in bpf_program__exit()
765 prog->nr_reloc = 0; in bpf_program__exit()
766 prog->insns_cnt = 0; in bpf_program__exit()
767 prog->sec_idx = -1; in bpf_program__exit()
772 return BPF_CLASS(insn->code) == BPF_JMP && in insn_is_subprog_call()
773 BPF_OP(insn->code) == BPF_CALL && in insn_is_subprog_call()
774 BPF_SRC(insn->code) == BPF_K && in insn_is_subprog_call()
775 insn->src_reg == BPF_PSEUDO_CALL && in insn_is_subprog_call()
776 insn->dst_reg == 0 && in insn_is_subprog_call()
777 insn->off == 0; in insn_is_subprog_call()
782 return insn->code == (BPF_JMP | BPF_CALL); in is_call_insn()
787 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; in insn_is_pseudo_func()
798 return -EINVAL; in bpf_object__init_prog()
802 prog->obj = obj; in bpf_object__init_prog()
804 prog->sec_idx = sec_idx; in bpf_object__init_prog()
805 prog->sec_insn_off = sec_off / BPF_INSN_SZ; in bpf_object__init_prog()
806 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; in bpf_object__init_prog()
808 prog->insns_cnt = prog->sec_insn_cnt; in bpf_object__init_prog()
810 prog->type = BPF_PROG_TYPE_UNSPEC; in bpf_object__init_prog()
811 prog->fd = -1; in bpf_object__init_prog()
812 prog->exception_cb_idx = -1; in bpf_object__init_prog()
819 prog->autoload = false; in bpf_object__init_prog()
823 prog->autoload = true; in bpf_object__init_prog()
826 prog->autoattach = true; in bpf_object__init_prog()
829 prog->log_level = obj->log_level; in bpf_object__init_prog()
831 prog->sec_name = strdup(sec_name); in bpf_object__init_prog()
832 if (!prog->sec_name) in bpf_object__init_prog()
835 prog->name = strdup(name); in bpf_object__init_prog()
836 if (!prog->name) in bpf_object__init_prog()
839 prog->insns = malloc(insn_data_sz); in bpf_object__init_prog()
840 if (!prog->insns) in bpf_object__init_prog()
842 memcpy(prog->insns, insn_data, insn_data_sz); in bpf_object__init_prog()
848 return -ENOMEM; in bpf_object__init_prog()
855 Elf_Data *symbols = obj->efile.symbols; in bpf_object__add_programs()
857 void *data = sec_data->d_buf; in bpf_object__add_programs()
858 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; in bpf_object__add_programs()
863 progs = obj->programs; in bpf_object__add_programs()
864 nr_progs = obj->nr_programs; in bpf_object__add_programs()
865 nr_syms = symbols->d_size / sizeof(Elf64_Sym); in bpf_object__add_programs()
870 if (sym->st_shndx != sec_idx) in bpf_object__add_programs()
872 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) in bpf_object__add_programs()
875 prog_sz = sym->st_size; in bpf_object__add_programs()
876 sec_off = sym->st_value; in bpf_object__add_programs()
878 name = elf_sym_str(obj, sym->st_name); in bpf_object__add_programs()
882 return -LIBBPF_ERRNO__FORMAT; in bpf_object__add_programs()
888 return -LIBBPF_ERRNO__FORMAT; in bpf_object__add_programs()
891 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { in bpf_object__add_programs()
893 return -ENOTSUP; in bpf_object__add_programs()
902 * In this case the original obj->programs in bpf_object__add_programs()
908 return -ENOMEM; in bpf_object__add_programs()
910 obj->programs = progs; in bpf_object__add_programs()
919 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL) in bpf_object__add_programs()
920 prog->sym_global = true; in bpf_object__add_programs()
927 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN in bpf_object__add_programs()
928 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) in bpf_object__add_programs()
929 prog->mark_btf_static = true; in bpf_object__add_programs()
932 obj->nr_programs = nr_progs; in bpf_object__add_programs()
960 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) in find_member_by_name()
1012 if (kern_data_member->type == kern_type_id) in find_struct_ops_kern_types()
1018 return -EINVAL; in find_struct_ops_kern_types()
1032 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; in bpf_map__is_struct_ops()
1048 st_ops = map->st_ops; in bpf_map__init_kern_struct_ops()
1049 type = st_ops->type; in bpf_map__init_kern_struct_ops()
1050 tname = st_ops->tname; in bpf_map__init_kern_struct_ops()
1059 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); in bpf_map__init_kern_struct_ops()
1061 map->def.value_size = kern_vtype->size; in bpf_map__init_kern_struct_ops()
1062 map->btf_vmlinux_value_type_id = kern_vtype_id; in bpf_map__init_kern_struct_ops()
1064 st_ops->kern_vdata = calloc(1, kern_vtype->size); in bpf_map__init_kern_struct_ops()
1065 if (!st_ops->kern_vdata) in bpf_map__init_kern_struct_ops()
1066 return -ENOMEM; in bpf_map__init_kern_struct_ops()
1068 data = st_ops->data; in bpf_map__init_kern_struct_ops()
1069 kern_data_off = kern_data_member->offset / 8; in bpf_map__init_kern_struct_ops()
1070 kern_data = st_ops->kern_vdata + kern_data_off; in bpf_map__init_kern_struct_ops()
1082 mname = btf__name_by_offset(btf, member->name_off); in bpf_map__init_kern_struct_ops()
1086 map->name, mname); in bpf_map__init_kern_struct_ops()
1087 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1090 kern_member_idx = kern_member - btf_members(kern_type); in bpf_map__init_kern_struct_ops()
1094 map->name, mname); in bpf_map__init_kern_struct_ops()
1095 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1098 moff = member->offset / 8; in bpf_map__init_kern_struct_ops()
1099 kern_moff = kern_member->offset / 8; in bpf_map__init_kern_struct_ops()
1104 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); in bpf_map__init_kern_struct_ops()
1105 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, in bpf_map__init_kern_struct_ops()
1107 if (BTF_INFO_KIND(mtype->info) != in bpf_map__init_kern_struct_ops()
1108 BTF_INFO_KIND(kern_mtype->info)) { in bpf_map__init_kern_struct_ops()
1110 map->name, mname, BTF_INFO_KIND(mtype->info), in bpf_map__init_kern_struct_ops()
1111 BTF_INFO_KIND(kern_mtype->info)); in bpf_map__init_kern_struct_ops()
1112 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1118 prog = st_ops->progs[i]; in bpf_map__init_kern_struct_ops()
1123 kern_mtype->type, in bpf_map__init_kern_struct_ops()
1126 /* mtype->type must be a func_proto which was in bpf_map__init_kern_struct_ops()
1132 map->name, mname); in bpf_map__init_kern_struct_ops()
1133 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1136 prog->attach_btf_id = kern_type_id; in bpf_map__init_kern_struct_ops()
1137 prog->expected_attach_type = kern_member_idx; in bpf_map__init_kern_struct_ops()
1139 st_ops->kern_func_off[i] = kern_data_off + kern_moff; in bpf_map__init_kern_struct_ops()
1142 map->name, mname, prog->name, moff, in bpf_map__init_kern_struct_ops()
1152 map->name, mname, (ssize_t)msize, in bpf_map__init_kern_struct_ops()
1154 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1158 map->name, mname, (unsigned int)msize, in bpf_map__init_kern_struct_ops()
1172 for (i = 0; i < obj->nr_maps; i++) { in bpf_object__init_kern_struct_ops_maps()
1173 map = &obj->maps[i]; in bpf_object__init_kern_struct_ops_maps()
1178 err = bpf_map__init_kern_struct_ops(map, obj->btf, in bpf_object__init_kern_struct_ops_maps()
1179 obj->btf_vmlinux); in bpf_object__init_kern_struct_ops_maps()
1199 if (shndx == -1) in init_struct_ops_maps()
1202 btf = obj->btf; in init_struct_ops_maps()
1208 return -EINVAL; in init_struct_ops_maps()
1214 type = btf__type_by_id(obj->btf, vsi->type); in init_struct_ops_maps()
1215 var_name = btf__name_by_offset(obj->btf, type->name_off); in init_struct_ops_maps()
1217 type_id = btf__resolve_type(obj->btf, vsi->type); in init_struct_ops_maps()
1220 vsi->type, sec_name); in init_struct_ops_maps()
1221 return -EINVAL; in init_struct_ops_maps()
1224 type = btf__type_by_id(obj->btf, type_id); in init_struct_ops_maps()
1225 tname = btf__name_by_offset(obj->btf, type->name_off); in init_struct_ops_maps()
1228 return -ENOTSUP; in init_struct_ops_maps()
1232 return -EINVAL; in init_struct_ops_maps()
1239 map->sec_idx = shndx; in init_struct_ops_maps()
1240 map->sec_offset = vsi->offset; in init_struct_ops_maps()
1241 map->name = strdup(var_name); in init_struct_ops_maps()
1242 if (!map->name) in init_struct_ops_maps()
1243 return -ENOMEM; in init_struct_ops_maps()
1245 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; in init_struct_ops_maps()
1246 map->def.key_size = sizeof(int); in init_struct_ops_maps()
1247 map->def.value_size = type->size; in init_struct_ops_maps()
1248 map->def.max_entries = 1; in init_struct_ops_maps()
1249 map->def.map_flags = map_flags; in init_struct_ops_maps()
1251 map->st_ops = calloc(1, sizeof(*map->st_ops)); in init_struct_ops_maps()
1252 if (!map->st_ops) in init_struct_ops_maps()
1253 return -ENOMEM; in init_struct_ops_maps()
1254 st_ops = map->st_ops; in init_struct_ops_maps()
1255 st_ops->data = malloc(type->size); in init_struct_ops_maps()
1256 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); in init_struct_ops_maps()
1257 st_ops->kern_func_off = malloc(btf_vlen(type) * in init_struct_ops_maps()
1258 sizeof(*st_ops->kern_func_off)); in init_struct_ops_maps()
1259 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) in init_struct_ops_maps()
1260 return -ENOMEM; in init_struct_ops_maps()
1262 if (vsi->offset + type->size > data->d_size) { in init_struct_ops_maps()
1265 return -EINVAL; in init_struct_ops_maps()
1268 memcpy(st_ops->data, in init_struct_ops_maps()
1269 data->d_buf + vsi->offset, in init_struct_ops_maps()
1270 type->size); in init_struct_ops_maps()
1271 st_ops->tname = tname; in init_struct_ops_maps()
1272 st_ops->type = type; in init_struct_ops_maps()
1273 st_ops->type_id = type_id; in init_struct_ops_maps()
1276 tname, type_id, var_name, vsi->offset); in init_struct_ops_maps()
1286 err = init_struct_ops_maps(obj, STRUCT_OPS_SEC, obj->efile.st_ops_shndx, in bpf_object_init_struct_ops()
1287 obj->efile.st_ops_data, 0); in bpf_object_init_struct_ops()
1289 obj->efile.st_ops_link_shndx, in bpf_object_init_struct_ops()
1290 obj->efile.st_ops_link_data, in bpf_object_init_struct_ops()
1306 return ERR_PTR(-ENOMEM); in bpf_object__new()
1309 strcpy(obj->path, path); in bpf_object__new()
1311 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); in bpf_object__new()
1314 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); in bpf_object__new()
1315 end = strchr(obj->name, '.'); in bpf_object__new()
1320 obj->efile.fd = -1; in bpf_object__new()
1327 obj->efile.obj_buf = obj_buf; in bpf_object__new()
1328 obj->efile.obj_buf_sz = obj_buf_sz; in bpf_object__new()
1329 obj->efile.btf_maps_shndx = -1; in bpf_object__new()
1330 obj->efile.st_ops_shndx = -1; in bpf_object__new()
1331 obj->efile.st_ops_link_shndx = -1; in bpf_object__new()
1332 obj->kconfig_map_idx = -1; in bpf_object__new()
1334 obj->kern_version = get_kernel_version(); in bpf_object__new()
1335 obj->loaded = false; in bpf_object__new()
1342 if (!obj->efile.elf) in bpf_object__elf_finish()
1345 elf_end(obj->efile.elf); in bpf_object__elf_finish()
1347 if (obj->efile.shstring) { in bpf_object__elf_finish()
1348 elfio_string_section_accessor_delete(obj->efile.shstring); in bpf_object__elf_finish()
1350 if (obj->efile.strstring) { in bpf_object__elf_finish()
1351 elfio_string_section_accessor_delete(obj->efile.strstring); in bpf_object__elf_finish()
1353 elfio_delete(obj->efile.elf); in bpf_object__elf_finish()
1355 obj->efile.elf = NULL; in bpf_object__elf_finish()
1356 obj->efile.symbols = NULL; in bpf_object__elf_finish()
1357 obj->efile.st_ops_data = NULL; in bpf_object__elf_finish()
1358 obj->efile.st_ops_link_data = NULL; in bpf_object__elf_finish()
1360 zfree(&obj->efile.secs); in bpf_object__elf_finish()
1361 obj->efile.sec_cnt = 0; in bpf_object__elf_finish()
1362 zclose(obj->efile.fd); in bpf_object__elf_finish()
1363 obj->efile.obj_buf = NULL; in bpf_object__elf_finish()
1364 obj->efile.obj_buf_sz = 0; in bpf_object__elf_finish()
1377 if (obj->efile.elf) {
1379 return -LIBBPF_ERRNO__LIBELF;
1382 if (obj->efile.obj_buf_sz > 0) {
1385 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1390 ftruncate(fdm, obj->efile.obj_buf_sz);
1391 write(fdm, (char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1396 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1397 if (obj->efile.fd < 0) {
1400 err = -errno;
1402 pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1406 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1411 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1412 err = -LIBBPF_ERRNO__LIBELF;
1416 obj->efile.elf = elf;
1419 err = -LIBBPF_ERRNO__FORMAT;
1420 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1428 err = -LIBBPF_ERRNO__FORMAT;
1429 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1433 obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1435 obj->efile.ehdr = ehdr = (Elf64_Ehdr*)obj->efile.obj_buf;
1437 if (!obj->efile.ehdr) {
1438 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1439 err = -LIBBPF_ERRNO__FORMAT;
1444 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1446 obj->path, elf_errmsg(-1));
1447 err = -LIBBPF_ERRNO__FORMAT;
1452 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1454 obj->path, elf_errmsg(-1));
1455 err = -LIBBPF_ERRNO__FORMAT;
1459 obj->efile.shstrndx = elfio_get_section_name_str_index(elf);
1462 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1463 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1464 err = -LIBBPF_ERRNO__FORMAT;
1477 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1480 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1485 pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1486 return -LIBBPF_ERRNO__ENDIAN;
1493 pr_warn("invalid license section in %s\n", obj->path);
1494 return -LIBBPF_ERRNO__FORMAT;
1496 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1499 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1500 pr_debug("license of %s is %s\n", obj->path, obj->license);
1510 pr_warn("invalid kver section in %s\n", obj->path);
1511 return -LIBBPF_ERRNO__FORMAT;
1514 obj->kern_version = kver;
1515 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1535 return -EINVAL;
1545 *size = data->d_size;
1549 return -ENOENT;
1554 Elf_Data *symbols = obj->efile.symbols;
1558 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1561 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1564 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1565 ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1568 sname = elf_sym_str(obj, sym->st_name);
1571 return ERR_PTR(-EIO);
1577 return ERR_PTR(-ENOENT);
1585 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1586 sizeof(*obj->maps), obj->nr_maps + 1);
1590 map = &obj->maps[obj->nr_maps++];
1591 map->obj = obj;
1592 map->fd = -1;
1593 map->inner_map_fd = -1;
1594 map->autocreate = true;
1613 if (!map->mmaped)
1614 return -EINVAL;
1619 mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1621 return -errno;
1623 memcpy(mmaped, map->mmaped, min(old_sz, new_sz));
1624 munmap(map->mmaped, old_sz);
1625 map->mmaped = mmaped;
1659 * '.rodata.abracad' kernel and user-visible name.
1668 sfx_len = BPF_OBJ_NAME_LEN - 1;
1674 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1676 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1701 if (!map->btf_value_type_id)
1704 t = btf__type_by_id(obj->btf, map->btf_value_type_id);
1710 vt = btf__type_by_id(obj->btf, vsi->type);
1714 if (btf_var(vt)->linkage != BTF_VAR_STATIC)
1734 map->libbpf_type = type;
1735 map->sec_idx = sec_idx;
1736 map->sec_offset = 0;
1737 map->real_name = strdup(real_name);
1738 map->name = internal_map_name(obj, real_name);
1739 if (!map->real_name || !map->name) {
1740 zfree(&map->real_name);
1741 zfree(&map->name);
1742 return -ENOMEM;
1745 def = &map->def;
1746 def->type = BPF_MAP_TYPE_ARRAY;
1747 def->key_size = sizeof(int);
1748 def->value_size = data_sz;
1749 def->max_entries = 1;
1750 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1757 def->map_flags |= BPF_F_MMAPABLE;
1760 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1762 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
1763 map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1764 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1765 if (map->mmaped == MAP_FAILED) {
1766 err = -errno;
1767 map->mmaped = NULL;
1769 map->name, err);
1770 zfree(&map->real_name);
1771 zfree(&map->name);
1776 memcpy(map->mmaped, data, data_sz);
1778 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1789 * Populate obj->maps with libbpf internal maps.
1791 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
1792 sec_desc = &obj->efile.secs[sec_idx];
1795 if (!sec_desc->data || sec_desc->data->d_size == 0)
1798 switch (sec_desc->sec_type) {
1807 sec_desc->data->d_buf,
1808 sec_desc->data->d_size);
1811 obj->has_rodata = true;
1819 sec_desc->data->d_buf,
1820 sec_desc->data->d_size);
1831 sec_desc->data->d_size);
1849 for (i = 0; i < obj->nr_extern; i++) {
1850 if (strcmp(obj->externs[i].name, name) == 0)
1851 return &obj->externs[i];
1859 switch (ext->kcfg.type) {
1863 ext->name, value);
1864 return -EINVAL;
1884 ext->name, value);
1885 return -EINVAL;
1887 ext->is_set = true;
1896 if (ext->kcfg.type != KCFG_CHAR_ARR) {
1898 ext->name, value);
1899 return -EINVAL;
1903 if (len < 2 || value[len - 1] != '"') {
1905 ext->name, value);
1906 return -EINVAL;
1910 len -= 2;
1911 if (len >= ext->kcfg.sz) {
1913 ext->name, value, len, ext->kcfg.sz - 1);
1914 len = ext->kcfg.sz - 1;
1918 ext->is_set = true;
1930 err = -errno;
1936 return -EINVAL;
1943 int bit_sz = ext->kcfg.sz * 8;
1945 if (ext->kcfg.sz == 8)
1948 /* Validate that value stored in u64 fits in integer of `ext->sz`
1953 * -2^(Y-1) <= X <= 2^(Y-1) - 1
1954 * 0 <= X + 2^(Y-1) <= 2^Y - 1
1955 * 0 <= X + 2^(Y-1) < 2^Y
1957 * For unsigned target integer, check that all the (64 - Y) bits are
1960 if (ext->kcfg.is_signed)
1961 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1969 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
1970 ext->kcfg.type != KCFG_BOOL) {
1972 ext->name, (unsigned long long)value);
1973 return -EINVAL;
1975 if (ext->kcfg.type == KCFG_BOOL && value > 1) {
1977 ext->name, (unsigned long long)value);
1978 return -EINVAL;
1983 ext->name, (unsigned long long)value, ext->kcfg.sz);
1984 return -ERANGE;
1986 switch (ext->kcfg.sz) {
2000 return -EINVAL;
2002 ext->is_set = true;
2021 return -EINVAL;
2026 if (buf[len - 1] == '\n')
2027 buf[len - 1] = '\0';
2033 return -EINVAL;
2037 if (!ext || ext->is_set)
2040 ext_val = data + ext->kcfg.data_off;
2054 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
2057 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
2058 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
2059 return -EINVAL;
2066 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
2078 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
2080 return -EINVAL;
2082 return -ENAMETOOLONG;
2091 return -ENOENT;
2117 err = -errno;
2118 pr_warn("failed to open in-memory Kconfig: %d\n", err);
2125 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
2141 for (i = 0; i < obj->nr_extern; i++) {
2142 ext = &obj->externs[i];
2143 if (ext->type == EXT_KCFG)
2150 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
2152 ".kconfig", obj->efile.symbols_shndx,
2157 obj->kconfig_map_idx = obj->nr_maps - 1;
2172 *res_id = t->type;
2173 t = btf__type_by_id(btf, t->type);
2188 t = skip_mods_and_typedefs(btf, t->type, res_id);
2216 default: return "unknown";
2235 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2236 const char *name = btf__name_by_offset(btf, m->name_off);
2246 arr_t = btf__type_by_id(btf, t->type);
2249 map_name, name, t->type);
2258 *res = arr_info->nelems;
2268 return -EINVAL;
2270 return -ENAMETOOLONG;
2309 const char *name = btf__name_by_offset(btf, m->name_off);
2313 return -EINVAL;
2316 if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2317 return -EINVAL;
2318 map_def->parts |= MAP_DEF_MAP_TYPE;
2320 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2321 return -EINVAL;
2322 map_def->parts |= MAP_DEF_MAX_ENTRIES;
2324 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2325 return -EINVAL;
2326 map_def->parts |= MAP_DEF_MAP_FLAGS;
2328 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2329 return -EINVAL;
2330 map_def->parts |= MAP_DEF_NUMA_NODE;
2335 return -EINVAL;
2336 if (map_def->key_size && map_def->key_size != sz) {
2338 map_name, map_def->key_size, sz);
2339 return -EINVAL;
2341 map_def->key_size = sz;
2342 map_def->parts |= MAP_DEF_KEY_SIZE;
2346 t = btf__type_by_id(btf, m->type);
2349 map_name, m->type);
2350 return -EINVAL;
2355 return -EINVAL;
2357 sz = btf__resolve_size(btf, t->type);
2360 map_name, t->type, (ssize_t)sz);
2363 if (map_def->key_size && map_def->key_size != sz) {
2365 map_name, map_def->key_size, (ssize_t)sz);
2366 return -EINVAL;
2368 map_def->key_size = sz;
2369 map_def->key_type_id = t->type;
2370 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2375 return -EINVAL;
2376 if (map_def->value_size && map_def->value_size != sz) {
2378 map_name, map_def->value_size, sz);
2379 return -EINVAL;
2381 map_def->value_size = sz;
2382 map_def->parts |= MAP_DEF_VALUE_SIZE;
2386 t = btf__type_by_id(btf, m->type);
2389 map_name, m->type);
2390 return -EINVAL;
2395 return -EINVAL;
2397 sz = btf__resolve_size(btf, t->type);
2400 map_name, t->type, (ssize_t)sz);
2403 if (map_def->value_size && map_def->value_size != sz) {
2405 map_name, map_def->value_size, (ssize_t)sz);
2406 return -EINVAL;
2408 map_def->value_size = sz;
2409 map_def->value_type_id = t->type;
2410 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2413 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2414 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2415 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2420 pr_warn("map '%s': multi-level inner maps not supported.\n",
2422 return -ENOTSUP;
2424 if (i != vlen - 1) {
2427 return -EINVAL;
2430 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2432 return -ENOTSUP;
2434 if (map_def->value_size && map_def->value_size != 4) {
2436 map_name, map_def->value_size);
2437 return -EINVAL;
2439 map_def->value_size = 4;
2440 t = btf__type_by_id(btf, m->type);
2443 map_name, desc, m->type);
2444 return -EINVAL;
2446 if (!btf_is_array(t) || btf_array(t)->nelems) {
2447 pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2449 return -EINVAL;
2451 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2455 return -EINVAL;
2457 t = skip_mods_and_typedefs(btf, t->type, NULL);
2460 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2462 return -EINVAL;
2467 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2469 return -EINVAL;
2477 map_def->parts |= MAP_DEF_INNER_MAP;
2483 return -EINVAL;
2486 return -EINVAL;
2490 return -EINVAL;
2492 map_def->pinning = val;
2493 map_def->parts |= MAP_DEF_PINNING;
2498 return -EINVAL;
2499 map_def->map_extra = map_extra;
2500 map_def->parts |= MAP_DEF_MAP_EXTRA;
2503 pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
2504 return -ENOTSUP;
2506 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
2510 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2512 return -EINVAL;
2527 * a power-of-2 multiple of kernel's page size. If user diligently
2534 * user-set size to satisfy both user size request and kernel
2543 * very close to UINT_MAX but is not a power-of-2 multiple of
2551 return map->def.type == BPF_MAP_TYPE_RINGBUF ||
2552 map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2557 map->def.type = def->map_type;
2558 map->def.key_size = def->key_size;
2559 map->def.value_size = def->value_size;
2560 map->def.max_entries = def->max_entries;
2561 map->def.map_flags = def->map_flags;
2562 map->map_extra = def->map_extra;
2564 map->numa_node = def->numa_node;
2565 map->btf_key_type_id = def->key_type_id;
2566 map->btf_value_type_id = def->value_type_id;
2568 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2570 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2572 if (def->parts & MAP_DEF_MAP_TYPE)
2573 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2575 if (def->parts & MAP_DEF_KEY_TYPE)
2577 map->name, def->key_type_id, def->key_size);
2578 else if (def->parts & MAP_DEF_KEY_SIZE)
2579 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2581 if (def->parts & MAP_DEF_VALUE_TYPE)
2583 map->name, def->value_type_id, def->value_size);
2584 else if (def->parts & MAP_DEF_VALUE_SIZE)
2585 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2587 if (def->parts & MAP_DEF_MAX_ENTRIES)
2588 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2589 if (def->parts & MAP_DEF_MAP_FLAGS)
2590 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2591 if (def->parts & MAP_DEF_MAP_EXTRA)
2592 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2593 (unsigned long long)def->map_extra);
2594 if (def->parts & MAP_DEF_PINNING)
2595 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2596 if (def->parts & MAP_DEF_NUMA_NODE)
2597 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2599 if (def->parts & MAP_DEF_INNER_MAP)
2600 pr_debug("map '%s': found inner map definition.\n", map->name);
2609 default: return "unknown";
2628 var = btf__type_by_id(obj->btf, vi->type);
2630 map_name = btf__name_by_offset(obj->btf, var->name_off);
2634 return -EINVAL;
2636 if ((__u64)vi->offset + vi->size > data->d_size) {
2638 return -EINVAL;
2643 return -EINVAL;
2645 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2647 map_name, btf_var_linkage_str(var_extra->linkage));
2648 return -EOPNOTSUPP;
2651 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2655 return -EINVAL;
2657 if (def->size > vi->size) {
2659 return -EINVAL;
2665 map->name = strdup(map_name);
2666 if (!map->name) {
2668 return -ENOMEM;
2670 map->libbpf_type = LIBBPF_MAP_UNSPEC;
2671 map->def.type = BPF_MAP_TYPE_UNSPEC;
2672 map->sec_idx = sec_idx;
2673 map->sec_offset = vi->offset;
2674 map->btf_var_idx = var_idx;
2676 map_name, map->sec_idx, map->sec_offset);
2678 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2687 pr_warn("map '%s': couldn't build pin path.\n", map->name);
2693 map->inner_map = calloc(1, sizeof(*map->inner_map));
2694 if (!map->inner_map)
2695 return -ENOMEM;
2696 map->inner_map->fd = -1;
2697 map->inner_map->sec_idx = sec_idx;
2698 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2699 if (!map->inner_map->name)
2700 return -ENOMEM;
2701 sprintf(map->inner_map->name, "%s.inner", map_name);
2703 fill_map_from_def(map->inner_map, &inner_def);
2725 if (obj->efile.btf_maps_shndx < 0)
2728 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2733 data = elf_sec_data_by_idx(obj, obj->efile.btf_maps_shndx, &realdata);
2737 MAPS_ELF_SEC, obj->path);
2738 return -EINVAL;
2741 nr_types = btf__type_cnt(obj->btf);
2743 t = btf__type_by_id(obj->btf, i);
2746 name = btf__name_by_offset(obj->btf, t->name_off);
2749 obj->efile.btf_maps_sec_btf_id = i;
2756 return -ENOENT;
2762 obj->efile.btf_maps_shndx,
2802 return sh->sh_flags & SHF_EXECINSTR;
2837 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2843 t->size = 1;
2852 name = (char *)btf__name_by_offset(btf, t->name_off);
2860 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2863 m->offset = v->offset * 8;
2864 m->type = v->type;
2866 vt = (void *)btf__type_by_id(btf, v->type);
2867 m->name_off = vt->name_off;
2872 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2873 t->size = sizeof(__u32); /* kernel enforced */
2876 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2879 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2881 /* replace FLOAT with an equally-sized empty STRUCT;
2885 t->name_off = 0;
2886 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
2889 t->name_off = 0;
2890 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
2893 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
2908 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
2910 m->type = enum64_placeholder_id;
2911 m->offset = 0;
2921 return obj->efile.btf_maps_shndx >= 0 ||
2922 obj->efile.st_ops_shndx >= 0 ||
2923 obj->efile.st_ops_link_shndx >= 0 ||
2924 obj->nr_extern > 0;
2929 return obj->efile.st_ops_shndx >= 0 || obj->efile.st_ops_link_shndx >= 0;
2936 int err = -ENOENT;
2939 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2940 err = libbpf_get_error(obj->btf);
2942 obj->btf = NULL;
2946 /* enforce 8-byte pointers for BPF-targeted BTFs */
2947 btf__set_pointer_size(obj->btf, 8);
2953 if (!obj->btf) {
2958 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
2959 err = libbpf_get_error(obj->btf_ext);
2963 obj->btf_ext = NULL;
2968 ext_segs[0] = &obj->btf_ext->func_info;
2969 ext_segs[1] = &obj->btf_ext->line_info;
2970 ext_segs[2] = &obj->btf_ext->core_relo_info;
2981 if (seg->sec_cnt == 0)
2984 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
2985 if (!seg->sec_idxs) {
2986 err = -ENOMEM;
2997 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
3005 pelfio_t elf = obj->efile.elf;
3011 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
3013 seg->sec_idxs[sec_num - 1] = elfio_section_get_index(sec_obj);
3031 return a->offset - b->offset;
3038 const char *sec_name = btf__name_by_offset(btf, t->name_off);
3045 return -ENOENT;
3048 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and
3060 * to be optional. But the STV_HIDDEN handling is non-optional for any
3061 * non-extern DATASEC, so the variable fixup loop below handles both
3062 * functions at the same time, paying the cost of BTF VAR <-> ELF
3065 if (t->size == 0) {
3070 return -ENOENT;
3073 t->size = size;
3083 t_var = btf__type_by_id(btf, vsi->type);
3085 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);
3086 return -EINVAL;
3090 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)
3093 var_name = btf__name_by_offset(btf, t_var->name_off);
3097 return -ENOENT;
3104 return -ENOENT;
3108 vsi->offset = sym->st_value;
3117 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
3118 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)
3119 var->linkage = BTF_VAR_STATIC;
3131 if (!obj->btf)
3134 n = btf__type_cnt(obj->btf);
3136 struct btf_type *t = btf_type_by_id(obj->btf, i);
3144 err = btf_fixup_datasec(obj, obj->btf, t);
3155 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3156 prog->type == BPF_PROG_TYPE_LSM)
3162 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3173 /* CO-RE relocations need kernel BTF, only when btf_custom_path
3176 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
3180 for (i = 0; i < obj->nr_extern; i++) {
3183 ext = &obj->externs[i];
3184 if (ext->type == EXT_KSYM && ext->ksym.type_id)
3189 if (!prog->autoload)
3203 if (obj->btf_vmlinux || obj->gen_loader)
3209 obj->btf_vmlinux = btf__load_vmlinux_btf();
3210 err = libbpf_get_error(obj->btf_vmlinux);
3213 obj->btf_vmlinux = NULL;
3221 struct btf *kern_btf = obj->btf;
3225 if (!obj->btf)
3230 err = -EOPNOTSUPP;
3245 for (i = 0; i < obj->nr_programs; i++) {
3246 struct bpf_program *prog = &obj->programs[i];
3251 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3254 n = btf__type_cnt(obj->btf);
3256 t = btf_type_by_id(obj->btf, j);
3260 name = btf__str_by_offset(obj->btf, t->name_off);
3261 if (strcmp(name, prog->name) != 0)
3264 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3271 for (i = 0; i < obj->nr_programs; i++) {
3272 struct bpf_program *prog = &obj->programs[i];
3277 n = btf__type_cnt(obj->btf);
3283 t = btf_type_by_id(obj->btf, j);
3284 if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1)
3287 name = btf__str_by_offset(obj->btf, t->name_off);
3291 t = btf_type_by_id(obj->btf, t->type);
3294 prog->name);
3295 return -EINVAL;
3297 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)))
3303 if (prog->exception_cb_idx >= 0) {
3304 prog->exception_cb_idx = -1;
3311 prog->name);
3312 return -EINVAL;
3315 for (k = 0; k < obj->nr_programs; k++) {
3316 struct bpf_program *subprog = &obj->programs[k];
3320 if (strcmp(name, subprog->name))
3322 /* Enforce non-hidden, as from verifier point of
3326 if (!subprog->sym_global || subprog->mark_btf_static) {
3327 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n",
3328 prog->name, subprog->name);
3329 return -EINVAL;
3332 if (prog->exception_cb_idx >= 0) {
3334 prog->name, subprog->name);
3335 return -EINVAL;
3337 prog->exception_cb_idx = k;
3341 if (prog->exception_cb_idx >= 0)
3343 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name);
3344 return -ENOENT;
3355 raw_data = btf__raw_data(obj->btf, &sz);
3361 /* enforce 8-byte pointers for BPF-targeted BTFs */
3362 btf__set_pointer_size(obj->btf, 8);
3368 if (obj->gen_loader) {
3373 return -ENOMEM;
3374 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3376 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3381 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3382 obj->log_level ? 1 : 0);
3387 btf__set_fd(obj->btf, btf__fd(kern_btf));
3388 btf__set_fd(kern_btf, -1);
3408 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3410 name = elfio_string_get_string(obj->efile.strstring, off);
3414 off, obj->path, elf_errmsg(-1));
3425 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3427 name = elfio_string_get_string(obj->efile.shstring, off);
3432 off, obj->path, elf_errmsg(-1));
3444 scn = elf_getscn(obj->efile.elf, idx);
3447 idx, obj->path, elf_errmsg(-1));
3456 Elf *elf = obj->efile.elf;
3482 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3501 name = elf_sec_str(obj, sh->sh_name);
3504 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3513 psection_t psection = elfio_get_section_by_index(obj->efile.elf, idx);
3515 sheader->sh_name = elfio_section_get_name_string_offset(psection);
3516 sheader->sh_type = elfio_section_get_type(psection);
3517 sheader->sh_flags = elfio_section_get_flags(psection);
3518 sheader->sh_addr = elfio_section_get_address(psection);
3519 sheader->sh_offset = elfio_section_get_offset(psection);
3520 sheader->sh_size = elfio_section_get_size(psection);
3521 sheader->sh_link = elfio_section_get_link(psection);
3522 sheader->sh_info = elfio_section_get_info(psection);
3523 sheader->sh_addralign = elfio_section_get_addr_align(psection);
3524 sheader->sh_entsize = elfio_section_get_entry_size(psection);
3539 idx, obj->path, elf_errmsg(-1));
3559 obj->path, elf_errmsg(-1));
3568 pelfio_t elf = obj->efile.elf;
3570 data->d_buf = (void*)elfio_section_get_data(psection_name);
3571 data->d_size = elfio_section_get_size(psection_name);
3578 pelfio_t elf = obj->efile.elf;
3580 data->d_buf = (void*)elfio_section_get_data(psection_index);
3581 data->d_size = elfio_section_get_size(psection_index);
3589 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3592 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3597 if (idx >= data->d_size / sizeof(Elf64_Rel))
3600 return (Elf64_Rel *)data->d_buf + idx;
3612 if (hdr->sh_type == SHT_STRTAB)
3616 if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3620 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3629 name += sizeof(".rel") - 1;
3648 if (a->sec_idx != b->sec_idx)
3649 return a->sec_idx < b->sec_idx ? -1 : 1;
3652 return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3659 Elf *elf = obj->efile.elf;
3661 pelfio_t elf = obj->efile.elf;
3677 /* ELF section indices are 0-based, but sec #0 is special "invalid"
3683 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3685 obj->path, elf_errmsg(-1));
3686 return -LIBBPF_ERRNO__FORMAT;
3689 obj->efile.sec_cnt = elfio_get_sections_num(elf);
3691 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3692 if (!obj->efile.secs)
3693 return -ENOMEM;
3709 return -LIBBPF_ERRNO__FORMAT;
3711 if (sh->sh_type == SHT_SYMTAB) {
3712 if (obj->efile.symbols) {
3713 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3714 return -LIBBPF_ERRNO__FORMAT;
3722 return -LIBBPF_ERRNO__FORMAT;
3728 obj->efile.symbols = data;
3730 obj->efile.realsymbols.d_buf = data->d_buf;
3731 obj->efile.realsymbols.d_size = data->d_size;
3732 obj->efile.symbols = &(obj->efile.realsymbols);
3736 obj->efile.symbols_shndx = idx;
3738 obj->efile.symbols_shndx = i;
3740 obj->efile.strtabidx = sh->sh_link;
3748 psection_t psection = elfio_get_section_by_index(elf, obj->efile.strtabidx);
3750 return -LIBBPF_ERRNO__FORMAT;
3753 psection = elfio_get_section_by_index(elf, obj->efile.shstrndx);
3755 return -LIBBPF_ERRNO__FORMAT;
3759 return -LIBBPF_ERRNO__FORMAT;
3760 obj->efile.strstring = strstring;
3761 obj->efile.shstring = shstring;
3764 if (!obj->efile.symbols) {
3766 obj->path);
3767 return -ENOENT;
3784 sec_desc = &obj->efile.secs[idx];
3793 return -LIBBPF_ERRNO__FORMAT;
3795 name = elf_sec_str(obj, sh->sh_name);
3797 return -LIBBPF_ERRNO__FORMAT;
3805 data = elf_sec_data_by_idx(obj, i, &sec_desc->realdata);
3808 return -LIBBPF_ERRNO__FORMAT;
3811 idx, name, (unsigned long)data->d_size,
3812 (int)sh->sh_link, (unsigned long)sh->sh_flags,
3813 (int)sh->sh_type);
3816 err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3820 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3825 return -ENOTSUP;
3827 obj->efile.btf_maps_shndx = idx;
3829 if (sh->sh_type != SHT_PROGBITS)
3830 return -LIBBPF_ERRNO__FORMAT;
3833 if (sh->sh_type != SHT_PROGBITS)
3834 return -LIBBPF_ERRNO__FORMAT;
3836 } else if (sh->sh_type == SHT_SYMTAB) {
3838 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
3839 if (sh->sh_flags & SHF_EXECINSTR) {
3841 obj->efile.text_shndx = idx;
3847 sec_desc->sec_type = SEC_DATA;
3849 sec_desc->shdr = sh;
3850 sec_desc->data = data;
3852 sec_desc->psection = ptmpsection;
3853 sec_desc->realdata.d_buf = data->d_buf;
3854 sec_desc->realdata.d_size = data->d_size;
3855 sec_desc->data = &(sec_desc->realdata);
3859 sec_desc->sec_type = SEC_RODATA;
3861 sec_desc->shdr = sh;
3862 sec_desc->data = data;
3864 sec_desc->psection = ptmpsection;
3865 sec_desc->realdata.d_buf = data->d_buf;
3866 sec_desc->realdata.d_size = data->d_size;
3867 sec_desc->data = &(sec_desc->realdata);
3872 obj->efile.st_ops_data = data;
3874 obj->efile.realst_ops_data.d_buf = data->d_buf;
3875 obj->efile.realst_ops_data.d_size = data->d_size;
3876 obj->efile.st_ops_data = &(obj->efile.realst_ops_data);
3878 obj->efile.st_ops_shndx = idx;
3881 obj->efile.st_ops_link_data = data;
3883 obj->efile.realst_ops_link_data.d_buf = data->d_buf;
3884 obj->efile.realst_ops_link_data.d_size = data->d_size;
3885 obj->efile.st_ops_link_data = &(obj->efile.realst_ops_link_data);
3887 obj->efile.st_ops_link_shndx = idx;
3892 } else if (sh->sh_type == SHT_REL) {
3893 int targ_sec_idx = sh->sh_info; /* points to other section */
3895 if (sh->sh_entsize != sizeof(Elf64_Rel) ||
3896 targ_sec_idx >= obj->efile.sec_cnt)
3897 return -LIBBPF_ERRNO__FORMAT;
3916 sec_desc->sec_type = SEC_RELO;
3918 sec_desc->shdr = sh;
3920 sec_desc->psection = ptmpsection;
3922 sec_desc->data = data;
3923 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||
3925 sec_desc->sec_type = SEC_BSS;
3927 sec_desc->shdr = sh;
3929 sec_desc->psection = ptmpsection;
3931 sec_desc->data = data;
3934 (size_t)sh->sh_size);
3938 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3939 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3940 return -LIBBPF_ERRNO__FORMAT;
3943 /* sort BPF programs by section name and in-section instruction offset
3946 if (obj->nr_programs)
3947 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3954 int bind = ELF64_ST_BIND(sym->st_info);
3956 return sym->st_shndx == SHN_UNDEF &&
3958 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
3963 int bind = ELF64_ST_BIND(sym->st_info);
3964 int type = ELF64_ST_TYPE(sym->st_info);
3967 if (sym->st_shndx != text_shndx)
3985 return -ESRCH;
3994 tname = btf__name_by_offset(btf, t->name_off);
3999 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
4000 return -EINVAL;
4003 return -EINVAL;
4008 return -ENOENT;
4017 return -ESRCH;
4028 if (vs->type == ext_btf_id)
4033 return -ENOENT;
4043 name = btf__name_by_offset(btf, t->name_off);
4052 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
4055 if (t->size == 1)
4057 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
4062 if (t->size != 4)
4072 if (btf_array(t)->nelems == 0)
4074 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
4087 if (a->type != b->type)
4088 return a->type < b->type ? -1 : 1;
4090 if (a->type == EXT_KCFG) {
4092 if (a->kcfg.align != b->kcfg.align)
4093 return a->kcfg.align > b->kcfg.align ? -1 : 1;
4095 if (a->kcfg.sz != b->kcfg.sz)
4096 return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
4100 return strcmp(a->name, b->name);
4138 vt = btf__type_by_id(btf, vs->type);
4172 if (!obj->efile.symbols)
4176 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
4180 sh = elf_sec_hdr_by_idx(obj, obj->efile.symbols_shndx, sh);
4183 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
4184 return -LIBBPF_ERRNO__FORMAT;
4186 dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
4190 n = sh->sh_size / sh->sh_entsize;
4197 return -LIBBPF_ERRNO__FORMAT;
4200 ext_name = elf_sym_str(obj, sym->st_name);
4204 ext = obj->externs;
4205 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
4207 return -ENOMEM;
4208 obj->externs = ext;
4209 ext = &ext[obj->nr_extern];
4211 obj->nr_extern++;
4213 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
4214 if (ext->btf_id <= 0) {
4216 ext_name, ext->btf_id);
4217 return ext->btf_id;
4219 t = btf__type_by_id(obj->btf, ext->btf_id);
4220 ext->name = btf__name_by_offset(obj->btf, t->name_off);
4221 ext->sym_idx = i;
4222 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
4224 ext_essent_len = bpf_core_essential_name_len(ext->name);
4225 ext->essent_name = NULL;
4226 if (ext_essent_len != strlen(ext->name)) {
4227 ext->essent_name = strndup(ext->name, ext_essent_len);
4228 if (!ext->essent_name)
4229 return -ENOMEM;
4232 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
4233 if (ext->sec_btf_id <= 0) {
4235 ext_name, ext->btf_id, ext->sec_btf_id);
4236 return ext->sec_btf_id;
4238 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
4239 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
4244 ext->name, KCONFIG_SEC);
4245 return -ENOTSUP;
4248 ext->type = EXT_KCFG;
4249 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
4250 if (ext->kcfg.sz <= 0) {
4252 ext_name, ext->kcfg.sz);
4253 return ext->kcfg.sz;
4255 ext->kcfg.align = btf__align_of(obj->btf, t->type);
4256 if (ext->kcfg.align <= 0) {
4258 ext_name, ext->kcfg.align);
4259 return -EINVAL;
4261 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
4262 &ext->kcfg.is_signed);
4263 if (ext->kcfg.type == KCFG_UNKNOWN) {
4265 return -ENOTSUP;
4269 ext->type = EXT_KSYM;
4270 skip_mods_and_typedefs(obj->btf, t->type,
4271 &ext->ksym.type_id);
4274 return -ENOTSUP;
4277 pr_debug("collected %d externs total\n", obj->nr_extern);
4279 if (!obj->nr_extern)
4283 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
4287 * pretending that each extern is a 8-byte variable
4290 /* find existing 4-byte integer type in BTF to use for fake
4293 int int_btf_id = find_int_btf_id(obj->btf);
4295 * will be used to replace the vs->type and
4301 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
4302 for (i = 0; i < obj->nr_extern; i++) {
4303 ext = &obj->externs[i];
4304 if (ext->type != EXT_KSYM)
4307 i, ext->sym_idx, ext->name);
4316 vt = (void *)btf__type_by_id(obj->btf, vs->type);
4317 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
4322 return -ESRCH;
4329 func_proto = btf__type_by_id(obj->btf,
4330 vt->type);
4338 dummy_var->name_off;
4339 vs->type = dummy_var_btf_id;
4340 vt->info &= ~0xffff;
4341 vt->info |= BTF_FUNC_GLOBAL;
4343 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4344 vt->type = int_btf_id;
4346 vs->offset = off;
4347 vs->size = sizeof(int);
4349 sec->size = off;
4356 for (i = 0; i < obj->nr_extern; i++) {
4357 ext = &obj->externs[i];
4358 if (ext->type != EXT_KCFG)
4361 ext->kcfg.data_off = roundup(off, ext->kcfg.align);
4362 off = ext->kcfg.data_off + ext->kcfg.sz;
4364 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
4366 sec->size = off;
4371 t = btf__type_by_id(obj->btf, vs->type);
4372 ext_name = btf__name_by_offset(obj->btf, t->name_off);
4377 return -ESRCH;
4379 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4380 vs->offset = ext->kcfg.data_off;
4388 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
4400 if (!strcmp(prog->name, name))
4409 switch (obj->efile.secs[shndx].sec_type) {
4422 return shndx == obj->efile.btf_maps_shndx;
4428 if (shndx == obj->efile.symbols_shndx)
4431 switch (obj->efile.secs[shndx].sec_type) {
4448 struct bpf_insn *insn = &prog->insns[insn_idx];
4449 size_t map_idx, nr_maps = prog->obj->nr_maps;
4450 struct bpf_object *obj = prog->obj;
4451 __u32 shdr_idx = sym->st_shndx;
4458 prog->name, sym_name, insn_idx, insn->code);
4459 return -LIBBPF_ERRNO__RELOC;
4463 int sym_idx = ELF64_R_SYM(rel->r_info);
4464 int i, n = obj->nr_extern;
4468 ext = &obj->externs[i];
4469 if (ext->sym_idx == sym_idx)
4474 prog->name, sym_name, sym_idx);
4475 return -LIBBPF_ERRNO__RELOC;
4478 prog->name, i, ext->name, ext->sym_idx, insn_idx);
4479 if (insn->code == (BPF_JMP | BPF_CALL))
4480 reloc_desc->type = RELO_EXTERN_CALL;
4482 reloc_desc->type = RELO_EXTERN_LD64;
4483 reloc_desc->insn_idx = insn_idx;
4484 reloc_desc->ext_idx = i;
4488 /* sub-program call relocation */
4490 if (insn->src_reg != BPF_PSEUDO_CALL) {
4491 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4492 return -LIBBPF_ERRNO__RELOC;
4495 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
4502 prog->name, sym_name, sym_sec_name);
4503 return -LIBBPF_ERRNO__RELOC;
4505 if (sym->st_value % BPF_INSN_SZ) {
4507 prog->name, sym_name, (size_t)sym->st_value);
4508 return -LIBBPF_ERRNO__RELOC;
4510 reloc_desc->type = RELO_CALL;
4511 reloc_desc->insn_idx = insn_idx;
4512 reloc_desc->sym_off = sym->st_value;
4518 prog->name, sym_name, shdr_idx);
4519 return -LIBBPF_ERRNO__RELOC;
4523 if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4524 /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4525 * local_func: sym->st_value = 0, insn->imm = offset in the section.
4527 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4529 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4530 return -LIBBPF_ERRNO__RELOC;
4533 reloc_desc->type = RELO_SUBPROG_ADDR;
4534 reloc_desc->insn_idx = insn_idx;
4535 reloc_desc->sym_off = sym->st_value;
4549 prog->name, sym_name, sym_sec_name);
4550 return -LIBBPF_ERRNO__RELOC;
4553 map = &obj->maps[map_idx];
4554 if (map->libbpf_type != type ||
4555 map->sec_idx != sym->st_shndx ||
4556 map->sec_offset != sym->st_value)
4559 prog->name, map_idx, map->name, map->sec_idx,
4560 map->sec_offset, insn_idx);
4565 prog->name, sym_sec_name, (size_t)sym->st_value);
4566 return -LIBBPF_ERRNO__RELOC;
4568 reloc_desc->type = RELO_LD64;
4569 reloc_desc->insn_idx = insn_idx;
4570 reloc_desc->map_idx = map_idx;
4571 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4578 prog->name, sym_sec_name);
4579 return -LIBBPF_ERRNO__RELOC;
4582 map = &obj->maps[map_idx];
4583 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4586 prog->name, map_idx, map->name, map->sec_idx,
4587 map->sec_offset, insn_idx);
4592 prog->name, sym_sec_name);
4593 return -LIBBPF_ERRNO__RELOC;
4596 reloc_desc->type = RELO_DATA;
4597 reloc_desc->insn_idx = insn_idx;
4598 reloc_desc->map_idx = map_idx;
4599 reloc_desc->sym_off = sym->st_value;
4605 return insn_idx >= prog->sec_insn_off &&
4606 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4612 int l = 0, r = obj->nr_programs - 1, m;
4615 if (!obj->nr_programs)
4619 m = l + (r - l + 1) / 2;
4620 prog = &obj->programs[m];
4622 if (prog->sec_idx < sec_idx ||
4623 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4626 r = m - 1;
4631 prog = &obj->programs[l];
4632 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4641 size_t sec_idx = shdr->sh_info, sym_idx;
4654 if (sec_idx >= obj->efile.sec_cnt)
4655 return -EINVAL;
4661 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4664 return -EINVAL;
4669 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4672 return -EINVAL;
4677 nrels = shdr->sh_size / shdr->sh_entsize;
4683 return -LIBBPF_ERRNO__FORMAT;
4686 sym_idx = ELF64_R_SYM(rel->r_info);
4691 return -LIBBPF_ERRNO__FORMAT;
4694 if (sym->st_shndx >= obj->efile.sec_cnt) {
4696 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4697 return -LIBBPF_ERRNO__FORMAT;
4700 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4702 relo_sec_name, (size_t)rel->r_offset, i);
4703 return -LIBBPF_ERRNO__FORMAT;
4706 insn_idx = rel->r_offset / BPF_INSN_SZ;
4713 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4715 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4717 sym_name = elf_sec_name_by_idx(obj, sym->st_shndx);
4720 sym_name = elf_sym_str(obj, sym->st_name);
4733 relos = libbpf_reallocarray(prog->reloc_desc,
4734 prog->nr_reloc + 1, sizeof(*relos));
4736 return -ENOMEM;
4737 prog->reloc_desc = relos;
4740 insn_idx -= prog->sec_insn_off;
4741 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4746 prog->nr_reloc++;
4755 if (!obj->btf)
4756 return -ENOENT;
4758 /* if it's BTF-defined map, we don't need to search for type IDs.
4762 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4770 return -ENOENT;
4772 id = btf__find_by_name(obj->btf, map->real_name);
4776 map->btf_key_type_id = 0;
4777 map->btf_value_type_id = id;
4793 err = -errno;
4801 info->type = val;
4803 info->key_size = val;
4805 info->value_size = val;
4807 info->max_entries = val;
4809 info->map_flags = val;
4819 return map->autocreate;
4824 if (map->obj->loaded)
4825 return libbpf_err(-EBUSY);
4827 map->autocreate = autocreate;
4846 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
4847 new_name = strdup(map->name);
4852 return libbpf_err(-errno);
4861 err = -errno;
4865 err = zclose(map->fd);
4867 err = -errno;
4870 free(map->name);
4872 map->fd = new_fd;
4873 map->name = new_name;
4874 map->def.type = info.type;
4875 map->def.key_size = info.key_size;
4876 map->def.value_size = info.value_size;
4877 map->def.max_entries = info.max_entries;
4878 map->def.map_flags = info.map_flags;
4879 map->btf_key_type_id = info.btf_key_type_id;
4880 map->btf_value_type_id = info.btf_value_type_id;
4881 map->reused = true;
4882 map->map_extra = info.map_extra;
4895 return map->def.max_entries;
4900 if (!bpf_map_type__is_map_in_map(map->def.type))
4903 return map->inner_map;
4908 if (map->obj->loaded)
4909 return libbpf_err(-EBUSY);
4911 map->def.max_entries = max_entries;
4913 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
4915 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
4930 if (obj->gen_loader)
4948 return -ret;
4997 ret = -errno;
5000 __func__, cp, -ret);
5100 BTF_TYPE_DECL_TAG_ENC(1, 2, -1),
5142 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
5154 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */
5177 ret = -errno;
5180 __func__, cp, -ret);
5238 return -errno;
5243 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL);
5244 err = -errno; /* close() can clobber errno */
5250 return link_fd < 0 && err == -EBADF;
5269 return -errno;
5271 /* Creating uprobe in '/' binary should fail with -EBADF. */
5276 link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts);
5277 err = -errno; /* close() can clobber errno */
5279 if (link_fd >= 0 || err != -EBADF) {
5285 /* Initial multi-uprobe support in kernel didn't handle PID filtering
5288 * we'll pretend multi-uprobes are not supported, if not.
5289 * Multi-uprobes are used in USDT attachment logic, and we need to be
5290 * conservative here, because multi-uprobe selection happens early at
5292 * attachment time, at which point it's too late to undo multi-uprobe
5295 * Creating uprobe with pid == -1 for (invalid) '/' binary will fail
5296 * early with -EINVAL on kernels with fixed PID filtering logic;
5297 * otherwise -ESRCH would be returned if passed correct binary path
5298 * (but we'll just get -BADF, of course).
5300 link_opts.uprobe_multi.pid = -1; /* invalid PID */
5305 link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts);
5306 err = -errno; /* close() can clobber errno */
5312 return link_fd < 0 && err == -EINVAL;
5400 "memcg-based memory accounting", probe_memcg_account,
5412 "BPF multi-uprobe link support", probe_uprobe_multi_link,
5421 if (obj && obj->gen_loader)
5427 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
5428 ret = feat->probe();
5430 WRITE_ONCE(feat->res, FEAT_SUPPORTED);
5432 WRITE_ONCE(feat->res, FEAT_MISSING);
5434 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
5435 WRITE_ONCE(feat->res, FEAT_MISSING);
5439 return READ_ONCE(feat->res) == FEAT_SUPPORTED;
5459 return (map_info.type == map->def.type &&
5460 map_info.key_size == map->def.key_size &&
5461 map_info.value_size == map->def.value_size &&
5462 map_info.max_entries == map->def.max_entries &&
5463 map_info.map_flags == map->def.map_flags &&
5464 map_info.map_extra == map->map_extra);
5473 pin_fd = bpf_obj_get(map->pin_path);
5475 err = -errno;
5476 if (err == -ENOENT) {
5478 map->pin_path);
5482 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
5484 map->pin_path, cp);
5490 map->pin_path);
5492 return -EINVAL;
5500 map->pinned = true;
5501 pr_debug("reused pinned map at '%s'\n", map->pin_path);
5509 enum libbpf_map_type map_type = map->libbpf_type;
5513 if (obj->gen_loader) {
5514 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
5515 map->mmaped, map->def.value_size);
5517 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
5520 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
5522 err = -errno;
5525 map->name, cp);
5529 /* Freeze .rodata and .kconfig map as read-only from syscall side. */
5531 err = bpf_map_freeze(map->fd);
5533 err = -errno;
5535 pr_warn("Error freezing map(%s) as read-only: %s\n",
5536 map->name, cp);
5548 struct bpf_map_def *def = &map->def;
5553 map_name = map->name;
5554 create_attr.map_ifindex = map->map_ifindex;
5555 create_attr.map_flags = def->map_flags;
5556 create_attr.numa_node = map->numa_node;
5557 create_attr.map_extra = map->map_extra;
5560 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
5562 if (obj->btf && btf__fd(obj->btf) >= 0) {
5563 create_attr.btf_fd = btf__fd(obj->btf);
5564 create_attr.btf_key_type_id = map->btf_key_type_id;
5565 create_attr.btf_value_type_id = map->btf_value_type_id;
5568 if (bpf_map_type__is_map_in_map(def->type)) {
5569 if (map->inner_map) {
5570 err = bpf_object__create_map(obj, map->inner_map, true);
5573 map->name, err);
5576 map->inner_map_fd = bpf_map__fd(map->inner_map);
5578 if (map->inner_map_fd >= 0)
5579 create_attr.inner_map_fd = map->inner_map_fd;
5582 switch (def->type) {
5599 map->btf_key_type_id = 0;
5600 map->btf_value_type_id = 0;
5606 if (obj->gen_loader) {
5607 bpf_gen__map_create(obj->gen_loader, def->type, map_name,
5608 def->key_size, def->value_size, def->max_entries,
5609 &create_attr, is_inner ? -1 : map - obj->maps);
5611 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
5613 map->fd = 0;
5615 map->fd = bpf_map_create(def->type, map_name,
5616 def->key_size, def->value_size,
5617 def->max_entries, &create_attr);
5619 if (map->fd < 0 && (create_attr.btf_key_type_id ||
5623 err = -errno;
5626 map->name, cp, err);
5630 map->btf_key_type_id = 0;
5631 map->btf_value_type_id = 0;
5632 map->fd = bpf_map_create(def->type, map_name,
5633 def->key_size, def->value_size,
5634 def->max_entries, &create_attr);
5637 err = map->fd < 0 ? -errno : 0;
5639 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
5640 if (obj->gen_loader)
5641 map->inner_map->fd = -1;
5642 bpf_map__destroy(map->inner_map);
5643 zfree(&map->inner_map);
5655 for (i = 0; i < map->init_slots_sz; i++) {
5656 if (!map->init_slots[i])
5659 targ_map = map->init_slots[i];
5662 if (obj->gen_loader) {
5663 bpf_gen__populate_outer_map(obj->gen_loader,
5664 map - obj->maps, i,
5665 targ_map - obj->maps);
5667 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5670 err = -errno;
5672 map->name, i, targ_map->name, fd, err);
5676 map->name, i, targ_map->name, fd);
5679 zfree(&map->init_slots);
5680 map->init_slots_sz = 0;
5691 if (obj->gen_loader)
5692 return -ENOTSUP;
5694 for (i = 0; i < map->init_slots_sz; i++) {
5695 if (!map->init_slots[i])
5698 targ_prog = map->init_slots[i];
5701 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5703 err = -errno;
5705 map->name, i, targ_prog->name, fd, err);
5709 map->name, i, targ_prog->name, fd);
5712 zfree(&map->init_slots);
5713 map->init_slots_sz = 0;
5723 for (i = 0; i < obj->nr_maps; i++) {
5724 map = &obj->maps[i];
5726 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
5731 zclose(map->fd);
5740 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
5746 map->name, nr_cpus);
5749 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
5750 map->def.max_entries = nr_cpus;
5765 for (i = 0; i < obj->nr_maps; i++) {
5766 map = &obj->maps[i];
5770 * loading, if we detect that at least one of the to-be-loaded
5775 * but also it allows to have CO-RE applications that use
5777 * If those global variable-using programs are not loaded at
5783 map->autocreate = false;
5785 if (!map->autocreate) {
5786 pr_debug("map '%s': skipped auto-creating...\n", map->name);
5796 if (map->pin_path) {
5800 map->name);
5803 if (retried && map->fd < 0) {
5805 map->name);
5806 err = -ENOENT;
5811 if (map->fd >= 0) {
5813 map->name, map->fd);
5820 map->name, map->fd);
5825 zclose(map->fd);
5830 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
5833 zclose(map->fd);
5839 if (map->pin_path && !map->pinned) {
5842 zclose(map->fd);
5843 if (!retried && err == -EEXIST) {
5847 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
5848 map->name, map->pin_path, err);
5858 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
5861 zclose(obj->maps[j].fd);
5875 * underscore is ignored by BPF CO-RE relocation during relocation matching.
5882 for (i = n - 5; i >= 0; i--) {
5894 free(cands->cands);
5911 local_t = btf__type_by_id(local_cand->btf, local_cand->id);
5912 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
5920 targ_name = btf__name_by_offset(targ_btf, t->name_off);
5931 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
5932 local_cand->id, btf_kind_str(local_t),
5935 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
5936 sizeof(*cands->cands));
5938 return -ENOMEM;
5940 cand = &new_cands[cands->len];
5941 cand->btf = targ_btf;
5942 cand->id = i;
5944 cands->cands = new_cands;
5945 cands->len++;
5959 if (obj->btf_modules_loaded)
5962 if (obj->gen_loader)
5966 obj->btf_modules_loaded = true;
5981 err = -errno;
5990 err = -errno;
6002 err = -errno;
6007 /* ignore non-module BTFs */
6013 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
6021 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
6022 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
6026 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
6028 mod_btf->btf = btf;
6029 mod_btf->id = id;
6030 mod_btf->fd = fd;
6031 mod_btf->name = strdup(name);
6032 if (!mod_btf->name) {
6033 err = -ENOMEM;
6061 return ERR_PTR(-EINVAL);
6063 local_name = btf__name_by_offset(local_btf, local_t->name_off);
6065 return ERR_PTR(-EINVAL);
6070 return ERR_PTR(-ENOMEM);
6073 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
6079 if (cands->len)
6083 if (obj->btf_vmlinux_override)
6091 for (i = 0; i < obj->btf_module_cnt; i++) {
6093 obj->btf_modules[i].btf,
6094 obj->btf_modules[i].name,
6095 btf__type_cnt(obj->btf_vmlinux),
6108 * type-based CO-RE relocations and follow slightly different rules than
6109 * field-based relocations. This function assumes that root types were already
6110 * checked for name match. Beyond that initial root-level name check, names
6112 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
6115 * - for ENUMs, the size is ignored;
6116 * - for INT, size and signedness are ignored;
6117 * - for ARRAY, dimensionality is ignored, element types are checked for
6119 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
6120 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
6121 * - FUNC_PROTOs are compatible if they have compatible signature: same
6124 * more experience with using BPF CO-RE relocations.
6153 relos = libbpf_reallocarray(prog->reloc_desc,
6154 prog->nr_reloc + 1, sizeof(*relos));
6156 return -ENOMEM;
6157 relo = &relos[prog->nr_reloc];
6158 relo->type = RELO_CORE;
6159 relo->insn_idx = insn_idx;
6160 relo->core_relo = core_relo;
6161 prog->reloc_desc = relos;
6162 prog->nr_reloc++;
6171 for (i = 0; i < prog->nr_reloc; i++) {
6172 relo = &prog->reloc_desc[i];
6173 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)
6176 return relo->core_relo;
6191 const char *prog_name = prog->name;
6194 __u32 local_id = relo->type_id;
6199 return -EINVAL;
6201 local_name = btf__name_by_offset(local_btf, local_type->name_off);
6203 return -EINVAL;
6205 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
6207 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
6239 if (obj->btf_ext->core_relo_info.len == 0)
6243 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
6244 err = libbpf_get_error(obj->btf_vmlinux_override);
6257 seg = &obj->btf_ext->core_relo_info;
6260 sec_idx = seg->sec_idxs[sec_num];
6263 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6265 err = -EINVAL;
6269 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);
6272 if (rec->insn_off % BPF_INSN_SZ)
6273 return -EINVAL;
6274 insn_idx = rec->insn_off / BPF_INSN_SZ;
6281 * This is similar to what x86-64 linker does for relocations.
6285 …pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subpro…
6289 /* no need to apply CO-RE relocation if the program is
6292 if (!prog->autoload)
6296 * program's frame of reference; (sub-)program code is not yet
6297 * relocated, so it's enough to just subtract in-section offset
6299 insn_idx = insn_idx - prog->sec_insn_off;
6300 if (insn_idx >= prog->insns_cnt)
6301 return -EINVAL;
6302 insn = &prog->insns[insn_idx];
6307 prog->name, i, err);
6311 if (prog->obj->gen_loader)
6314 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
6317 prog->name, i, err);
6321 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
6324 prog->name, i, insn_idx, err);
6331 /* obj->btf_vmlinux and module BTFs are freed after object load */
6332 btf__free(obj->btf_vmlinux_override);
6333 obj->btf_vmlinux_override = NULL;
6337 bpf_core_free_cands(entry->pvalue);
6355 prog->name, relo_idx, insn_idx, map_idx, map->name);
6359 insn->code = BPF_JMP | BPF_CALL;
6360 insn->dst_reg = 0;
6361 insn->src_reg = 0;
6362 insn->off = 0;
6365 * invalid func unknown#2001000123
6366 * where lower 123 is map index into obj->maps[] array
6368 insn->imm = POISON_LDIMM64_MAP_BASE + map_idx;
6383 prog->name, relo_idx, insn_idx, ext->name);
6386 insn->code = BPF_JMP | BPF_CALL;
6387 insn->dst_reg = 0;
6388 insn->src_reg = 0;
6389 insn->off = 0;
6392 * invalid func unknown#2001000123
6393 * where lower 123 is extern index into obj->externs[] array
6395 insn->imm = POISON_CALL_KFUNC_BASE + ext_idx;
6399 * - map references;
6400 * - global variable references;
6401 * - extern references.
6408 for (i = 0; i < prog->nr_reloc; i++) {
6409 struct reloc_desc *relo = &prog->reloc_desc[i];
6410 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6414 switch (relo->type) {
6416 map = &obj->maps[relo->map_idx];
6417 if (obj->gen_loader) {
6419 insn[0].imm = relo->map_idx;
6420 } else if (map->autocreate) {
6422 insn[0].imm = map->fd;
6424 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6425 relo->map_idx, map);
6429 map = &obj->maps[relo->map_idx];
6430 insn[1].imm = insn[0].imm + relo->sym_off;
6431 if (obj->gen_loader) {
6433 insn[0].imm = relo->map_idx;
6434 } else if (map->autocreate) {
6436 insn[0].imm = map->fd;
6438 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6439 relo->map_idx, map);
6443 ext = &obj->externs[relo->ext_idx];
6444 if (ext->type == EXT_KCFG) {
6445 if (obj->gen_loader) {
6447 insn[0].imm = obj->kconfig_map_idx;
6450 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6452 insn[1].imm = ext->kcfg.data_off;
6454 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
6456 insn[0].imm = ext->ksym.kernel_btf_id;
6457 insn[1].imm = ext->ksym.kernel_btf_obj_fd;
6459 insn[0].imm = (__u32)ext->ksym.addr;
6460 insn[1].imm = ext->ksym.addr >> 32;
6465 ext = &obj->externs[relo->ext_idx];
6467 if (ext->is_set) {
6468 insn[0].imm = ext->ksym.kernel_btf_id;
6469 insn[0].off = ext->ksym.btf_fd_idx;
6471 poison_kfunc_call(prog, i, relo->insn_idx, insn,
6472 relo->ext_idx, ext);
6478 prog->name, i);
6479 return -EINVAL;
6491 prog->name, i, relo->type);
6492 return -EINVAL;
6513 sec_idx = ext_info->sec_idxs[sec_num];
6515 if (prog->sec_idx != sec_idx)
6521 if (insn_off < prog->sec_insn_off)
6523 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6528 copy_end = rec + ext_info->rec_size;
6532 return -ENOENT;
6534 /* append func/line info of a given (sub-)program to the main
6537 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
6538 new_sz = old_sz + (copy_end - copy_start);
6541 return -ENOMEM;
6543 *prog_rec_cnt = new_sz / ext_info->rec_size;
6544 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6546 /* Kernel instruction offsets are in units of 8-byte
6552 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6555 for (; rec < rec_end; rec += ext_info->rec_size) {
6560 *prog_rec_sz = ext_info->rec_size;
6564 return -ENOENT;
6577 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
6583 if (main_prog != prog && !main_prog->func_info)
6586 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6587 &main_prog->func_info,
6588 &main_prog->func_info_cnt,
6589 &main_prog->func_info_rec_size);
6591 if (err != -ENOENT) {
6593 prog->name, err);
6596 if (main_prog->func_info) {
6601 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6606 prog->name);
6611 if (main_prog != prog && !main_prog->line_info)
6614 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6615 &main_prog->line_info,
6616 &main_prog->line_info_cnt,
6617 &main_prog->line_info_rec_size);
6619 if (err != -ENOENT) {
6621 prog->name, err);
6624 if (main_prog->line_info) {
6629 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6634 prog->name);
6644 if (insn_idx == relo->insn_idx)
6646 return insn_idx < relo->insn_idx ? -1 : 1;
6651 if (!prog->nr_reloc)
6653 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6654 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6659 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
6665 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
6671 return -ENOMEM;
6672 if (subprog->nr_reloc)
6673 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
6674 sizeof(*relos) * subprog->nr_reloc);
6676 for (i = main_prog->nr_reloc; i < new_cnt; i++)
6677 relos[i].insn_idx += subprog->sub_insn_off;
6681 main_prog->reloc_desc = relos;
6682 main_prog->nr_reloc = new_cnt;
6694 subprog->sub_insn_off = main_prog->insns_cnt;
6696 new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6697 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6699 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6700 return -ENOMEM;
6702 main_prog->insns = insns;
6703 main_prog->insns_cnt = new_cnt;
6705 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6706 subprog->insns_cnt * sizeof(*insns));
6708 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6709 main_prog->name, subprog->insns_cnt, subprog->name);
6732 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6733 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6738 if (relo && relo->type == RELO_EXTERN_CALL)
6743 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6745 prog->name, insn_idx, relo->type);
6746 return -LIBBPF_ERRNO__RELOC;
6749 /* sub-program instruction index is a combination of
6752 * call always has imm = -1, but for static functions
6753 * relocation is against STT_SECTION and insn->imm
6756 * for subprog addr relocation, the relo->sym_off + insn->imm is
6759 if (relo->type == RELO_CALL)
6760 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6762 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6769 prog->name, insn_idx);
6770 return -LIBBPF_ERRNO__RELOC;
6775 * offset necessary, insns->imm is relative to
6778 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6781 /* we enforce that sub-programs should be in .text section */
6782 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6784 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6785 prog->name);
6786 return -LIBBPF_ERRNO__RELOC;
6792 * - append it at the end of main program's instructions blog;
6793 * - process is recursively, while current program is put on hold;
6794 * - if that subprogram calls some other not yet processes
6799 if (subprog->sub_insn_off == 0) {
6808 /* main_prog->insns memory could have been re-allocated, so
6811 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6818 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6821 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6828 * Relocate sub-program calls.
6830 * Algorithm operates as follows. Each entry-point BPF program (referred to as
6831 * main prog) is processed separately. For each subprog (non-entry functions,
6840 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6856 * subprog->sub_insn_off as zero at all times and won't be appended to current
6865 * +--------+ +-------+
6867 * +--+---+ +--+-+-+ +---+--+
6869 * +--+---+ +------+ +---+--+
6872 * +---+-------+ +------+----+
6874 * +-----------+ +-----------+
6879 * +-----------+------+
6881 * +-----------+------+
6886 * +-----------+------+------+
6888 * +-----------+------+------+
6897 * +-----------+------+
6899 * +-----------+------+
6902 * +-----------+------+------+
6904 * +-----------+------+------+
6917 for (i = 0; i < obj->nr_programs; i++) {
6918 subprog = &obj->programs[i];
6922 subprog->sub_insn_off = 0;
6939 for (i = 0; i < obj->nr_programs; i++) {
6940 prog = &obj->programs[i];
6941 zfree(&prog->reloc_desc);
6942 prog->nr_reloc = 0;
6951 if (a->insn_idx != b->insn_idx)
6952 return a->insn_idx < b->insn_idx ? -1 : 1;
6955 if (a->type != b->type)
6956 return a->type < b->type ? -1 : 1;
6965 for (i = 0; i < obj->nr_programs; i++) {
6966 struct bpf_program *p = &obj->programs[i];
6968 if (!p->nr_reloc)
6971 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6982 if (obj->btf_ext) {
6985 pr_warn("failed to perform CO-RE relocations: %d\n",
6992 /* Before relocating calls pre-process relocations and mark
6999 for (i = 0; i < obj->nr_programs; i++) {
7000 prog = &obj->programs[i];
7001 for (j = 0; j < prog->nr_reloc; j++) {
7002 struct reloc_desc *relo = &prog->reloc_desc[j];
7003 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
7006 if (relo->type == RELO_SUBPROG_ADDR)
7018 for (i = 0; i < obj->nr_programs; i++) {
7019 prog = &obj->programs[i];
7020 /* sub-program's sub-calls are relocated within the context of
7025 if (!prog->autoload)
7031 prog->name, err);
7036 if (prog->exception_cb_idx >= 0) {
7037 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx];
7044 if (subprog->sub_insn_off == 0) {
7055 for (i = 0; i < obj->nr_programs; i++) {
7056 prog = &obj->programs[i];
7059 if (!prog->autoload)
7064 prog->name, err);
7092 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
7093 return -EINVAL;
7094 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
7096 return -EINVAL;
7098 nrels = shdr->sh_size / shdr->sh_entsize;
7103 return -LIBBPF_ERRNO__FORMAT;
7106 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
7109 i, (size_t)ELF64_R_SYM(rel->r_info));
7110 return -LIBBPF_ERRNO__FORMAT;
7112 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
7114 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",
7115 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
7116 (size_t)rel->r_offset, sym->st_name, name);
7118 for (j = 0; j < obj->nr_maps; j++) {
7119 map = &obj->maps[j];
7120 if (map->sec_idx != obj->efile.btf_maps_shndx)
7123 vi = btf_var_secinfos(sec) + map->btf_var_idx;
7124 if (vi->offset <= rel->r_offset &&
7125 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
7128 if (j == obj->nr_maps) {
7129 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
7130 i, name, (size_t)rel->r_offset);
7131 return -EINVAL;
7134 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
7135 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
7138 if (sym->st_shndx != obj->efile.btf_maps_shndx) {
7139 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
7141 return -LIBBPF_ERRNO__RELOC;
7143 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
7144 map->def.key_size != sizeof(int)) {
7145 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
7146 i, map->name, sizeof(int));
7147 return -EINVAL;
7153 return -ESRCH;
7160 return -ESRCH;
7162 if (targ_prog->sec_idx != sym->st_shndx ||
7163 targ_prog->sec_insn_off * 8 != sym->st_value ||
7165 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
7167 return -LIBBPF_ERRNO__RELOC;
7170 return -EINVAL;
7173 var = btf__type_by_id(obj->btf, vi->type);
7174 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
7176 return -EINVAL;
7177 member = btf_members(def) + btf_vlen(def) - 1;
7178 mname = btf__name_by_offset(obj->btf, member->name_off);
7180 return -EINVAL;
7182 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
7183 if (rel->r_offset - vi->offset < moff)
7184 return -EINVAL;
7186 moff = rel->r_offset - vi->offset - moff;
7191 return -EINVAL;
7193 if (moff >= map->init_slots_sz) {
7195 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
7197 return -ENOMEM;
7198 map->init_slots = tmp;
7199 memset(map->init_slots + map->init_slots_sz, 0,
7200 (new_sz - map->init_slots_sz) * host_ptr_sz);
7201 map->init_slots_sz = new_sz;
7203 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
7206 i, map->name, moff, type, name);
7216 for (i = 0; i < obj->efile.sec_cnt; i++) {
7217 struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
7223 if (sec_desc->sec_type != SEC_RELO)
7227 shdr = sec_desc->shdr;
7231 data = sec_desc->data;
7232 idx = shdr->sh_info;
7234 if (shdr->sh_type != SHT_REL) {
7236 return -LIBBPF_ERRNO__INTERNAL;
7239 if (idx == obj->efile.st_ops_shndx || idx == obj->efile.st_ops_link_shndx)
7241 else if (idx == obj->efile.btf_maps_shndx)
7255 if (BPF_CLASS(insn->code) == BPF_JMP &&
7256 BPF_OP(insn->code) == BPF_CALL &&
7257 BPF_SRC(insn->code) == BPF_K &&
7258 insn->src_reg == 0 &&
7259 insn->dst_reg == 0) {
7260 *func_id = insn->imm;
7268 struct bpf_insn *insn = prog->insns;
7272 if (obj->gen_loader)
7275 for (i = 0; i < prog->insns_cnt; i++, insn++) {
7287 insn->imm = BPF_FUNC_probe_read;
7292 insn->imm = BPF_FUNC_probe_read_str;
7304 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
7311 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
7312 opts->expected_attach_type = 0;
7315 opts->prog_flags |= BPF_F_SLEEPABLE;
7317 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
7318 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
7321 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))
7322 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7324 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
7328 attach_name = strchr(prog->sec_name, '/');
7339 …pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\…
7340 prog->name);
7341 return -EINVAL;
7350 prog->attach_btf_obj_fd = btf_obj_fd;
7351 prog->attach_btf_id = btf_type_id;
7354 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
7358 opts->attach_btf_obj_fd = btf_obj_fd;
7359 opts->attach_btf_id = btf_type_id;
7377 __u32 log_level = prog->log_level;
7379 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
7385 prog->name, prog->sec_name);
7386 return -EINVAL;
7390 return -EINVAL;
7393 prog_name = prog->name;
7394 load_attr.attach_prog_fd = prog->attach_prog_fd;
7395 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
7396 load_attr.attach_btf_id = prog->attach_btf_id;
7398 load_attr.prog_ifindex = prog->prog_ifindex;
7404 load_attr.func_info = prog->func_info;
7405 load_attr.func_info_rec_size = prog->func_info_rec_size;
7406 load_attr.func_info_cnt = prog->func_info_cnt;
7407 load_attr.line_info = prog->line_info;
7408 load_attr.line_info_rec_size = prog->line_info_rec_size;
7409 load_attr.line_info_cnt = prog->line_info_cnt;
7412 load_attr.prog_flags = prog->prog_flags;
7413 load_attr.fd_array = obj->fd_array;
7416 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
7417 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
7420 prog->name, err);
7423 insns = prog->insns;
7424 insns_cnt = prog->insns_cnt;
7428 load_attr.expected_attach_type = prog->expected_attach_type;
7430 if (obj->gen_loader) {
7431 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
7433 prog - obj->programs);
7434 *prog_fd = -1;
7445 if (prog->log_buf) {
7446 log_buf = prog->log_buf;
7447 log_buf_size = prog->log_size;
7449 } else if (obj->log_buf) {
7450 log_buf = obj->log_buf;
7451 log_buf_size = obj->log_size;
7457 ret = -ENOMEM;
7470 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
7473 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7474 prog->name, log_buf);
7477 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
7481 for (i = 0; i < obj->nr_maps; i++) {
7482 map = &prog->obj->maps[i];
7483 if (map->libbpf_type != LIBBPF_MAP_RODATA)
7489 prog->name, map->real_name, cp);
7509 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
7514 ret = -errno;
7516 /* post-process verifier log to improve error descriptions */
7520 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
7524 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7525 prog->name, log_buf);
7541 p = cur - 1;
7542 while (p - 1 >= buf && *(p - 1) != '\n')
7543 p--;
7551 /* size of the remaining log content to the right from the to-be-replaced part */
7552 size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
7557 * shift log contents by (patch_sz - orig_sz) bytes to the right
7558 * starting from after to-be-replaced part of the log.
7561 * shift log contents by (orig_sz - patch_sz) bytes to the left
7562 * starting from after to-be-replaced part of the log
7571 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
7573 } else if (patch_sz - orig_sz > buf_sz - log_sz) {
7575 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
7589 /* Expected log for failed and not properly guarded CO-RE relocation:
7590 * line1 -> 123: (85) call unknown#195896080
7591 * line2 -> invalid func unknown#195896080
7592 * line3 -> <anything else or end of buffer>
7595 * instruction index to find corresponding CO-RE relocation and
7597 * failed CO-RE relocation.
7604 if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1)
7611 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
7617 "%d: <invalid CO-RE relocation>\n"
7618 "failed to resolve CO-RE relocation %s%s\n",
7621 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7629 * line1 -> 123: (85) call unknown#2001000345
7630 * line2 -> invalid func unknown#2001000345
7631 * line3 -> <anything else or end of buffer>
7634 * "345" in "2001000345" is a map index in obj->maps to fetch map name.
7636 struct bpf_object *obj = prog->obj;
7641 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2)
7644 map_idx -= POISON_LDIMM64_MAP_BASE;
7645 if (map_idx < 0 || map_idx >= obj->nr_maps)
7647 map = &obj->maps[map_idx];
7652 insn_idx, map->name);
7654 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7662 * line1 -> 123: (85) call unknown#2002000345
7663 * line2 -> invalid func unknown#2002000345
7664 * line3 -> <anything else or end of buffer>
7667 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name.
7669 struct bpf_object *obj = prog->obj;
7674 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &ext_idx) != 2)
7677 ext_idx -= POISON_CALL_KFUNC_BASE;
7678 if (ext_idx < 0 || ext_idx >= obj->nr_extern)
7680 ext = &obj->externs[ext_idx];
7685 insn_idx, ext->name);
7687 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7702 next_line = buf + log_sz - 1;
7709 if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) {
7714 /* failed CO-RE relocation case */
7718 } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_LDIMM64_MAP_PFX)) {
7727 } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_CALL_KFUNC_PFX)) {
7742 struct bpf_object *obj = prog->obj;
7745 for (i = 0; i < prog->nr_reloc; i++) {
7746 struct reloc_desc *relo = &prog->reloc_desc[i];
7747 struct extern_desc *ext = &obj->externs[relo->ext_idx];
7750 switch (relo->type) {
7752 if (ext->type != EXT_KSYM)
7754 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ?
7756 bpf_gen__record_extern(obj->gen_loader, ext->name,
7757 ext->is_weak, !ext->ksym.type_id,
7758 true, kind, relo->insn_idx);
7761 bpf_gen__record_extern(obj->gen_loader, ext->name,
7762 ext->is_weak, false, false, BTF_KIND_FUNC,
7763 relo->insn_idx);
7767 .insn_off = relo->insn_idx * 8,
7768 .type_id = relo->core_relo->type_id,
7769 .access_str_off = relo->core_relo->access_str_off,
7770 .kind = relo->core_relo->kind,
7773 bpf_gen__record_relo_core(obj->gen_loader, &cr);
7790 for (i = 0; i < obj->nr_programs; i++) {
7791 prog = &obj->programs[i];
7797 for (i = 0; i < obj->nr_programs; i++) {
7798 prog = &obj->programs[i];
7801 if (!prog->autoload) {
7802 pr_debug("prog '%s': skipped loading\n", prog->name);
7805 prog->log_level |= log_level;
7807 if (obj->gen_loader)
7810 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7811 obj->license, obj->kern_version, &prog->fd);
7813 pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
7830 prog->sec_def = find_sec_def(prog->sec_name);
7831 if (!prog->sec_def) {
7834 prog->name, prog->sec_name);
7838 prog->type = prog->sec_def->prog_type;
7839 prog->expected_attach_type = prog->sec_def->expected_attach_type;
7844 if (prog->sec_def->prog_setup_fn) {
7845 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
7848 prog->name, err);
7869 return ERR_PTR(-EINVAL);
7874 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7879 return ERR_PTR(-EINVAL);
7893 return ERR_PTR(-EINVAL);
7895 return ERR_PTR(-EINVAL);
7901 obj->log_buf = log_buf;
7902 obj->log_size = log_size;
7903 obj->log_level = log_level;
7908 err = -ENAMETOOLONG;
7911 obj->btf_custom_path = strdup(btf_tmp_path);
7912 if (!obj->btf_custom_path) {
7913 err = -ENOMEM;
7920 obj->kconfig = strdup(kconfig);
7921 if (!obj->kconfig) {
7922 err = -ENOMEM;
7950 return libbpf_err_ptr(-EINVAL);
7967 return libbpf_err_ptr(-EINVAL);
7970 snprintf(tmp_name, sizeof(tmp_name), "%lx-%zx", (unsigned long)obj_buf, obj_buf_sz);
7980 return libbpf_err(-EINVAL);
7982 for (i = 0; i < obj->nr_maps; i++) {
7983 zclose(obj->maps[i].fd);
7984 if (obj->maps[i].st_ops)
7985 zfree(&obj->maps[i].st_ops->kern_vdata);
7988 for (i = 0; i < obj->nr_programs; i++)
7989 bpf_program__unload(&obj->programs[i]);
8002 m->def.map_flags &= ~BPF_F_MMAPABLE;
8017 err = -errno;
8029 err = -EINVAL;
8050 if (!ext || ext->type != EXT_KSYM)
8053 t = btf__type_by_id(obj->btf, ext->btf_id);
8057 if (ext->is_set && ext->ksym.addr != sym_addr) {
8059 sym_name, ext->ksym.addr, sym_addr);
8060 return -EINVAL;
8062 if (!ext->is_set) {
8063 ext->is_set = true;
8064 ext->ksym.addr = sym_addr;
8083 btf = obj->btf_vmlinux;
8087 if (id == -ENOENT) {
8092 for (i = 0; i < obj->btf_module_cnt; i++) {
8094 mod_btf = &obj->btf_modules[i];
8095 btf = mod_btf->btf;
8097 if (id != -ENOENT)
8102 return -ESRCH;
8119 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
8121 if (id == -ESRCH && ext->is_weak)
8124 ext->name);
8129 local_type_id = ext->ksym.type_id;
8133 targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
8134 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
8136 err = bpf_core_types_are_compat(obj->btf, local_type_id,
8142 local_type = btf__type_by_id(obj->btf, local_type_id);
8143 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
8144 targ_name = btf__name_by_offset(btf, targ_type->name_off);
8147 ext->name, local_type_id,
8150 return -EINVAL;
8153 ext->is_set = true;
8154 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8155 ext->ksym.kernel_btf_id = id;
8157 ext->name, id, btf_kind_str(targ_var), targ_var_name);
8171 local_func_proto_id = ext->ksym.type_id;
8173 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf,
8176 if (kfunc_id == -ESRCH && ext->is_weak)
8179 ext->name);
8184 kfunc_proto_id = kern_func->type;
8186 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
8189 if (ext->is_weak)
8193 ext->name, local_func_proto_id,
8194 mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id);
8195 return -EINVAL;
8199 if (mod_btf && !mod_btf->fd_array_idx) {
8200 /* insn->off is s16 */
8201 if (obj->fd_array_cnt == INT16_MAX) {
8203 ext->name, mod_btf->fd_array_idx);
8204 return -E2BIG;
8207 if (!obj->fd_array_cnt)
8208 obj->fd_array_cnt = 1;
8210 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
8211 obj->fd_array_cnt + 1);
8214 mod_btf->fd_array_idx = obj->fd_array_cnt;
8216 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
8219 ext->is_set = true;
8220 ext->ksym.kernel_btf_id = kfunc_id;
8221 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
8224 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call.
8225 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64.
8227 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8229 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id);
8240 for (i = 0; i < obj->nr_extern; i++) {
8241 ext = &obj->externs[i];
8242 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
8245 if (obj->gen_loader) {
8246 ext->is_set = true;
8247 ext->ksym.kernel_btf_obj_fd = 0;
8248 ext->ksym.kernel_btf_id = 0;
8251 t = btf__type_by_id(obj->btf, ext->btf_id);
8271 if (obj->nr_extern == 0)
8274 if (obj->kconfig_map_idx >= 0)
8275 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
8277 for (i = 0; i < obj->nr_extern; i++) {
8278 ext = &obj->externs[i];
8280 if (ext->type == EXT_KSYM) {
8281 if (ext->ksym.type_id)
8286 } else if (ext->type == EXT_KCFG) {
8287 void *ext_ptr = kcfg_data + ext->kcfg.data_off;
8291 if (str_has_pfx(ext->name, "CONFIG_")) {
8297 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
8300 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
8301 return -EINVAL;
8303 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
8305 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
8307 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
8315 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
8316 return -EINVAL;
8323 ext->name, (long long)value);
8325 pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
8326 return -EINVAL;
8332 return -EINVAL;
8334 for (i = 0; i < obj->nr_extern; i++) {
8335 ext = &obj->externs[i];
8336 if (ext->type == EXT_KCFG && !ext->is_set) {
8345 return -EINVAL;
8350 return -EINVAL;
8355 return -EINVAL;
8357 for (i = 0; i < obj->nr_extern; i++) {
8358 ext = &obj->externs[i];
8360 if (!ext->is_set && !ext->is_weak) {
8361 pr_warn("extern '%s' (strong): not resolved\n", ext->name);
8362 return -ESRCH;
8363 } else if (!ext->is_set) {
8365 ext->name);
8377 st_ops = map->st_ops;
8378 for (i = 0; i < btf_vlen(st_ops->type); i++) {
8379 struct bpf_program *prog = st_ops->progs[i];
8387 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
8396 for (i = 0; i < obj->nr_maps; i++)
8397 if (bpf_map__is_struct_ops(&obj->maps[i]))
8398 bpf_map_prepare_vdata(&obj->maps[i]);
8408 return libbpf_err(-EINVAL);
8410 if (obj->loaded) {
8411 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
8412 return libbpf_err(-EINVAL);
8415 if (obj->gen_loader)
8416 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
8420 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
8425 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
8430 if (obj->gen_loader) {
8432 if (obj->btf)
8433 btf__set_fd(obj->btf, -1);
8434 for (i = 0; i < obj->nr_maps; i++)
8435 obj->maps[i].fd = -1;
8437 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
8441 zfree(&obj->fd_array);
8444 for (i = 0; i < obj->btf_module_cnt; i++) {
8445 close(obj->btf_modules[i].fd);
8446 btf__free(obj->btf_modules[i].btf);
8447 free(obj->btf_modules[i].name);
8449 free(obj->btf_modules);
8452 btf__free(obj->btf_vmlinux);
8453 obj->btf_vmlinux = NULL;
8455 obj->loaded = true; /* doesn't matter if successfully or not */
8462 /* unpin any maps that were auto-pinned during load */
8463 for (i = 0; i < obj->nr_maps; i++)
8464 if (obj->maps[i].pinned && !obj->maps[i].reused)
8465 bpf_map__unpin(&obj->maps[i], NULL);
8468 pr_warn("failed to load object '%s'\n", obj->path);
8485 return -ENOMEM;
8489 err = -errno;
8493 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8507 return -EINVAL;
8511 return -ENOMEM;
8517 err = -errno;
8523 err = -EINVAL;
8534 if (prog->fd < 0) {
8535 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
8536 return libbpf_err(-EINVAL);
8547 if (bpf_obj_pin(prog->fd, path)) {
8548 err = -errno;
8550 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
8554 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
8562 if (prog->fd < 0) {
8563 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
8564 return libbpf_err(-EINVAL);
8573 return libbpf_err(-errno);
8575 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
8586 return libbpf_err(-EINVAL);
8589 if (map->pin_path) {
8590 if (path && strcmp(path, map->pin_path)) {
8592 bpf_map__name(map), map->pin_path, path);
8593 return libbpf_err(-EINVAL);
8594 } else if (map->pinned) {
8595 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
8596 bpf_map__name(map), map->pin_path);
8603 return libbpf_err(-EINVAL);
8604 } else if (map->pinned) {
8606 return libbpf_err(-EEXIST);
8609 map->pin_path = strdup(path);
8610 if (!map->pin_path) {
8611 err = -errno;
8616 err = make_parent_dir(map->pin_path);
8620 err = check_path(map->pin_path);
8624 if (bpf_obj_pin(map->fd, map->pin_path)) {
8625 err = -errno;
8629 map->pinned = true;
8630 pr_debug("pinned map '%s'\n", map->pin_path);
8635 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8646 return libbpf_err(-EINVAL);
8649 if (map->pin_path) {
8650 if (path && strcmp(path, map->pin_path)) {
8652 bpf_map__name(map), map->pin_path, path);
8653 return libbpf_err(-EINVAL);
8655 path = map->pin_path;
8659 return libbpf_err(-EINVAL);
8668 return libbpf_err(-errno);
8670 map->pinned = false;
8683 return libbpf_err(-errno);
8686 free(map->pin_path);
8687 map->pin_path = new;
8696 return map->pin_path;
8701 return map->pinned;
8720 return libbpf_err(-ENOENT);
8722 if (!obj->loaded) {
8724 return libbpf_err(-ENOENT);
8731 if (!map->autocreate)
8740 } else if (!map->pin_path) {
8753 if (!map->pin_path)
8768 return libbpf_err(-ENOENT);
8780 } else if (!map->pin_path) {
8799 return libbpf_err(-ENOENT);
8801 if (!obj->loaded) {
8803 return libbpf_err(-ENOENT);
8807 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8820 if (pathname_concat(buf, sizeof(buf), path, prog->name))
8835 return libbpf_err(-ENOENT);
8840 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8886 if (map->inner_map) {
8887 bpf_map__destroy(map->inner_map);
8888 zfree(&map->inner_map);
8891 zfree(&map->init_slots);
8892 map->init_slots_sz = 0;
8894 if (map->mmaped) {
8897 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
8898 munmap(map->mmaped, mmap_sz);
8899 map->mmaped = NULL;
8902 if (map->st_ops) {
8903 zfree(&map->st_ops->data);
8904 zfree(&map->st_ops->progs);
8905 zfree(&map->st_ops->kern_func_off);
8906 zfree(&map->st_ops);
8909 zfree(&map->name);
8910 zfree(&map->real_name);
8911 zfree(&map->pin_path);
8913 if (map->fd >= 0)
8914 zclose(map->fd);
8924 usdt_manager_free(obj->usdt_man);
8925 obj->usdt_man = NULL;
8927 bpf_gen__free(obj->gen_loader);
8930 btf__free(obj->btf);
8931 btf__free(obj->btf_vmlinux);
8932 btf_ext__free(obj->btf_ext);
8934 for (i = 0; i < obj->nr_maps; i++)
8935 bpf_map__destroy(&obj->maps[i]);
8937 zfree(&obj->btf_custom_path);
8938 zfree(&obj->kconfig);
8940 for (i = 0; i < obj->nr_extern; i++)
8941 zfree(&obj->externs[i].essent_name);
8943 zfree(&obj->externs);
8944 obj->nr_extern = 0;
8946 zfree(&obj->maps);
8947 obj->nr_maps = 0;
8949 if (obj->programs && obj->nr_programs) {
8950 for (i = 0; i < obj->nr_programs; i++)
8951 bpf_program__exit(&obj->programs[i]);
8953 zfree(&obj->programs);
8960 return obj ? obj->name : libbpf_err_ptr(-EINVAL);
8965 return obj ? obj->kern_version : 0;
8970 return obj ? obj->btf : NULL;
8975 return obj->btf ? btf__fd(obj->btf) : -1;
8980 if (obj->loaded)
8981 return libbpf_err(-EINVAL);
8983 obj->kern_version = kern_version;
8993 return -EFAULT;
8995 return -EINVAL;
8998 return -ENOMEM;
8999 gen->opts = opts;
9000 obj->gen_loader = gen;
9008 size_t nr_programs = obj->nr_programs;
9016 return forward ? &obj->programs[0] :
9017 &obj->programs[nr_programs - 1];
9019 if (p->obj != obj) {
9024 idx = (p - obj->programs) + (forward ? 1 : -1);
9025 if (idx >= obj->nr_programs || idx < 0)
9027 return &obj->programs[idx];
9056 prog->prog_ifindex = ifindex;
9061 return prog->name;
9066 return prog->sec_name;
9071 return prog->autoload;
9076 if (prog->obj->loaded)
9077 return libbpf_err(-EINVAL);
9079 prog->autoload = autoload;
9085 return prog->autoattach;
9090 prog->autoattach = autoattach;
9095 return prog->insns;
9100 return prog->insns_cnt;
9108 if (prog->obj->loaded)
9109 return -EBUSY;
9111 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
9114 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
9115 return -ENOMEM;
9119 prog->insns = insns;
9120 prog->insns_cnt = new_insn_cnt;
9127 return libbpf_err(-EINVAL);
9129 if (prog->fd < 0)
9130 return libbpf_err(-ENOENT);
9132 return prog->fd;
9140 return prog->type;
9151 if (prog->obj->loaded)
9152 return libbpf_err(-EBUSY);
9155 if (prog->type == type)
9158 prog->type = type;
9162 * fallback handler, which by definition is program type-agnostic and
9163 * is a catch-all custom handler, optionally set by the application,
9166 if (prog->sec_def != &custom_fallback_def)
9167 prog->sec_def = NULL;
9176 return prog->expected_attach_type;
9182 if (prog->obj->loaded)
9183 return libbpf_err(-EBUSY);
9185 prog->expected_attach_type = type;
9191 return prog->prog_flags;
9196 if (prog->obj->loaded)
9197 return libbpf_err(-EBUSY);
9199 prog->prog_flags = flags;
9205 return prog->log_level;
9210 if (prog->obj->loaded)
9211 return libbpf_err(-EBUSY);
9213 prog->log_level = log_level;
9219 *log_size = prog->log_size;
9220 return prog->log_buf;
9226 return -EINVAL;
9227 if (prog->log_size > UINT_MAX)
9228 return -EINVAL;
9229 if (prog->obj->loaded)
9230 return -EBUSY;
9232 prog->log_buf = log_buf;
9233 prog->log_size = log_size;
9372 return libbpf_err(-EINVAL);
9375 return libbpf_err(-E2BIG);
9381 return libbpf_err(-ENOMEM);
9387 return libbpf_err(-EBUSY);
9392 sec_def->sec = sec ? strdup(sec) : NULL;
9393 if (sec && !sec_def->sec)
9394 return libbpf_err(-ENOMEM);
9396 sec_def->prog_type = prog_type;
9397 sec_def->expected_attach_type = exp_attach_type;
9398 sec_def->cookie = OPTS_GET(opts, cookie, 0);
9400 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
9401 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
9402 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
9404 sec_def->handler_id = ++last_custom_sec_def_handler_id;
9411 return sec_def->handler_id;
9420 return libbpf_err(-EINVAL);
9434 return libbpf_err(-ENOENT);
9438 custom_sec_defs[i - 1] = custom_sec_defs[i];
9439 custom_sec_def_cnt--;
9455 size_t len = strlen(sec_def->sec);
9458 if (sec_def->sec[len - 1] == '/') {
9459 if (str_has_pfx(sec_name, sec_def->sec))
9465 * well-formed SEC("type/extras") with proper '/' separator
9467 if (sec_def->sec[len - 1] == '+') {
9468 len--;
9470 if (strncmp(sec_name, sec_def->sec, len) != 0)
9478 return strcmp(sec_name, sec_def->sec) == 0;
9523 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9526 if (!(sec_def->cookie & SEC_ATTACHABLE))
9548 return libbpf_err(-EINVAL);
9552 *prog_type = sec_def->prog_type;
9553 *expected_attach_type = sec_def->expected_attach_type;
9564 return libbpf_err(-ESRCH);
9606 for (i = 0; i < obj->nr_maps; i++) {
9607 map = &obj->maps[i];
9610 if (map->sec_idx == sec_idx &&
9611 map->sec_offset <= offset &&
9612 offset - map->sec_offset < map->def.value_size)
9619 /* Collect the reloc from ELF and populate the st_ops->progs[] */
9636 btf = obj->btf;
9637 nrels = shdr->sh_size / shdr->sh_entsize;
9642 return -LIBBPF_ERRNO__FORMAT;
9645 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
9648 (size_t)ELF64_R_SYM(rel->r_info));
9649 return -LIBBPF_ERRNO__FORMAT;
9652 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
9653 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset);
9655 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
9656 (size_t)rel->r_offset);
9657 return -EINVAL;
9660 moff = rel->r_offset - map->sec_offset;
9661 shdr_idx = sym->st_shndx;
9662 st_ops = map->st_ops;
9663 …pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %…
9664 map->name,
9665 (long long)(rel->r_info >> 32),
9666 (long long)sym->st_value,
9667 shdr_idx, (size_t)rel->r_offset,
9668 map->sec_offset, sym->st_name, name);
9671 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
9672 map->name, (size_t)rel->r_offset, shdr_idx);
9673 return -LIBBPF_ERRNO__RELOC;
9675 if (sym->st_value % BPF_INSN_SZ) {
9677 map->name, (unsigned long long)sym->st_value);
9678 return -LIBBPF_ERRNO__FORMAT;
9680 insn_idx = sym->st_value / BPF_INSN_SZ;
9682 member = find_member_by_offset(st_ops->type, moff * 8);
9685 map->name, moff);
9686 return -EINVAL;
9688 member_idx = member - btf_members(st_ops->type);
9689 name = btf__name_by_offset(btf, member->name_off);
9691 if (!resolve_func_ptr(btf, member->type, NULL)) {
9693 map->name, name);
9694 return -EINVAL;
9700 map->name, shdr_idx, name);
9701 return -EINVAL;
9705 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
9707 map->name, prog->name);
9708 return -EINVAL;
9714 if (!prog->attach_btf_id) {
9715 prog->attach_btf_id = st_ops->type_id;
9716 prog->expected_attach_type = member_idx;
9719 /* struct_ops BPF prog can be re-used between multiple
9724 if (prog->attach_btf_id != st_ops->type_id ||
9725 prog->expected_attach_type != member_idx) {
9727 map->name, prog->name, prog->sec_name, prog->type,
9728 prog->attach_btf_id, prog->expected_attach_type, name);
9729 return -EINVAL;
9732 st_ops->progs[member_idx] = prog;
9779 return -ENAMETOOLONG;
9829 err = -EINVAL;
9856 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
9862 if (ret != -ENOENT)
9869 for (i = 0; i < obj->btf_module_cnt; i++) {
9870 const struct module_btf *mod = &obj->btf_modules[i];
9872 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
9874 *btf_obj_fd = mod->fd;
9878 if (ret == -ENOENT)
9884 return -ESRCH;
9890 enum bpf_attach_type attach_type = prog->expected_attach_type;
9891 __u32 attach_prog_fd = prog->attach_prog_fd;
9895 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
9897 pr_warn("prog '%s': attach program FD is not set\n", prog->name);
9898 return -EINVAL;
9903 prog->name, attach_prog_fd, attach_name, err);
9912 if (prog->obj->gen_loader) {
9913 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
9917 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9921 prog->name, attach_name, err);
9934 return libbpf_err(-EINVAL);
9945 return libbpf_err(-EINVAL);
9948 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9949 return libbpf_err(-EINVAL);
9950 if (!(sec_def->cookie & SEC_ATTACHABLE))
9951 return libbpf_err(-EINVAL);
9953 *attach_type = sec_def->expected_attach_type;
9959 return map ? map->fd : libbpf_err(-EINVAL);
9965 * their user-visible name differs from kernel-visible name. Users see
9970 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
9972 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
9983 return map->real_name;
9985 return map->name;
9990 return map->def.type;
9995 if (map->fd >= 0)
9996 return libbpf_err(-EBUSY);
9997 map->def.type = type;
10003 return map->def.map_flags;
10008 if (map->fd >= 0)
10009 return libbpf_err(-EBUSY);
10010 map->def.map_flags = flags;
10016 return map->map_extra;
10021 if (map->fd >= 0)
10022 return libbpf_err(-EBUSY);
10023 map->map_extra = map_extra;
10029 return map->numa_node;
10034 if (map->fd >= 0)
10035 return libbpf_err(-EBUSY);
10036 map->numa_node = numa_node;
10042 return map->def.key_size;
10047 if (map->fd >= 0)
10048 return libbpf_err(-EBUSY);
10049 map->def.key_size = size;
10055 return map->def.value_size;
10069 btf = bpf_object__btf(map->obj);
10071 return -ENOENT;
10078 return -EINVAL;
10086 return -EINVAL;
10090 var = &btf_var_secinfos(datasec_type)[vlen - 1];
10091 var_type = btf_type_by_id(btf, var->type);
10092 array_type = skip_mods_and_typedefs(btf, var_type->type, NULL);
10096 return -EINVAL;
10101 element_sz = btf__resolve_size(btf, array->type);
10102 if (element_sz <= 0 || (size - var->offset) % element_sz != 0) {
10105 return -EINVAL;
10109 nr_elements = (size - var->offset) / element_sz;
10110 new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements);
10117 datasec_type = btf_type_by_id(btf, map->btf_value_type_id);
10118 var = &btf_var_secinfos(datasec_type)[vlen - 1];
10119 var_type = btf_type_by_id(btf, var->type);
10122 datasec_type->size = size;
10123 var->size = size - var->offset;
10124 var_type->type = new_array_id;
10131 if (map->fd >= 0)
10132 return libbpf_err(-EBUSY);
10134 if (map->mmaped) {
10138 mmap_old_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
10139 mmap_new_sz = bpf_map_mmap_sz(size, map->def.max_entries);
10142 pr_warn("map '%s': failed to resize memory-mapped region: %d\n",
10147 if (err && err != -ENOENT) {
10150 map->btf_value_type_id = 0;
10151 map->btf_key_type_id = 0;
10155 map->def.value_size = size;
10161 return map ? map->btf_key_type_id : 0;
10166 return map ? map->btf_value_type_id : 0;
10172 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
10173 size != map->def.value_size || map->fd >= 0)
10174 return libbpf_err(-EINVAL);
10176 memcpy(map->mmaped, data, size);
10182 if (!map->mmaped)
10184 *psize = map->def.value_size;
10185 return map->mmaped;
10190 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
10195 return map->map_ifindex;
10200 if (map->fd >= 0)
10201 return libbpf_err(-EBUSY);
10202 map->map_ifindex = ifindex;
10208 if (!bpf_map_type__is_map_in_map(map->def.type)) {
10210 return libbpf_err(-EINVAL);
10212 if (map->inner_map_fd != -1) {
10214 return libbpf_err(-EINVAL);
10216 if (map->inner_map) {
10217 bpf_map__destroy(map->inner_map);
10218 zfree(&map->inner_map);
10220 map->inner_map_fd = fd;
10230 if (!obj || !obj->maps)
10233 s = obj->maps;
10234 e = obj->maps + obj->nr_maps;
10242 idx = (m - obj->maps) + i;
10243 if (idx >= obj->nr_maps || idx < 0)
10245 return &obj->maps[idx];
10252 return obj->maps;
10261 if (!obj->nr_maps)
10263 return obj->maps + obj->nr_maps - 1;
10266 return __bpf_map__iter(next, obj, -1);
10280 if (pos->real_name && strcmp(pos->real_name, name) == 0)
10286 if (strcmp(pos->real_name, name) == 0)
10290 if (strcmp(pos->name, name) == 0)
10305 if (map->fd <= 0)
10306 return -ENOENT;
10308 if (map->def.key_size != key_sz) {
10310 map->name, key_sz, map->def.key_size);
10311 return -EINVAL;
10317 switch (map->def.type) {
10323 size_t elem_sz = roundup(map->def.value_size, 8);
10326 … pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",
10327 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
10328 return -EINVAL;
10333 if (map->def.value_size != value_sz) {
10335 map->name, value_sz, map->def.value_size);
10336 return -EINVAL;
10353 return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
10366 return bpf_map_update_elem(map->fd, key, value, flags);
10378 return bpf_map_delete_elem_flags(map->fd, key, flags);
10391 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
10403 return bpf_map_get_next_key(map->fd, cur_key, next_key);
10412 errno = -PTR_ERR(ptr);
10419 return -errno;
10443 link->disconnected = true;
10453 if (!link->disconnected && link->detach)
10454 err = link->detach(link);
10455 if (link->pin_path)
10456 free(link->pin_path);
10457 if (link->dealloc)
10458 link->dealloc(link);
10467 return link->fd;
10472 return link->pin_path;
10477 return libbpf_err_errno(close(link->fd));
10487 fd = -errno;
10495 return libbpf_err_ptr(-ENOMEM);
10497 link->detach = &bpf_link__detach_fd;
10498 link->fd = fd;
10500 link->pin_path = strdup(path);
10501 if (!link->pin_path) {
10503 return libbpf_err_ptr(-ENOMEM);
10511 return bpf_link_detach(link->fd) ? -errno : 0;
10518 if (link->pin_path)
10519 return libbpf_err(-EBUSY);
10527 link->pin_path = strdup(path);
10528 if (!link->pin_path)
10529 return libbpf_err(-ENOMEM);
10531 if (bpf_obj_pin(link->fd, link->pin_path)) {
10532 err = -errno;
10533 zfree(&link->pin_path);
10537 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
10545 if (!link->pin_path)
10546 return libbpf_err(-EINVAL);
10548 err = unlink(link->pin_path);
10550 return -errno;
10552 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
10553 zfree(&link->pin_path);
10574 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
10575 err = -errno;
10577 if (perf_link->perf_event_fd != link->fd)
10578 close(perf_link->perf_event_fd);
10579 close(link->fd);
10582 if (perf_link->legacy_probe_name) {
10583 if (perf_link->legacy_is_kprobe) {
10584 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
10585 perf_link->legacy_is_retprobe);
10587 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
10588 perf_link->legacy_is_retprobe);
10599 free(perf_link->legacy_probe_name);
10608 int prog_fd, link_fd = -1, err;
10612 return libbpf_err_ptr(-EINVAL);
10616 prog->name, pfd);
10617 return libbpf_err_ptr(-EINVAL);
10622 prog->name);
10623 return libbpf_err_ptr(-EINVAL);
10628 return libbpf_err_ptr(-ENOMEM);
10629 link->link.detach = &bpf_link_perf_detach;
10630 link->link.dealloc = &bpf_link_perf_dealloc;
10631 link->perf_event_fd = pfd;
10634 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {
10640 err = -errno;
10642 prog->name, pfd,
10646 link->link.fd = link_fd;
10649 pr_warn("prog '%s': user context value is not supported\n", prog->name);
10650 err = -EOPNOTSUPP;
10655 err = -errno;
10657 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10658 if (err == -EPROTO)
10660 prog->name, pfd);
10663 link->link.fd = pfd;
10666 err = -errno;
10668 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10672 return &link->link;
10686 * this function is expected to parse integer in the range of [0, 2^31-1] from
10698 err = -errno;
10705 err = err == EOF ? -EIO : -errno;
10755 return -EINVAL;
10787 pid < 0 ? -1 : pid /* pid */,
10788 pid == -1 ? 0 : -1 /* cpu */,
10789 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10790 return pfd >= 0 ? pfd : -errno;
10804 return -EINVAL;
10808 return -errno;
10811 err = -errno;
10822 static int has_debugfs = -1;
10884 return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
10928 pid < 0 ? -1 : pid, /* pid */
10929 pid == -1 ? 0 : -1, /* cpu */
10930 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10932 err = -errno;
11018 return libbpf_err_ptr(-EINVAL);
11033 return libbpf_err_ptr(-ENOTSUP);
11037 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
11038 return libbpf_err_ptr(-ENOTSUP);
11043 return libbpf_err_ptr(-EINVAL);
11049 -1 /* pid */, 0 /* ref_ctr_off */);
11058 return libbpf_err_ptr(-ENOMEM);
11061 offset, -1 /* pid */);
11064 err = -errno;
11066 prog->name, retprobe ? "kretprobe" : "kprobe",
11076 prog->name, retprobe ? "kretprobe" : "kprobe",
11084 perf_link->legacy_probe_name = legacy_probe;
11085 perf_link->legacy_is_kprobe = true;
11086 perf_link->legacy_is_retprobe = retprobe;
11118 return libbpf_err_ptr(-EINVAL);
11120 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
11187 struct kprobe_multi_resolve *res = data->res;
11190 if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp))
11193 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1);
11197 res->addrs[res->cnt++] = (unsigned long)sym_addr;
11213 err = -errno;
11227 err = -EINVAL;
11231 if (!glob_match(sym_name, res->pattern))
11240 err = -errno;
11249 err = -ENOENT;
11261 if (res->cnt == 0)
11262 err = -ENOENT;
11275 return access(tracefs_available_filter_functions_addrs(), R_OK) != -1;
11288 err = -errno;
11301 err = -EINVAL;
11305 if (!glob_match(sym_name, res->pattern))
11308 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap,
11309 sizeof(*res->addrs), res->cnt + 1);
11313 res->addrs[res->cnt++] = (unsigned long)sym_addr;
11316 if (res->cnt == 0)
11317 err = -ENOENT;
11343 return libbpf_err_ptr(-EINVAL);
11351 return libbpf_err_ptr(-EINVAL);
11353 return libbpf_err_ptr(-EINVAL);
11355 return libbpf_err_ptr(-EINVAL);
11357 return libbpf_err_ptr(-EINVAL);
11380 err = -ENOMEM;
11383 link->detach = &bpf_link__detach_fd;
11388 err = -errno;
11390 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11393 link->fd = link_fd;
11413 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
11414 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
11417 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
11419 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
11421 func_name = prog->sec_name + sizeof("kprobe/") - 1;
11423 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
11426 return -EINVAL;
11431 return -EINVAL;
11447 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
11448 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
11451 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
11453 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
11455 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
11458 return *link ? 0 : -errno;
11470 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
11471 if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
11472 strcmp(prog->sec_name, "kretprobe.multi") == 0)
11475 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
11477 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
11479 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
11481 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
11484 return -EINVAL;
11496 int n, ret = -EINVAL;
11500 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
11504 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
11509 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
11513 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
11514 prog->sec_name);
11548 return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
11589 pid < 0 ? -1 : pid, /* pid */
11590 pid == -1 ? 0 : -1, /* cpu */
11591 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11593 err = -errno;
11645 ret = -LIBBPF_ERRNO__FORMAT;
11661 elf_errmsg(-1));
11662 ret = -LIBBPF_ERRNO__LIBELF;
11692 return "/lib/x86_64-linux-gnu";
11694 return "/lib/i386-linux-gnu";
11696 return "/lib/s390x-linux-gnu";
11698 return "/lib/s390-linux-gnu";
11700 return "/lib/arm-linux-gnueabi";
11702 return "/lib/arm-linux-gnueabihf";
11704 return "/lib/aarch64-linux-gnu";
11706 return "/lib/mips64el-linux-gnuabi64";
11708 return "/lib/mipsel-linux-gnu";
11710 return "/lib/powerpc64le-linux-gnu";
11712 return "/lib/sparc64-linux-gnu";
11714 return "/lib/riscv64-linux-gnu";
11749 seg_len = next_path ? next_path - s : strlen(s);
11760 return -ENOENT;
11782 return libbpf_err_ptr(-EINVAL);
11798 * - syms and offsets are mutually exclusive
11799 * - ref_ctr_offsets and cookies are optional
11805 return libbpf_err_ptr(-EINVAL);
11807 return libbpf_err_ptr(-EINVAL);
11811 return libbpf_err_ptr(-EINVAL);
11814 return libbpf_err_ptr(-EINVAL);
11822 prog->name, path, err);
11854 err = -ENOMEM;
11857 link->detach = &bpf_link__detach_fd;
11862 err = -errno;
11863 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
11864 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11867 link->fd = link_fd;
11894 return libbpf_err_ptr(-EINVAL);
11902 return libbpf_err_ptr(-EINVAL);
11909 min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1)));
11916 prog->name, binary_path, err);
11945 return libbpf_err_ptr(-ENOTSUP);
11949 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
11950 return libbpf_err_ptr(-ENOTSUP);
11955 return libbpf_err_ptr(-EINVAL);
11965 return libbpf_err_ptr(-EINVAL);
11972 return libbpf_err_ptr(-ENOMEM);
11978 err = -errno;
11980 prog->name, retprobe ? "uretprobe" : "uprobe",
11991 prog->name, retprobe ? "uretprobe" : "uprobe",
11999 perf_link->legacy_probe_name = legacy_probe;
12000 perf_link->legacy_is_kprobe = false;
12001 perf_link->legacy_is_retprobe = retprobe;
12013 /* Format of u[ret]probe section definition supporting auto-attach:
12020 * specified (and auto-attach is not possible) or the above format is specified for
12021 * auto-attach.
12027 int n, c, ret = -EINVAL;
12032 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
12036 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
12041 prog->name, prog->sec_name);
12059 prog->name);
12063 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
12067 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
12068 prog->sec_name);
12095 struct bpf_object *obj = prog->obj;
12101 return libbpf_err_ptr(-EINVAL);
12105 prog->name);
12106 return libbpf_err_ptr(-EINVAL);
12110 return libbpf_err_ptr(-EINVAL);
12116 prog->name, binary_path, err);
12125 if (IS_ERR(obj->usdt_man))
12126 return libbpf_ptr(obj->usdt_man);
12127 if (!obj->usdt_man) {
12128 obj->usdt_man = usdt_manager_new(obj);
12129 if (IS_ERR(obj->usdt_man))
12130 return libbpf_ptr(obj->usdt_man);
12134 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
12152 /* no auto-attach for just SEC("usdt") */
12161 err = -EINVAL;
12163 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
12183 return -errno;
12187 return -E2BIG;
12213 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
12214 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
12216 err = -errno;
12236 return libbpf_err_ptr(-EINVAL);
12243 prog->name, tp_category, tp_name,
12252 prog->name, tp_category, tp_name,
12272 /* no auto-attach for SEC("tp") or SEC("tracepoint") */
12273 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
12276 sec_name = strdup(prog->sec_name);
12278 return -ENOMEM;
12281 if (str_has_pfx(prog->sec_name, "tp/"))
12282 tp_cat = sec_name + sizeof("tp/") - 1;
12284 tp_cat = sec_name + sizeof("tracepoint/") - 1;
12288 return -EINVAL;
12307 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12308 return libbpf_err_ptr(-EINVAL);
12313 return libbpf_err_ptr(-ENOMEM);
12314 link->detach = &bpf_link__detach_fd;
12318 pfd = -errno;
12321 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12324 link->fd = pfd;
12344 if (!str_has_pfx(prog->sec_name, prefixes[i]))
12348 /* no auto-attach case of, e.g., SEC("raw_tp") */
12349 if (prog->sec_name[pfx_len] == '\0')
12352 if (prog->sec_name[pfx_len] != '/')
12355 tp_name = prog->sec_name + pfx_len + 1;
12361 prog->name, prog->sec_name);
12362 return -EINVAL;
12379 return libbpf_err_ptr(-EINVAL);
12383 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12384 return libbpf_err_ptr(-EINVAL);
12389 return libbpf_err_ptr(-ENOMEM);
12390 link->detach = &bpf_link__detach_fd;
12396 pfd = -errno;
12399 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12402 link->fd = pfd;
12446 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12447 return libbpf_err_ptr(-EINVAL);
12452 return libbpf_err_ptr(-ENOMEM);
12453 link->detach = &bpf_link__detach_fd;
12458 link_fd = -errno;
12461 prog->name, target_name,
12465 link->fd = link_fd;
12496 return libbpf_err_ptr(-EINVAL);
12501 /* validate we don't have unexpected combinations of non-zero fields */
12504 prog->name);
12505 return libbpf_err_ptr(-EINVAL);
12509 prog->name);
12510 return libbpf_err_ptr(-EINVAL);
12531 return libbpf_err_ptr(-EINVAL);
12536 /* validate we don't have unexpected combinations of non-zero fields */
12539 prog->name);
12540 return libbpf_err_ptr(-EINVAL);
12544 prog->name);
12545 return libbpf_err_ptr(-EINVAL);
12564 prog->name);
12565 return libbpf_err_ptr(-EINVAL);
12568 if (prog->type != BPF_PROG_TYPE_EXT) {
12570 prog->name);
12571 return libbpf_err_ptr(-EINVAL);
12604 return libbpf_err_ptr(-EINVAL);
12611 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12612 return libbpf_err_ptr(-EINVAL);
12617 return libbpf_err_ptr(-ENOMEM);
12618 link->detach = &bpf_link__detach_fd;
12623 link_fd = -errno;
12626 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12629 link->fd = link_fd;
12647 return libbpf_err_ptr(-EINVAL);
12651 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12652 return libbpf_err_ptr(-EINVAL);
12657 return libbpf_err_ptr(-ENOMEM);
12659 link->detach = &bpf_link__detach_fd;
12670 link_fd = -errno;
12673 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12676 link->fd = link_fd;
12686 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
12687 return libbpf_err_ptr(-EOPNOTSUPP);
12689 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
12693 /* When calling bpf_program__attach() explicitly, auto-attach support
12699 return libbpf_err_ptr(-EOPNOTSUPP);
12716 if (st_link->map_fd < 0)
12718 return bpf_map_delete_elem(link->fd, &zero);
12720 return close(link->fd);
12729 if (!bpf_map__is_struct_ops(map) || map->fd == -1)
12730 return libbpf_err_ptr(-EINVAL);
12734 return libbpf_err_ptr(-EINVAL);
12737 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
12743 if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) {
12748 link->link.detach = bpf_link__detach_struct_ops;
12750 if (!(map->def.map_flags & BPF_F_LINK)) {
12752 link->link.fd = map->fd;
12753 link->map_fd = -1;
12754 return &link->link;
12757 fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL);
12763 link->link.fd = fd;
12764 link->map_fd = map->fd;
12766 return &link->link;
12778 if (!bpf_map__is_struct_ops(map) || map->fd < 0)
12779 return -EINVAL;
12783 if (st_ops_link->map_fd < 0)
12784 return -EINVAL;
12786 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
12792 if (err && err != -EBUSY)
12795 err = bpf_link_update(link->fd, map->fd, NULL);
12799 st_ops_link->map_fd = map->fd;
12814 __u64 data_tail = header->data_tail;
12821 ehdr = base + (data_tail & (mmap_size - 1));
12822 ehdr_size = ehdr->size;
12826 size_t len_first = base + mmap_size - copy_start;
12827 size_t len_secnd = ehdr_size - len_first;
12861 /* sample_cb and lost_cb are higher-level common-case callbacks */
12900 if (cpu_buf->base &&
12901 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
12902 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
12903 if (cpu_buf->fd >= 0) {
12904 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
12905 close(cpu_buf->fd);
12907 free(cpu_buf->buf);
12917 if (pb->cpu_bufs) {
12918 for (i = 0; i < pb->cpu_cnt; i++) {
12919 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
12924 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
12927 free(pb->cpu_bufs);
12929 if (pb->epoll_fd >= 0)
12930 close(pb->epoll_fd);
12931 free(pb->events);
12945 return ERR_PTR(-ENOMEM);
12947 cpu_buf->pb = pb;
12948 cpu_buf->cpu = cpu;
12949 cpu_buf->map_key = map_key;
12951 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
12952 -1, PERF_FLAG_FD_CLOEXEC);
12953 if (cpu_buf->fd < 0) {
12954 err = -errno;
12960 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
12962 cpu_buf->fd, 0);
12963 if (cpu_buf->base == MAP_FAILED) {
12964 cpu_buf->base = NULL;
12965 err = -errno;
12971 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
12972 err = -errno;
13000 return libbpf_err_ptr(-EINVAL);
13030 return libbpf_err_ptr(-EINVAL);
13033 return libbpf_err_ptr(-EINVAL);
13056 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {
13059 return ERR_PTR(-EINVAL);
13062 /* best-effort sanity checks */
13067 err = -errno;
13069 * -EBADFD, -EFAULT, or -E2BIG on real error
13071 if (err != -EINVAL) {
13082 return ERR_PTR(-EINVAL);
13088 return ERR_PTR(-ENOMEM);
13090 pb->event_cb = p->event_cb;
13091 pb->sample_cb = p->sample_cb;
13092 pb->lost_cb = p->lost_cb;
13093 pb->ctx = p->ctx;
13095 pb->page_size = getpagesize();
13096 pb->mmap_size = pb->page_size * page_cnt;
13097 pb->map_fd = map_fd;
13099 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
13100 if (pb->epoll_fd < 0) {
13101 err = -errno;
13107 if (p->cpu_cnt > 0) {
13108 pb->cpu_cnt = p->cpu_cnt;
13110 pb->cpu_cnt = libbpf_num_possible_cpus();
13111 if (pb->cpu_cnt < 0) {
13112 err = pb->cpu_cnt;
13115 if (map.max_entries && map.max_entries < pb->cpu_cnt)
13116 pb->cpu_cnt = map.max_entries;
13119 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
13120 if (!pb->events) {
13121 err = -ENOMEM;
13125 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
13126 if (!pb->cpu_bufs) {
13127 err = -ENOMEM;
13138 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
13142 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
13143 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
13148 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
13151 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
13157 pb->cpu_bufs[j] = cpu_buf;
13159 err = bpf_map_update_elem(pb->map_fd, &map_key,
13160 &cpu_buf->fd, 0);
13162 err = -errno;
13163 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
13164 cpu, map_key, cpu_buf->fd,
13169 pb->events[j].events = EPOLLIN;
13170 pb->events[j].data.ptr = cpu_buf;
13171 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
13172 &pb->events[j]) < 0) {
13173 err = -errno;
13175 cpu, cpu_buf->fd,
13181 pb->cpu_cnt = j;
13210 struct perf_buffer *pb = cpu_buf->pb;
13214 if (pb->event_cb)
13215 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
13217 switch (e->type) {
13221 if (pb->sample_cb)
13222 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
13228 if (pb->lost_cb)
13229 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
13233 pr_warn("unknown perf sample type %d\n", e->type);
13244 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
13245 pb->page_size, &cpu_buf->buf,
13246 &cpu_buf->buf_size,
13255 return pb->epoll_fd;
13262 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
13264 return -errno;
13267 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
13283 return pb->cpu_cnt;
13289 * select()/poll()/epoll() Linux syscalls.
13295 if (buf_idx >= pb->cpu_cnt)
13296 return libbpf_err(-EINVAL);
13298 cpu_buf = pb->cpu_bufs[buf_idx];
13300 return libbpf_err(-ENOENT);
13302 return cpu_buf->fd;
13309 if (buf_idx >= pb->cpu_cnt)
13310 return libbpf_err(-EINVAL);
13312 cpu_buf = pb->cpu_bufs[buf_idx];
13314 return libbpf_err(-ENOENT);
13316 *buf = cpu_buf->base;
13317 *buf_size = pb->mmap_size;
13326 * - 0 on success;
13327 * - <0 on failure.
13333 if (buf_idx >= pb->cpu_cnt)
13334 return libbpf_err(-EINVAL);
13336 cpu_buf = pb->cpu_bufs[buf_idx];
13338 return libbpf_err(-ENOENT);
13347 for (i = 0; i < pb->cpu_cnt; i++) {
13348 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
13369 return libbpf_err(-EINVAL);
13371 if (prog->obj->loaded)
13372 return libbpf_err(-EINVAL);
13378 prog->attach_prog_fd = attach_prog_fd;
13389 return libbpf_err(-EINVAL);
13392 err = bpf_object__load_vmlinux_btf(prog->obj, true);
13395 err = find_kernel_btf_id(prog->obj, attach_func_name,
13396 prog->expected_attach_type,
13402 prog->attach_btf_id = btf_id;
13403 prog->attach_btf_obj_fd = btf_obj_fd;
13404 prog->attach_prog_fd = attach_prog_fd;
13410 int err = 0, n, len, start, end = -1;
13416 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
13422 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
13425 err = -EINVAL;
13433 err = -EINVAL;
13438 err = -ENOMEM;
13442 memset(tmp + *mask_sz, 0, start - *mask_sz);
13443 memset(tmp + start, 1, end - start + 1);
13449 return -EINVAL;
13465 err = -errno;
13472 err = len ? -errno : -EINVAL;
13478 return -E2BIG;
13519 struct bpf_map **map = map_skel->map;
13520 const char *name = map_skel->name;
13521 void **mmaped = map_skel->mmaped;
13526 return -ESRCH;
13529 /* externs shouldn't be pre-setup from user code */
13530 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
13531 *mmaped = (*map)->mmaped;
13544 struct bpf_program **prog = prog_skel->prog;
13545 const char *name = prog_skel->name;
13550 return -ESRCH;
13562 obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts);
13565 pr_warn("failed to initialize skeleton BPF object '%s': %d\n", s->name, err);
13569 *s->obj = obj;
13570 err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz);
13572 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);
13576 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz);
13578 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);
13596 if (!s->obj)
13597 return libbpf_err(-EINVAL);
13599 btf = bpf_object__btf(s->obj);
13602 bpf_object__name(s->obj));
13603 return libbpf_err(-errno);
13606 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz);
13612 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz);
13618 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
13619 var_skel = (void *)s->vars + var_idx * s->var_skel_sz;
13620 map = *var_skel->map;
13628 return libbpf_err(-EINVAL);
13634 var_type = btf__type_by_id(btf, var->type);
13635 var_name = btf__name_by_offset(btf, var_type->name_off);
13636 if (strcmp(var_name, var_skel->name) == 0) {
13637 *var_skel->addr = map->mmaped + var->offset;
13649 free(s->maps);
13650 free(s->progs);
13651 free(s->vars);
13659 err = bpf_object__load(*s->obj);
13661 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
13665 for (i = 0; i < s->map_cnt; i++) {
13666 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
13667 struct bpf_map *map = *map_skel->map;
13668 size_t mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
13669 int prot, map_fd = map->fd;
13670 void **mmaped = map_skel->mmaped;
13675 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
13680 if (map->def.map_flags & BPF_F_RDONLY_PROG)
13685 /* Remap anonymous mmap()-ed "map initialization image" as
13686 * a BPF map-backed mmap()-ed memory, but preserving the same
13695 *mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map_fd, 0);
13697 err = -errno;
13699 pr_warn("failed to re-mmap() map '%s': %d\n",
13712 for (i = 0; i < s->prog_cnt; i++) {
13713 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;
13714 struct bpf_program *prog = *prog_skel->prog;
13715 struct bpf_link **link = prog_skel->link;
13717 if (!prog->autoload || !prog->autoattach)
13720 /* auto-attaching not supported for this program */
13721 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
13724 /* if user already set the link manually, don't attempt auto-attach */
13728 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
13730 pr_warn("prog '%s': failed to auto-attach: %d\n",
13735 /* It's possible that for some SEC() definitions auto-attach
13740 * auto-attached. But if not, it shouldn't trigger skeleton's
13754 for (i = 0; i < s->prog_cnt; i++) {
13755 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;
13756 struct bpf_link **link = prog_skel->link;
13768 if (s->progs)
13770 if (s->obj)
13771 bpf_object__close(*s->obj);
13772 free(s->maps);
13773 free(s->progs);