Lines Matching +full:mips64el +full:- +full:unknown +full:- +full:linux +full:- +full:gnuabi64
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
29 #include <linux/err.h>
30 #include <linux/kernel.h>
31 #include <linux/bpf.h>
32 #include <linux/btf.h>
33 #include <linux/filter.h>
34 #include <linux/limits.h>
35 #include <linux/perf_event.h>
36 #include <linux/ring_buffer.h>
37 #include <linux/version.h>
63 #include <linux/memfd.h>
92 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
272 if (err != -EPERM || geteuid() != 0) in pr_perm_msg()
289 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", in pr_perm_msg()
305 fd = -1; \
316 /* as of v1.0 libbpf_set_strict_mode() is a no-op */ in libbpf_set_strict_mode()
361 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
380 /* BPF program support non-linear XDP buffer */
398 * linux/filter.h.
416 * program. For the entry-point (main) BPF program, this is always
417 * zero. For a sub-program, this gets reset before each of main BPF
419 * whether sub-program was already appended to the main program, and
431 * entry-point BPF programs this includes the size of main program
432 * itself plus all the used sub-programs, appended at the end
481 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
585 /* BTF fd index to be patched in for insn->off, this is
586 * 0 for vmlinux BTF, index in obj->fd_array for module
683 /* Path to the custom BTF to be used for BPF CO-RE relocations as an
687 /* vmlinux BTF override for CO-RE relocations */
739 zclose(prog->fd); in bpf_program__unload()
741 zfree(&prog->func_info); in bpf_program__unload()
742 zfree(&prog->line_info); in bpf_program__unload()
751 zfree(&prog->name); in bpf_program__exit()
752 zfree(&prog->sec_name); in bpf_program__exit()
753 zfree(&prog->insns); in bpf_program__exit()
754 zfree(&prog->reloc_desc); in bpf_program__exit()
756 prog->nr_reloc = 0; in bpf_program__exit()
757 prog->insns_cnt = 0; in bpf_program__exit()
758 prog->sec_idx = -1; in bpf_program__exit()
763 return BPF_CLASS(insn->code) == BPF_JMP && in insn_is_subprog_call()
764 BPF_OP(insn->code) == BPF_CALL && in insn_is_subprog_call()
765 BPF_SRC(insn->code) == BPF_K && in insn_is_subprog_call()
766 insn->src_reg == BPF_PSEUDO_CALL && in insn_is_subprog_call()
767 insn->dst_reg == 0 && in insn_is_subprog_call()
768 insn->off == 0; in insn_is_subprog_call()
773 return insn->code == (BPF_JMP | BPF_CALL); in is_call_insn()
778 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; in insn_is_pseudo_func()
789 return -EINVAL; in bpf_object__init_prog()
793 prog->obj = obj; in bpf_object__init_prog()
795 prog->sec_idx = sec_idx; in bpf_object__init_prog()
796 prog->sec_insn_off = sec_off / BPF_INSN_SZ; in bpf_object__init_prog()
797 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; in bpf_object__init_prog()
799 prog->insns_cnt = prog->sec_insn_cnt; in bpf_object__init_prog()
801 prog->type = BPF_PROG_TYPE_UNSPEC; in bpf_object__init_prog()
802 prog->fd = -1; in bpf_object__init_prog()
809 prog->autoload = false; in bpf_object__init_prog()
813 prog->autoload = true; in bpf_object__init_prog()
816 prog->autoattach = true; in bpf_object__init_prog()
819 prog->log_level = obj->log_level; in bpf_object__init_prog()
821 prog->sec_name = strdup(sec_name); in bpf_object__init_prog()
822 if (!prog->sec_name) in bpf_object__init_prog()
825 prog->name = strdup(name); in bpf_object__init_prog()
826 if (!prog->name) in bpf_object__init_prog()
829 prog->insns = malloc(insn_data_sz); in bpf_object__init_prog()
830 if (!prog->insns) in bpf_object__init_prog()
832 memcpy(prog->insns, insn_data, insn_data_sz); in bpf_object__init_prog()
838 return -ENOMEM; in bpf_object__init_prog()
845 Elf_Data *symbols = obj->efile.symbols; in bpf_object__add_programs()
847 void *data = sec_data->d_buf; in bpf_object__add_programs()
848 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; in bpf_object__add_programs()
853 progs = obj->programs; in bpf_object__add_programs()
854 nr_progs = obj->nr_programs; in bpf_object__add_programs()
855 nr_syms = symbols->d_size / sizeof(Elf64_Sym); in bpf_object__add_programs()
861 if (sym->st_shndx != sec_idx) in bpf_object__add_programs()
863 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) in bpf_object__add_programs()
866 prog_sz = sym->st_size; in bpf_object__add_programs()
867 sec_off = sym->st_value; in bpf_object__add_programs()
869 name = elf_sym_str(obj, sym->st_name); in bpf_object__add_programs()
873 return -LIBBPF_ERRNO__FORMAT; in bpf_object__add_programs()
879 return -LIBBPF_ERRNO__FORMAT; in bpf_object__add_programs()
882 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { in bpf_object__add_programs()
884 return -ENOTSUP; in bpf_object__add_programs()
893 * In this case the original obj->programs in bpf_object__add_programs()
899 return -ENOMEM; in bpf_object__add_programs()
901 obj->programs = progs; in bpf_object__add_programs()
915 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL in bpf_object__add_programs()
916 && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN in bpf_object__add_programs()
917 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) in bpf_object__add_programs()
918 prog->mark_btf_static = true; in bpf_object__add_programs()
921 obj->nr_programs = nr_progs; in bpf_object__add_programs()
934 * Ubuntu 5.4.0-12.15-generic 5.4.8 in get_kernel_version()
985 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) in find_member_by_name()
1037 if (kern_data_member->type == kern_type_id) in find_struct_ops_kern_types()
1043 return -EINVAL; in find_struct_ops_kern_types()
1057 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; in bpf_map__is_struct_ops()
1073 st_ops = map->st_ops; in bpf_map__init_kern_struct_ops()
1074 type = st_ops->type; in bpf_map__init_kern_struct_ops()
1075 tname = st_ops->tname; in bpf_map__init_kern_struct_ops()
1084 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); in bpf_map__init_kern_struct_ops()
1086 map->def.value_size = kern_vtype->size; in bpf_map__init_kern_struct_ops()
1087 map->btf_vmlinux_value_type_id = kern_vtype_id; in bpf_map__init_kern_struct_ops()
1089 st_ops->kern_vdata = calloc(1, kern_vtype->size); in bpf_map__init_kern_struct_ops()
1090 if (!st_ops->kern_vdata) in bpf_map__init_kern_struct_ops()
1091 return -ENOMEM; in bpf_map__init_kern_struct_ops()
1093 data = st_ops->data; in bpf_map__init_kern_struct_ops()
1094 kern_data_off = kern_data_member->offset / 8; in bpf_map__init_kern_struct_ops()
1095 kern_data = st_ops->kern_vdata + kern_data_off; in bpf_map__init_kern_struct_ops()
1107 mname = btf__name_by_offset(btf, member->name_off); in bpf_map__init_kern_struct_ops()
1111 map->name, mname); in bpf_map__init_kern_struct_ops()
1112 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1115 kern_member_idx = kern_member - btf_members(kern_type); in bpf_map__init_kern_struct_ops()
1119 map->name, mname); in bpf_map__init_kern_struct_ops()
1120 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1123 moff = member->offset / 8; in bpf_map__init_kern_struct_ops()
1124 kern_moff = kern_member->offset / 8; in bpf_map__init_kern_struct_ops()
1129 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); in bpf_map__init_kern_struct_ops()
1130 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, in bpf_map__init_kern_struct_ops()
1132 if (BTF_INFO_KIND(mtype->info) != in bpf_map__init_kern_struct_ops()
1133 BTF_INFO_KIND(kern_mtype->info)) { in bpf_map__init_kern_struct_ops()
1135 map->name, mname, BTF_INFO_KIND(mtype->info), in bpf_map__init_kern_struct_ops()
1136 BTF_INFO_KIND(kern_mtype->info)); in bpf_map__init_kern_struct_ops()
1137 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1143 prog = st_ops->progs[i]; in bpf_map__init_kern_struct_ops()
1148 kern_mtype->type, in bpf_map__init_kern_struct_ops()
1151 /* mtype->type must be a func_proto which was in bpf_map__init_kern_struct_ops()
1157 map->name, mname); in bpf_map__init_kern_struct_ops()
1158 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1161 prog->attach_btf_id = kern_type_id; in bpf_map__init_kern_struct_ops()
1162 prog->expected_attach_type = kern_member_idx; in bpf_map__init_kern_struct_ops()
1164 st_ops->kern_func_off[i] = kern_data_off + kern_moff; in bpf_map__init_kern_struct_ops()
1167 map->name, mname, prog->name, moff, in bpf_map__init_kern_struct_ops()
1177 map->name, mname, (ssize_t)msize, in bpf_map__init_kern_struct_ops()
1179 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1183 map->name, mname, (unsigned int)msize, in bpf_map__init_kern_struct_ops()
1197 for (i = 0; i < obj->nr_maps; i++) { in bpf_object__init_kern_struct_ops_maps()
1198 map = &obj->maps[i]; in bpf_object__init_kern_struct_ops_maps()
1203 err = bpf_map__init_kern_struct_ops(map, obj->btf, in bpf_object__init_kern_struct_ops_maps()
1204 obj->btf_vmlinux); in bpf_object__init_kern_struct_ops_maps()
1223 if (obj->efile.st_ops_shndx == -1) in bpf_object__init_struct_ops_maps()
1226 btf = obj->btf; in bpf_object__init_struct_ops_maps()
1232 return -EINVAL; in bpf_object__init_struct_ops_maps()
1238 type = btf__type_by_id(obj->btf, vsi->type); in bpf_object__init_struct_ops_maps()
1239 var_name = btf__name_by_offset(obj->btf, type->name_off); in bpf_object__init_struct_ops_maps()
1241 type_id = btf__resolve_type(obj->btf, vsi->type); in bpf_object__init_struct_ops_maps()
1244 vsi->type, STRUCT_OPS_SEC); in bpf_object__init_struct_ops_maps()
1245 return -EINVAL; in bpf_object__init_struct_ops_maps()
1248 type = btf__type_by_id(obj->btf, type_id); in bpf_object__init_struct_ops_maps()
1249 tname = btf__name_by_offset(obj->btf, type->name_off); in bpf_object__init_struct_ops_maps()
1252 return -ENOTSUP; in bpf_object__init_struct_ops_maps()
1256 return -EINVAL; in bpf_object__init_struct_ops_maps()
1263 map->sec_idx = obj->efile.st_ops_shndx; in bpf_object__init_struct_ops_maps()
1264 map->sec_offset = vsi->offset; in bpf_object__init_struct_ops_maps()
1265 map->name = strdup(var_name); in bpf_object__init_struct_ops_maps()
1266 if (!map->name) in bpf_object__init_struct_ops_maps()
1267 return -ENOMEM; in bpf_object__init_struct_ops_maps()
1269 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; in bpf_object__init_struct_ops_maps()
1270 map->def.key_size = sizeof(int); in bpf_object__init_struct_ops_maps()
1271 map->def.value_size = type->size; in bpf_object__init_struct_ops_maps()
1272 map->def.max_entries = 1; in bpf_object__init_struct_ops_maps()
1274 map->st_ops = calloc(1, sizeof(*map->st_ops)); in bpf_object__init_struct_ops_maps()
1275 if (!map->st_ops) in bpf_object__init_struct_ops_maps()
1276 return -ENOMEM; in bpf_object__init_struct_ops_maps()
1277 st_ops = map->st_ops; in bpf_object__init_struct_ops_maps()
1278 st_ops->data = malloc(type->size); in bpf_object__init_struct_ops_maps()
1279 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); in bpf_object__init_struct_ops_maps()
1280 st_ops->kern_func_off = malloc(btf_vlen(type) * in bpf_object__init_struct_ops_maps()
1281 sizeof(*st_ops->kern_func_off)); in bpf_object__init_struct_ops_maps()
1282 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) in bpf_object__init_struct_ops_maps()
1283 return -ENOMEM; in bpf_object__init_struct_ops_maps()
1285 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) { in bpf_object__init_struct_ops_maps()
1288 return -EINVAL; in bpf_object__init_struct_ops_maps()
1291 memcpy(st_ops->data, in bpf_object__init_struct_ops_maps()
1292 obj->efile.st_ops_data->d_buf + vsi->offset, in bpf_object__init_struct_ops_maps()
1293 type->size); in bpf_object__init_struct_ops_maps()
1294 st_ops->tname = tname; in bpf_object__init_struct_ops_maps()
1295 st_ops->type = type; in bpf_object__init_struct_ops_maps()
1296 st_ops->type_id = type_id; in bpf_object__init_struct_ops_maps()
1299 tname, type_id, var_name, vsi->offset); in bpf_object__init_struct_ops_maps()
1316 return ERR_PTR(-ENOMEM); in bpf_object__new()
1319 strcpy(obj->path, path); in bpf_object__new()
1321 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); in bpf_object__new()
1324 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); in bpf_object__new()
1325 end = strchr(obj->name, '.'); in bpf_object__new()
1330 obj->efile.fd = -1; in bpf_object__new()
1337 obj->efile.obj_buf = obj_buf; in bpf_object__new()
1338 obj->efile.obj_buf_sz = obj_buf_sz; in bpf_object__new()
1339 obj->efile.btf_maps_shndx = -1; in bpf_object__new()
1340 obj->efile.st_ops_shndx = -1; in bpf_object__new()
1341 obj->kconfig_map_idx = -1; in bpf_object__new()
1343 obj->kern_version = get_kernel_version(); in bpf_object__new()
1344 obj->loaded = false; in bpf_object__new()
1351 if (!obj->efile.elf) in bpf_object__elf_finish()
1354 elf_end(obj->efile.elf); in bpf_object__elf_finish()
1356 if (obj->efile.shstring) { in bpf_object__elf_finish()
1357 elfio_string_section_accessor_delete(obj->efile.shstring); in bpf_object__elf_finish()
1359 if (obj->efile.strstring) { in bpf_object__elf_finish()
1360 elfio_string_section_accessor_delete(obj->efile.strstring); in bpf_object__elf_finish()
1362 elfio_delete(obj->efile.elf); in bpf_object__elf_finish()
1364 obj->efile.elf = NULL; in bpf_object__elf_finish()
1365 obj->efile.symbols = NULL; in bpf_object__elf_finish()
1366 obj->efile.st_ops_data = NULL; in bpf_object__elf_finish()
1368 zfree(&obj->efile.secs); in bpf_object__elf_finish()
1369 obj->efile.sec_cnt = 0; in bpf_object__elf_finish()
1370 zclose(obj->efile.fd); in bpf_object__elf_finish()
1371 obj->efile.obj_buf = NULL; in bpf_object__elf_finish()
1372 obj->efile.obj_buf_sz = 0; in bpf_object__elf_finish()
1385 if (obj->efile.elf) {
1387 return -LIBBPF_ERRNO__LIBELF;
1390 if (obj->efile.obj_buf_sz > 0) {
1393 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1398 ftruncate(fdm, obj->efile.obj_buf_sz);
1399 write(fdm, (char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1404 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1405 if (obj->efile.fd < 0) {
1408 err = -errno;
1410 pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1414 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1419 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1420 err = -LIBBPF_ERRNO__LIBELF;
1424 obj->efile.elf = elf;
1427 err = -LIBBPF_ERRNO__FORMAT;
1428 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1436 err = -LIBBPF_ERRNO__FORMAT;
1437 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1441 obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1443 obj->efile.ehdr = ehdr = (Elf64_Ehdr*)obj->efile.obj_buf;
1445 if (!obj->efile.ehdr) {
1446 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1447 err = -LIBBPF_ERRNO__FORMAT;
1452 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1454 obj->path, elf_errmsg(-1));
1455 err = -LIBBPF_ERRNO__FORMAT;
1460 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1462 obj->path, elf_errmsg(-1));
1463 err = -LIBBPF_ERRNO__FORMAT;
1467 obj->efile.shstrndx = elfio_get_section_name_str_index(elf);
1470 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1471 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1472 err = -LIBBPF_ERRNO__FORMAT;
1485 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1488 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1493 pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1494 return -LIBBPF_ERRNO__ENDIAN;
1501 pr_warn("invalid license section in %s\n", obj->path);
1502 return -LIBBPF_ERRNO__FORMAT;
1504 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1507 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1508 pr_debug("license of %s is %s\n", obj->path, obj->license);
1518 pr_warn("invalid kver section in %s\n", obj->path);
1519 return -LIBBPF_ERRNO__FORMAT;
1522 obj->kern_version = kver;
1523 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1543 return -EINVAL;
1553 *size = data->d_size;
1557 return -ENOENT;
1562 Elf_Data *symbols = obj->efile.symbols;
1566 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1569 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1572 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1573 ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1576 sname = elf_sym_str(obj, sym->st_name);
1579 return ERR_PTR(-EIO);
1585 return ERR_PTR(-ENOENT);
1593 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1594 sizeof(*obj->maps), obj->nr_maps + 1);
1598 map = &obj->maps[obj->nr_maps++];
1599 map->obj = obj;
1600 map->fd = -1;
1601 map->inner_map_fd = -1;
1602 map->autocreate = true;
1612 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1647 * '.rodata.abracad' kernel and user-visible name.
1656 sfx_len = BPF_OBJ_NAME_LEN - 1;
1662 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1664 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1689 if (!map->btf_value_type_id)
1692 t = btf__type_by_id(obj->btf, map->btf_value_type_id);
1698 vt = btf__type_by_id(obj->btf, vsi->type);
1702 if (btf_var(vt)->linkage != BTF_VAR_STATIC)
1721 map->libbpf_type = type;
1722 map->sec_idx = sec_idx;
1723 map->sec_offset = 0;
1724 map->real_name = strdup(real_name);
1725 map->name = internal_map_name(obj, real_name);
1726 if (!map->real_name || !map->name) {
1727 zfree(&map->real_name);
1728 zfree(&map->name);
1729 return -ENOMEM;
1732 def = &map->def;
1733 def->type = BPF_MAP_TYPE_ARRAY;
1734 def->key_size = sizeof(int);
1735 def->value_size = data_sz;
1736 def->max_entries = 1;
1737 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1744 def->map_flags |= BPF_F_MMAPABLE;
1747 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1749 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1750 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1751 if (map->mmaped == MAP_FAILED) {
1752 err = -errno;
1753 map->mmaped = NULL;
1755 map->name, err);
1756 zfree(&map->real_name);
1757 zfree(&map->name);
1762 memcpy(map->mmaped, data, data_sz);
1764 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1775 * Populate obj->maps with libbpf internal maps.
1777 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
1778 sec_desc = &obj->efile.secs[sec_idx];
1781 if (!sec_desc->data || sec_desc->data->d_size == 0)
1784 switch (sec_desc->sec_type) {
1793 sec_desc->data->d_buf,
1794 sec_desc->data->d_size);
1797 obj->has_rodata = true;
1805 sec_desc->data->d_buf,
1806 sec_desc->data->d_size);
1817 sec_desc->data->d_size);
1835 for (i = 0; i < obj->nr_extern; i++) {
1836 if (strcmp(obj->externs[i].name, name) == 0)
1837 return &obj->externs[i];
1845 switch (ext->kcfg.type) {
1849 ext->name, value);
1850 return -EINVAL;
1870 ext->name, value);
1871 return -EINVAL;
1873 ext->is_set = true;
1882 if (ext->kcfg.type != KCFG_CHAR_ARR) {
1884 ext->name, value);
1885 return -EINVAL;
1889 if (value[len - 1] != '"') {
1891 ext->name, value);
1892 return -EINVAL;
1896 len -= 2;
1897 if (len >= ext->kcfg.sz) {
1899 ext->name, value, len, ext->kcfg.sz - 1);
1900 len = ext->kcfg.sz - 1;
1904 ext->is_set = true;
1916 err = -errno;
1922 return -EINVAL;
1929 int bit_sz = ext->kcfg.sz * 8;
1931 if (ext->kcfg.sz == 8)
1934 /* Validate that value stored in u64 fits in integer of `ext->sz`
1939 * -2^(Y-1) <= X <= 2^(Y-1) - 1
1940 * 0 <= X + 2^(Y-1) <= 2^Y - 1
1941 * 0 <= X + 2^(Y-1) < 2^Y
1943 * For unsigned target integer, check that all the (64 - Y) bits are
1946 if (ext->kcfg.is_signed)
1947 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1955 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
1956 ext->kcfg.type != KCFG_BOOL) {
1958 ext->name, (unsigned long long)value);
1959 return -EINVAL;
1961 if (ext->kcfg.type == KCFG_BOOL && value > 1) {
1963 ext->name, (unsigned long long)value);
1964 return -EINVAL;
1969 ext->name, (unsigned long long)value, ext->kcfg.sz);
1970 return -ERANGE;
1972 switch (ext->kcfg.sz) {
1986 return -EINVAL;
1988 ext->is_set = true;
2007 return -EINVAL;
2012 if (buf[len - 1] == '\n')
2013 buf[len - 1] = '\0';
2019 return -EINVAL;
2023 if (!ext || ext->is_set)
2026 ext_val = data + ext->kcfg.data_off;
2040 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
2043 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
2044 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
2045 return -EINVAL;
2052 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
2064 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
2066 return -EINVAL;
2068 return -ENAMETOOLONG;
2077 return -ENOENT;
2103 err = -errno;
2104 pr_warn("failed to open in-memory Kconfig: %d\n", err);
2111 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
2127 for (i = 0; i < obj->nr_extern; i++) {
2128 ext = &obj->externs[i];
2129 if (ext->type == EXT_KCFG)
2136 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
2138 ".kconfig", obj->efile.symbols_shndx,
2143 obj->kconfig_map_idx = obj->nr_maps - 1;
2158 *res_id = t->type;
2159 t = btf__type_by_id(btf, t->type);
2174 t = skip_mods_and_typedefs(btf, t->type, res_id);
2202 default: return "unknown";
2221 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2222 const char *name = btf__name_by_offset(btf, m->name_off);
2232 arr_t = btf__type_by_id(btf, t->type);
2235 map_name, name, t->type);
2244 *res = arr_info->nelems;
2254 return -EINVAL;
2256 return -ENAMETOOLONG;
2295 const char *name = btf__name_by_offset(btf, m->name_off);
2299 return -EINVAL;
2302 if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2303 return -EINVAL;
2304 map_def->parts |= MAP_DEF_MAP_TYPE;
2306 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2307 return -EINVAL;
2308 map_def->parts |= MAP_DEF_MAX_ENTRIES;
2310 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2311 return -EINVAL;
2312 map_def->parts |= MAP_DEF_MAP_FLAGS;
2314 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2315 return -EINVAL;
2316 map_def->parts |= MAP_DEF_NUMA_NODE;
2321 return -EINVAL;
2322 if (map_def->key_size && map_def->key_size != sz) {
2324 map_name, map_def->key_size, sz);
2325 return -EINVAL;
2327 map_def->key_size = sz;
2328 map_def->parts |= MAP_DEF_KEY_SIZE;
2332 t = btf__type_by_id(btf, m->type);
2335 map_name, m->type);
2336 return -EINVAL;
2341 return -EINVAL;
2343 sz = btf__resolve_size(btf, t->type);
2346 map_name, t->type, (ssize_t)sz);
2349 if (map_def->key_size && map_def->key_size != sz) {
2351 map_name, map_def->key_size, (ssize_t)sz);
2352 return -EINVAL;
2354 map_def->key_size = sz;
2355 map_def->key_type_id = t->type;
2356 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2361 return -EINVAL;
2362 if (map_def->value_size && map_def->value_size != sz) {
2364 map_name, map_def->value_size, sz);
2365 return -EINVAL;
2367 map_def->value_size = sz;
2368 map_def->parts |= MAP_DEF_VALUE_SIZE;
2372 t = btf__type_by_id(btf, m->type);
2375 map_name, m->type);
2376 return -EINVAL;
2381 return -EINVAL;
2383 sz = btf__resolve_size(btf, t->type);
2386 map_name, t->type, (ssize_t)sz);
2389 if (map_def->value_size && map_def->value_size != sz) {
2391 map_name, map_def->value_size, (ssize_t)sz);
2392 return -EINVAL;
2394 map_def->value_size = sz;
2395 map_def->value_type_id = t->type;
2396 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2399 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2400 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2401 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2406 pr_warn("map '%s': multi-level inner maps not supported.\n",
2408 return -ENOTSUP;
2410 if (i != vlen - 1) {
2413 return -EINVAL;
2416 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2418 return -ENOTSUP;
2420 if (map_def->value_size && map_def->value_size != 4) {
2422 map_name, map_def->value_size);
2423 return -EINVAL;
2425 map_def->value_size = 4;
2426 t = btf__type_by_id(btf, m->type);
2429 map_name, desc, m->type);
2430 return -EINVAL;
2432 if (!btf_is_array(t) || btf_array(t)->nelems) {
2433 pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2435 return -EINVAL;
2437 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2441 return -EINVAL;
2443 t = skip_mods_and_typedefs(btf, t->type, NULL);
2446 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2448 return -EINVAL;
2453 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2455 return -EINVAL;
2463 map_def->parts |= MAP_DEF_INNER_MAP;
2469 return -EINVAL;
2472 return -EINVAL;
2476 return -EINVAL;
2478 map_def->pinning = val;
2479 map_def->parts |= MAP_DEF_PINNING;
2484 return -EINVAL;
2485 map_def->map_extra = map_extra;
2486 map_def->parts |= MAP_DEF_MAP_EXTRA;
2489 pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
2490 return -ENOTSUP;
2492 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
2496 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2498 return -EINVAL;
2513 * a power-of-2 multiple of kernel's page size. If user diligently
2520 * user-set size to satisfy both user size request and kernel
2529 * very close to UINT_MAX but is not a power-of-2 multiple of
2537 return map->def.type == BPF_MAP_TYPE_RINGBUF ||
2538 map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2543 map->def.type = def->map_type;
2544 map->def.key_size = def->key_size;
2545 map->def.value_size = def->value_size;
2546 map->def.max_entries = def->max_entries;
2547 map->def.map_flags = def->map_flags;
2548 map->map_extra = def->map_extra;
2550 map->numa_node = def->numa_node;
2551 map->btf_key_type_id = def->key_type_id;
2552 map->btf_value_type_id = def->value_type_id;
2554 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2556 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2558 if (def->parts & MAP_DEF_MAP_TYPE)
2559 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2561 if (def->parts & MAP_DEF_KEY_TYPE)
2563 map->name, def->key_type_id, def->key_size);
2564 else if (def->parts & MAP_DEF_KEY_SIZE)
2565 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2567 if (def->parts & MAP_DEF_VALUE_TYPE)
2569 map->name, def->value_type_id, def->value_size);
2570 else if (def->parts & MAP_DEF_VALUE_SIZE)
2571 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2573 if (def->parts & MAP_DEF_MAX_ENTRIES)
2574 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2575 if (def->parts & MAP_DEF_MAP_FLAGS)
2576 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2577 if (def->parts & MAP_DEF_MAP_EXTRA)
2578 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2579 (unsigned long long)def->map_extra);
2580 if (def->parts & MAP_DEF_PINNING)
2581 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2582 if (def->parts & MAP_DEF_NUMA_NODE)
2583 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2585 if (def->parts & MAP_DEF_INNER_MAP)
2586 pr_debug("map '%s': found inner map definition.\n", map->name);
2595 default: return "unknown";
2614 var = btf__type_by_id(obj->btf, vi->type);
2616 map_name = btf__name_by_offset(obj->btf, var->name_off);
2620 return -EINVAL;
2622 if ((__u64)vi->offset + vi->size > data->d_size) {
2624 return -EINVAL;
2629 return -EINVAL;
2631 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2633 map_name, btf_var_linkage_str(var_extra->linkage));
2634 return -EOPNOTSUPP;
2637 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2641 return -EINVAL;
2643 if (def->size > vi->size) {
2645 return -EINVAL;
2651 map->name = strdup(map_name);
2652 if (!map->name) {
2654 return -ENOMEM;
2656 map->libbpf_type = LIBBPF_MAP_UNSPEC;
2657 map->def.type = BPF_MAP_TYPE_UNSPEC;
2658 map->sec_idx = sec_idx;
2659 map->sec_offset = vi->offset;
2660 map->btf_var_idx = var_idx;
2662 map_name, map->sec_idx, map->sec_offset);
2664 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2673 pr_warn("map '%s': couldn't build pin path.\n", map->name);
2679 map->inner_map = calloc(1, sizeof(*map->inner_map));
2680 if (!map->inner_map)
2681 return -ENOMEM;
2682 map->inner_map->fd = -1;
2683 map->inner_map->sec_idx = sec_idx;
2684 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2685 if (!map->inner_map->name)
2686 return -ENOMEM;
2687 sprintf(map->inner_map->name, "%s.inner", map_name);
2689 fill_map_from_def(map->inner_map, &inner_def);
2711 if (obj->efile.btf_maps_shndx < 0)
2714 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2719 data = elf_sec_data_by_idx(obj, obj->efile.btf_maps_shndx, &realdata);
2723 MAPS_ELF_SEC, obj->path);
2724 return -EINVAL;
2727 nr_types = btf__type_cnt(obj->btf);
2729 t = btf__type_by_id(obj->btf, i);
2732 name = btf__name_by_offset(obj->btf, t->name_off);
2735 obj->efile.btf_maps_sec_btf_id = i;
2742 return -ENOENT;
2748 obj->efile.btf_maps_shndx,
2788 return sh->sh_flags & SHF_EXECINSTR;
2823 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2829 t->size = 1;
2838 name = (char *)btf__name_by_offset(btf, t->name_off);
2846 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2849 m->offset = v->offset * 8;
2850 m->type = v->type;
2852 vt = (void *)btf__type_by_id(btf, v->type);
2853 m->name_off = vt->name_off;
2858 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2859 t->size = sizeof(__u32); /* kernel enforced */
2862 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2865 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2867 /* replace FLOAT with an equally-sized empty STRUCT;
2871 t->name_off = 0;
2872 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
2875 t->name_off = 0;
2876 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
2879 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
2894 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
2896 m->type = enum64_placeholder_id;
2897 m->offset = 0;
2907 return obj->efile.btf_maps_shndx >= 0 ||
2908 obj->efile.st_ops_shndx >= 0 ||
2909 obj->nr_extern > 0;
2914 return obj->efile.st_ops_shndx >= 0;
2921 int err = -ENOENT;
2924 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2925 err = libbpf_get_error(obj->btf);
2927 obj->btf = NULL;
2931 /* enforce 8-byte pointers for BPF-targeted BTFs */
2932 btf__set_pointer_size(obj->btf, 8);
2938 if (!obj->btf) {
2943 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
2944 err = libbpf_get_error(obj->btf_ext);
2948 obj->btf_ext = NULL;
2953 ext_segs[0] = &obj->btf_ext->func_info;
2954 ext_segs[1] = &obj->btf_ext->line_info;
2955 ext_segs[2] = &obj->btf_ext->core_relo_info;
2966 if (seg->sec_cnt == 0)
2969 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
2970 if (!seg->sec_idxs) {
2971 err = -ENOMEM;
2982 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
2990 pelfio_t elf = obj->efile.elf;
2996 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
2998 seg->sec_idxs[sec_num - 1] = elfio_section_get_index(sec_obj);
3016 return a->offset - b->offset;
3023 const char *sec_name = btf__name_by_offset(btf, t->name_off);
3030 return -ENOENT;
3033 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and
3045 * to be optional. But the STV_HIDDEN handling is non-optional for any
3046 * non-extern DATASEC, so the variable fixup loop below handles both
3047 * functions at the same time, paying the cost of BTF VAR <-> ELF
3050 if (t->size == 0) {
3055 return -ENOENT;
3058 t->size = size;
3068 t_var = btf__type_by_id(btf, vsi->type);
3070 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);
3071 return -EINVAL;
3075 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)
3078 var_name = btf__name_by_offset(btf, t_var->name_off);
3082 return -ENOENT;
3089 return -ENOENT;
3093 vsi->offset = sym->st_value;
3102 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
3103 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)
3104 var->linkage = BTF_VAR_STATIC;
3116 if (!obj->btf)
3119 n = btf__type_cnt(obj->btf);
3121 struct btf_type *t = btf_type_by_id(obj->btf, i);
3129 err = btf_fixup_datasec(obj, obj->btf, t);
3140 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3141 prog->type == BPF_PROG_TYPE_LSM)
3147 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3158 /* CO-RE relocations need kernel BTF, only when btf_custom_path
3161 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
3165 for (i = 0; i < obj->nr_extern; i++) {
3168 ext = &obj->externs[i];
3169 if (ext->type == EXT_KSYM && ext->ksym.type_id)
3174 if (!prog->autoload)
3188 if (obj->btf_vmlinux || obj->gen_loader)
3194 obj->btf_vmlinux = btf__load_vmlinux_btf();
3195 err = libbpf_get_error(obj->btf_vmlinux);
3198 obj->btf_vmlinux = NULL;
3206 struct btf *kern_btf = obj->btf;
3210 if (!obj->btf)
3215 err = -EOPNOTSUPP;
3230 for (i = 0; i < obj->nr_programs; i++) {
3231 struct bpf_program *prog = &obj->programs[i];
3236 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3239 n = btf__type_cnt(obj->btf);
3241 t = btf_type_by_id(obj->btf, j);
3245 name = btf__str_by_offset(obj->btf, t->name_off);
3246 if (strcmp(name, prog->name) != 0)
3249 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3260 raw_data = btf__raw_data(obj->btf, &sz);
3266 /* enforce 8-byte pointers for BPF-targeted BTFs */
3267 btf__set_pointer_size(obj->btf, 8);
3273 if (obj->gen_loader) {
3278 return -ENOMEM;
3279 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3281 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3286 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3287 obj->log_level ? 1 : 0);
3292 btf__set_fd(obj->btf, btf__fd(kern_btf));
3293 btf__set_fd(kern_btf, -1);
3313 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3315 name = elfio_string_get_string(obj->efile.strstring, off);
3319 off, obj->path, elf_errmsg(-1));
3330 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3332 name = elfio_string_get_string(obj->efile.shstring, off);
3337 off, obj->path, elf_errmsg(-1));
3349 scn = elf_getscn(obj->efile.elf, idx);
3352 idx, obj->path, elf_errmsg(-1));
3361 Elf *elf = obj->efile.elf;
3387 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3406 name = elf_sec_str(obj, sh->sh_name);
3409 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3418 psection_t psection = elfio_get_section_by_index(obj->efile.elf, idx);
3420 sheader->sh_name = elfio_section_get_name_string_offset(psection);
3421 sheader->sh_type = elfio_section_get_type(psection);
3422 sheader->sh_flags = elfio_section_get_flags(psection);
3423 sheader->sh_addr = elfio_section_get_address(psection);
3424 sheader->sh_offset = elfio_section_get_offset(psection);
3425 sheader->sh_size = elfio_section_get_size(psection);
3426 sheader->sh_link = elfio_section_get_link(psection);
3427 sheader->sh_info = elfio_section_get_info(psection);
3428 sheader->sh_addralign = elfio_section_get_addr_align(psection);
3429 sheader->sh_entsize = elfio_section_get_entry_size(psection);
3444 idx, obj->path, elf_errmsg(-1));
3464 obj->path, elf_errmsg(-1));
3473 pelfio_t elf = obj->efile.elf;
3475 data->d_buf = (void*)elfio_section_get_data(psection_name);
3476 data->d_size = elfio_section_get_size(psection_name);
3483 pelfio_t elf = obj->efile.elf;
3485 data->d_buf = (void*)elfio_section_get_data(psection_index);
3486 data->d_size = elfio_section_get_size(psection_index);
3494 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3497 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3502 if (idx >= data->d_size / sizeof(Elf64_Rel))
3505 return (Elf64_Rel *)data->d_buf + idx;
3517 if (hdr->sh_type == SHT_STRTAB)
3521 if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3525 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3534 name += sizeof(".rel") - 1;
3553 if (a->sec_idx != b->sec_idx)
3554 return a->sec_idx < b->sec_idx ? -1 : 1;
3557 return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3564 Elf *elf = obj->efile.elf;
3566 pelfio_t elf = obj->efile.elf;
3582 /* ELF section indices are 0-based, but sec #0 is special "invalid"
3588 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3590 obj->path, elf_errmsg(-1));
3591 return -LIBBPF_ERRNO__FORMAT;
3594 obj->efile.sec_cnt = elfio_get_sections_num(elf);
3596 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3597 if (!obj->efile.secs)
3598 return -ENOMEM;
3614 return -LIBBPF_ERRNO__FORMAT;
3616 if (sh->sh_type == SHT_SYMTAB) {
3617 if (obj->efile.symbols) {
3618 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3619 return -LIBBPF_ERRNO__FORMAT;
3627 return -LIBBPF_ERRNO__FORMAT;
3633 obj->efile.symbols = data;
3635 obj->efile.realsymbols.d_buf = data->d_buf;
3636 obj->efile.realsymbols.d_size = data->d_size;
3637 obj->efile.symbols = &(obj->efile.realsymbols);
3641 obj->efile.symbols_shndx = idx;
3643 obj->efile.symbols_shndx = i;
3645 obj->efile.strtabidx = sh->sh_link;
3653 psection_t psection = elfio_get_section_by_index(elf, obj->efile.strtabidx);
3655 return -LIBBPF_ERRNO__FORMAT;
3658 psection = elfio_get_section_by_index(elf, obj->efile.shstrndx);
3660 return -LIBBPF_ERRNO__FORMAT;
3664 return -LIBBPF_ERRNO__FORMAT;
3665 obj->efile.strstring = strstring;
3666 obj->efile.shstring = shstring;
3669 if (!obj->efile.symbols) {
3671 obj->path);
3672 return -ENOENT;
3689 sec_desc = &obj->efile.secs[idx];
3698 return -LIBBPF_ERRNO__FORMAT;
3700 name = elf_sec_str(obj, sh->sh_name);
3702 return -LIBBPF_ERRNO__FORMAT;
3710 data = elf_sec_data_by_idx(obj, i, &sec_desc->realdata);
3713 return -LIBBPF_ERRNO__FORMAT;
3716 idx, name, (unsigned long)data->d_size,
3717 (int)sh->sh_link, (unsigned long)sh->sh_flags,
3718 (int)sh->sh_type);
3721 err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3725 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3730 return -ENOTSUP;
3732 obj->efile.btf_maps_shndx = idx;
3734 if (sh->sh_type != SHT_PROGBITS)
3735 return -LIBBPF_ERRNO__FORMAT;
3738 if (sh->sh_type != SHT_PROGBITS)
3739 return -LIBBPF_ERRNO__FORMAT;
3741 } else if (sh->sh_type == SHT_SYMTAB) {
3743 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
3744 if (sh->sh_flags & SHF_EXECINSTR) {
3746 obj->efile.text_shndx = idx;
3752 sec_desc->sec_type = SEC_DATA;
3754 sec_desc->shdr = sh;
3755 sec_desc->data = data;
3757 sec_desc->psection = ptmpsection;
3758 sec_desc->realdata.d_buf = data->d_buf;
3759 sec_desc->realdata.d_size = data->d_size;
3760 sec_desc->data = &(sec_desc->realdata);
3764 sec_desc->sec_type = SEC_RODATA;
3766 sec_desc->shdr = sh;
3767 sec_desc->data = data;
3769 sec_desc->psection = ptmpsection;
3770 sec_desc->realdata.d_buf = data->d_buf;
3771 sec_desc->realdata.d_size = data->d_size;
3772 sec_desc->data = &(sec_desc->realdata);
3777 obj->efile.st_ops_data = data;
3779 obj->efile.realst_ops_data.d_buf = data->d_buf;
3780 obj->efile.realst_ops_data.d_size = data->d_size;
3781 obj->efile.st_ops_data = &(obj->efile.realst_ops_data);
3783 obj->efile.st_ops_shndx = idx;
3788 } else if (sh->sh_type == SHT_REL) {
3789 int targ_sec_idx = sh->sh_info; /* points to other section */
3791 if (sh->sh_entsize != sizeof(Elf64_Rel) ||
3792 targ_sec_idx >= obj->efile.sec_cnt)
3793 return -LIBBPF_ERRNO__FORMAT;
3811 sec_desc->sec_type = SEC_RELO;
3813 sec_desc->shdr = sh;
3815 sec_desc->psection = ptmpsection;
3817 sec_desc->data = data;
3818 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||
3820 sec_desc->sec_type = SEC_BSS;
3822 sec_desc->shdr = sh;
3824 sec_desc->psection = ptmpsection;
3826 sec_desc->data = data;
3829 (size_t)sh->sh_size);
3833 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3834 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3835 return -LIBBPF_ERRNO__FORMAT;
3838 /* sort BPF programs by section name and in-section instruction offset
3841 if (obj->nr_programs)
3842 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3849 int bind = ELF64_ST_BIND(sym->st_info);
3851 return sym->st_shndx == SHN_UNDEF &&
3853 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
3858 int bind = ELF64_ST_BIND(sym->st_info);
3859 int type = ELF64_ST_TYPE(sym->st_info);
3862 if (sym->st_shndx != text_shndx)
3880 return -ESRCH;
3889 tname = btf__name_by_offset(btf, t->name_off);
3894 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
3895 return -EINVAL;
3898 return -EINVAL;
3903 return -ENOENT;
3912 return -ESRCH;
3923 if (vs->type == ext_btf_id)
3928 return -ENOENT;
3938 name = btf__name_by_offset(btf, t->name_off);
3947 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
3950 if (t->size == 1)
3952 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
3957 if (t->size != 4)
3967 if (btf_array(t)->nelems == 0)
3969 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
3982 if (a->type != b->type)
3983 return a->type < b->type ? -1 : 1;
3985 if (a->type == EXT_KCFG) {
3987 if (a->kcfg.align != b->kcfg.align)
3988 return a->kcfg.align > b->kcfg.align ? -1 : 1;
3990 if (a->kcfg.sz != b->kcfg.sz)
3991 return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
3995 return strcmp(a->name, b->name);
4033 vt = btf__type_by_id(btf, vs->type);
4066 if (!obj->efile.symbols)
4070 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
4074 sh = elf_sec_hdr_by_idx(obj, obj->efile.symbols_shndx, sh);
4077 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
4078 return -LIBBPF_ERRNO__FORMAT;
4080 dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
4084 n = sh->sh_size / sh->sh_entsize;
4091 return -LIBBPF_ERRNO__FORMAT;
4094 ext_name = elf_sym_str(obj, sym->st_name);
4098 ext = obj->externs;
4099 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
4101 return -ENOMEM;
4102 obj->externs = ext;
4103 ext = &ext[obj->nr_extern];
4105 obj->nr_extern++;
4107 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
4108 if (ext->btf_id <= 0) {
4110 ext_name, ext->btf_id);
4111 return ext->btf_id;
4113 t = btf__type_by_id(obj->btf, ext->btf_id);
4114 ext->name = btf__name_by_offset(obj->btf, t->name_off);
4115 ext->sym_idx = i;
4116 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
4118 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
4119 if (ext->sec_btf_id <= 0) {
4121 ext_name, ext->btf_id, ext->sec_btf_id);
4122 return ext->sec_btf_id;
4124 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
4125 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
4130 ext->name, KCONFIG_SEC);
4131 return -ENOTSUP;
4134 ext->type = EXT_KCFG;
4135 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
4136 if (ext->kcfg.sz <= 0) {
4138 ext_name, ext->kcfg.sz);
4139 return ext->kcfg.sz;
4141 ext->kcfg.align = btf__align_of(obj->btf, t->type);
4142 if (ext->kcfg.align <= 0) {
4144 ext_name, ext->kcfg.align);
4145 return -EINVAL;
4147 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
4148 &ext->kcfg.is_signed);
4149 if (ext->kcfg.type == KCFG_UNKNOWN) {
4151 return -ENOTSUP;
4155 ext->type = EXT_KSYM;
4156 skip_mods_and_typedefs(obj->btf, t->type,
4157 &ext->ksym.type_id);
4160 return -ENOTSUP;
4163 pr_debug("collected %d externs total\n", obj->nr_extern);
4165 if (!obj->nr_extern)
4169 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
4173 * pretending that each extern is a 8-byte variable
4176 /* find existing 4-byte integer type in BTF to use for fake
4179 int int_btf_id = find_int_btf_id(obj->btf);
4181 * will be used to replace the vs->type and
4187 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
4188 for (i = 0; i < obj->nr_extern; i++) {
4189 ext = &obj->externs[i];
4190 if (ext->type != EXT_KSYM)
4193 i, ext->sym_idx, ext->name);
4202 vt = (void *)btf__type_by_id(obj->btf, vs->type);
4203 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
4208 return -ESRCH;
4215 func_proto = btf__type_by_id(obj->btf,
4216 vt->type);
4224 dummy_var->name_off;
4225 vs->type = dummy_var_btf_id;
4226 vt->info &= ~0xffff;
4227 vt->info |= BTF_FUNC_GLOBAL;
4229 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4230 vt->type = int_btf_id;
4232 vs->offset = off;
4233 vs->size = sizeof(int);
4235 sec->size = off;
4242 for (i = 0; i < obj->nr_extern; i++) {
4243 ext = &obj->externs[i];
4244 if (ext->type != EXT_KCFG)
4247 ext->kcfg.data_off = roundup(off, ext->kcfg.align);
4248 off = ext->kcfg.data_off + ext->kcfg.sz;
4250 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
4252 sec->size = off;
4257 t = btf__type_by_id(obj->btf, vs->type);
4258 ext_name = btf__name_by_offset(obj->btf, t->name_off);
4263 return -ESRCH;
4265 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4266 vs->offset = ext->kcfg.data_off;
4274 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
4286 if (!strcmp(prog->name, name))
4295 switch (obj->efile.secs[shndx].sec_type) {
4308 return shndx == obj->efile.btf_maps_shndx;
4314 if (shndx == obj->efile.symbols_shndx)
4317 switch (obj->efile.secs[shndx].sec_type) {
4334 struct bpf_insn *insn = &prog->insns[insn_idx];
4335 size_t map_idx, nr_maps = prog->obj->nr_maps;
4336 struct bpf_object *obj = prog->obj;
4337 __u32 shdr_idx = sym->st_shndx;
4344 prog->name, sym_name, insn_idx, insn->code);
4345 return -LIBBPF_ERRNO__RELOC;
4349 int sym_idx = ELF64_R_SYM(rel->r_info);
4350 int i, n = obj->nr_extern;
4354 ext = &obj->externs[i];
4355 if (ext->sym_idx == sym_idx)
4360 prog->name, sym_name, sym_idx);
4361 return -LIBBPF_ERRNO__RELOC;
4364 prog->name, i, ext->name, ext->sym_idx, insn_idx);
4365 if (insn->code == (BPF_JMP | BPF_CALL))
4366 reloc_desc->type = RELO_EXTERN_FUNC;
4368 reloc_desc->type = RELO_EXTERN_VAR;
4369 reloc_desc->insn_idx = insn_idx;
4370 reloc_desc->sym_off = i; /* sym_off stores extern index */
4374 /* sub-program call relocation */
4376 if (insn->src_reg != BPF_PSEUDO_CALL) {
4377 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4378 return -LIBBPF_ERRNO__RELOC;
4381 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
4388 prog->name, sym_name, sym_sec_name);
4389 return -LIBBPF_ERRNO__RELOC;
4391 if (sym->st_value % BPF_INSN_SZ) {
4393 prog->name, sym_name, (size_t)sym->st_value);
4394 return -LIBBPF_ERRNO__RELOC;
4396 reloc_desc->type = RELO_CALL;
4397 reloc_desc->insn_idx = insn_idx;
4398 reloc_desc->sym_off = sym->st_value;
4404 prog->name, sym_name, shdr_idx);
4405 return -LIBBPF_ERRNO__RELOC;
4409 if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4410 /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4411 * local_func: sym->st_value = 0, insn->imm = offset in the section.
4413 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4415 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4416 return -LIBBPF_ERRNO__RELOC;
4419 reloc_desc->type = RELO_SUBPROG_ADDR;
4420 reloc_desc->insn_idx = insn_idx;
4421 reloc_desc->sym_off = sym->st_value;
4435 prog->name, sym_name, sym_sec_name);
4436 return -LIBBPF_ERRNO__RELOC;
4439 map = &obj->maps[map_idx];
4440 if (map->libbpf_type != type ||
4441 map->sec_idx != sym->st_shndx ||
4442 map->sec_offset != sym->st_value)
4445 prog->name, map_idx, map->name, map->sec_idx,
4446 map->sec_offset, insn_idx);
4451 prog->name, sym_sec_name, (size_t)sym->st_value);
4452 return -LIBBPF_ERRNO__RELOC;
4454 reloc_desc->type = RELO_LD64;
4455 reloc_desc->insn_idx = insn_idx;
4456 reloc_desc->map_idx = map_idx;
4457 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4464 prog->name, sym_sec_name);
4465 return -LIBBPF_ERRNO__RELOC;
4468 map = &obj->maps[map_idx];
4469 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4472 prog->name, map_idx, map->name, map->sec_idx,
4473 map->sec_offset, insn_idx);
4478 prog->name, sym_sec_name);
4479 return -LIBBPF_ERRNO__RELOC;
4482 reloc_desc->type = RELO_DATA;
4483 reloc_desc->insn_idx = insn_idx;
4484 reloc_desc->map_idx = map_idx;
4485 reloc_desc->sym_off = sym->st_value;
4491 return insn_idx >= prog->sec_insn_off &&
4492 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4498 int l = 0, r = obj->nr_programs - 1, m;
4501 if (!obj->nr_programs)
4505 m = l + (r - l + 1) / 2;
4506 prog = &obj->programs[m];
4508 if (prog->sec_idx < sec_idx ||
4509 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4512 r = m - 1;
4517 prog = &obj->programs[l];
4518 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4527 size_t sec_idx = shdr->sh_info, sym_idx;
4540 if (sec_idx >= obj->efile.sec_cnt)
4541 return -EINVAL;
4547 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4550 return -EINVAL;
4555 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4558 return -EINVAL;
4563 nrels = shdr->sh_size / shdr->sh_entsize;
4569 return -LIBBPF_ERRNO__FORMAT;
4572 sym_idx = ELF64_R_SYM(rel->r_info);
4577 return -LIBBPF_ERRNO__FORMAT;
4580 if (sym->st_shndx >= obj->efile.sec_cnt) {
4582 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4583 return -LIBBPF_ERRNO__FORMAT;
4586 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4588 relo_sec_name, (size_t)rel->r_offset, i);
4589 return -LIBBPF_ERRNO__FORMAT;
4592 insn_idx = rel->r_offset / BPF_INSN_SZ;
4599 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4601 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4603 sym_name = elf_sec_name_by_idx(obj, sym->st_shndx);
4606 sym_name = elf_sym_str(obj, sym->st_name);
4619 relos = libbpf_reallocarray(prog->reloc_desc,
4620 prog->nr_reloc + 1, sizeof(*relos));
4622 return -ENOMEM;
4623 prog->reloc_desc = relos;
4626 insn_idx -= prog->sec_insn_off;
4627 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4632 prog->nr_reloc++;
4641 if (!obj->btf)
4642 return -ENOENT;
4644 /* if it's BTF-defined map, we don't need to search for type IDs.
4648 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4656 return -ENOENT;
4658 id = btf__find_by_name(obj->btf, map->real_name);
4662 map->btf_key_type_id = 0;
4663 map->btf_value_type_id = id;
4679 err = -errno;
4687 info->type = val;
4689 info->key_size = val;
4691 info->value_size = val;
4693 info->max_entries = val;
4695 info->map_flags = val;
4705 return map->autocreate;
4710 if (map->obj->loaded)
4711 return libbpf_err(-EBUSY);
4713 map->autocreate = autocreate;
4732 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
4733 new_name = strdup(map->name);
4738 return libbpf_err(-errno);
4742 err = -errno;
4748 err = -errno;
4752 err = zclose(map->fd);
4754 err = -errno;
4757 free(map->name);
4759 map->fd = new_fd;
4760 map->name = new_name;
4761 map->def.type = info.type;
4762 map->def.key_size = info.key_size;
4763 map->def.value_size = info.value_size;
4764 map->def.max_entries = info.max_entries;
4765 map->def.map_flags = info.map_flags;
4766 map->btf_key_type_id = info.btf_key_type_id;
4767 map->btf_value_type_id = info.btf_value_type_id;
4768 map->reused = true;
4769 map->map_extra = info.map_extra;
4782 return map->def.max_entries;
4787 if (!bpf_map_type__is_map_in_map(map->def.type))
4790 return map->inner_map;
4795 if (map->obj->loaded)
4796 return libbpf_err(-EBUSY);
4798 map->def.max_entries = max_entries;
4800 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
4802 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
4817 if (obj->gen_loader)
4835 return -ret;
4884 ret = -errno;
4887 __func__, cp, -ret);
4987 BTF_TYPE_DECL_TAG_ENC(1, 2, -1),
5029 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
5041 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */
5064 ret = -errno;
5067 __func__, cp, -ret);
5125 return -errno;
5130 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL);
5131 err = -errno; /* close() can clobber errno */
5137 return link_fd < 0 && err == -EBADF;
5225 "memcg-based memory accounting", probe_memcg_account,
5243 if (obj && obj->gen_loader)
5249 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
5250 ret = feat->probe();
5252 WRITE_ONCE(feat->res, FEAT_SUPPORTED);
5254 WRITE_ONCE(feat->res, FEAT_MISSING);
5256 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
5257 WRITE_ONCE(feat->res, FEAT_MISSING);
5261 return READ_ONCE(feat->res) == FEAT_SUPPORTED;
5281 return (map_info.type == map->def.type &&
5282 map_info.key_size == map->def.key_size &&
5283 map_info.value_size == map->def.value_size &&
5284 map_info.max_entries == map->def.max_entries &&
5285 map_info.map_flags == map->def.map_flags &&
5286 map_info.map_extra == map->map_extra);
5295 pin_fd = bpf_obj_get(map->pin_path);
5297 err = -errno;
5298 if (err == -ENOENT) {
5300 map->pin_path);
5304 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
5306 map->pin_path, cp);
5312 map->pin_path);
5314 return -EINVAL;
5322 map->pinned = true;
5323 pr_debug("reused pinned map at '%s'\n", map->pin_path);
5331 enum libbpf_map_type map_type = map->libbpf_type;
5335 if (obj->gen_loader) {
5336 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
5337 map->mmaped, map->def.value_size);
5339 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
5342 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
5344 err = -errno;
5347 map->name, cp);
5351 /* Freeze .rodata and .kconfig map as read-only from syscall side. */
5353 err = bpf_map_freeze(map->fd);
5355 err = -errno;
5357 pr_warn("Error freezing map(%s) as read-only: %s\n",
5358 map->name, cp);
5370 struct bpf_map_def *def = &map->def;
5375 map_name = map->name;
5376 create_attr.map_ifindex = map->map_ifindex;
5377 create_attr.map_flags = def->map_flags;
5378 create_attr.numa_node = map->numa_node;
5379 create_attr.map_extra = map->map_extra;
5382 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
5384 if (obj->btf && btf__fd(obj->btf) >= 0) {
5385 create_attr.btf_fd = btf__fd(obj->btf);
5386 create_attr.btf_key_type_id = map->btf_key_type_id;
5387 create_attr.btf_value_type_id = map->btf_value_type_id;
5390 if (bpf_map_type__is_map_in_map(def->type)) {
5391 if (map->inner_map) {
5392 err = bpf_object__create_map(obj, map->inner_map, true);
5395 map->name, err);
5398 map->inner_map_fd = bpf_map__fd(map->inner_map);
5400 if (map->inner_map_fd >= 0)
5401 create_attr.inner_map_fd = map->inner_map_fd;
5404 switch (def->type) {
5421 map->btf_key_type_id = 0;
5422 map->btf_value_type_id = 0;
5428 if (obj->gen_loader) {
5429 bpf_gen__map_create(obj->gen_loader, def->type, map_name,
5430 def->key_size, def->value_size, def->max_entries,
5431 &create_attr, is_inner ? -1 : map - obj->maps);
5433 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
5435 map->fd = 0;
5437 map->fd = bpf_map_create(def->type, map_name,
5438 def->key_size, def->value_size,
5439 def->max_entries, &create_attr);
5441 if (map->fd < 0 && (create_attr.btf_key_type_id ||
5445 err = -errno;
5448 map->name, cp, err);
5452 map->btf_key_type_id = 0;
5453 map->btf_value_type_id = 0;
5454 map->fd = bpf_map_create(def->type, map_name,
5455 def->key_size, def->value_size,
5456 def->max_entries, &create_attr);
5459 err = map->fd < 0 ? -errno : 0;
5461 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
5462 if (obj->gen_loader)
5463 map->inner_map->fd = -1;
5464 bpf_map__destroy(map->inner_map);
5465 zfree(&map->inner_map);
5477 for (i = 0; i < map->init_slots_sz; i++) {
5478 if (!map->init_slots[i])
5481 targ_map = map->init_slots[i];
5484 if (obj->gen_loader) {
5485 bpf_gen__populate_outer_map(obj->gen_loader,
5486 map - obj->maps, i,
5487 targ_map - obj->maps);
5489 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5492 err = -errno;
5494 map->name, i, targ_map->name, fd, err);
5498 map->name, i, targ_map->name, fd);
5501 zfree(&map->init_slots);
5502 map->init_slots_sz = 0;
5513 if (obj->gen_loader)
5514 return -ENOTSUP;
5516 for (i = 0; i < map->init_slots_sz; i++) {
5517 if (!map->init_slots[i])
5520 targ_prog = map->init_slots[i];
5523 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5525 err = -errno;
5527 map->name, i, targ_prog->name, fd, err);
5531 map->name, i, targ_prog->name, fd);
5534 zfree(&map->init_slots);
5535 map->init_slots_sz = 0;
5545 for (i = 0; i < obj->nr_maps; i++) {
5546 map = &obj->maps[i];
5548 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
5553 zclose(map->fd);
5562 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
5568 map->name, nr_cpus);
5571 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
5572 map->def.max_entries = nr_cpus;
5587 for (i = 0; i < obj->nr_maps; i++) {
5588 map = &obj->maps[i];
5592 * loading, if we detect that at least one of the to-be-loaded
5597 * but also it allows to have CO-RE applications that use
5599 * If those global variable-using programs are not loaded at
5605 map->autocreate = false;
5607 if (!map->autocreate) {
5608 pr_debug("map '%s': skipped auto-creating...\n", map->name);
5618 if (map->pin_path) {
5622 map->name);
5625 if (retried && map->fd < 0) {
5627 map->name);
5628 err = -ENOENT;
5633 if (map->fd >= 0) {
5635 map->name, map->fd);
5642 map->name, map->fd);
5647 zclose(map->fd);
5652 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
5655 zclose(map->fd);
5661 if (map->pin_path && !map->pinned) {
5664 zclose(map->fd);
5665 if (!retried && err == -EEXIST) {
5669 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
5670 map->name, map->pin_path, err);
5680 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
5683 zclose(obj->maps[j].fd);
5697 * underscore is ignored by BPF CO-RE relocation during relocation matching.
5704 for (i = n - 5; i >= 0; i--) {
5716 free(cands->cands);
5733 local_t = btf__type_by_id(local_cand->btf, local_cand->id);
5734 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
5742 targ_name = btf__name_by_offset(targ_btf, t->name_off);
5753 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
5754 local_cand->id, btf_kind_str(local_t),
5757 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
5758 sizeof(*cands->cands));
5760 return -ENOMEM;
5762 cand = &new_cands[cands->len];
5763 cand->btf = targ_btf;
5764 cand->id = i;
5766 cands->cands = new_cands;
5767 cands->len++;
5781 if (obj->btf_modules_loaded)
5784 if (obj->gen_loader)
5788 obj->btf_modules_loaded = true;
5799 err = -errno;
5808 err = -errno;
5820 err = -errno;
5825 /* ignore non-module BTFs */
5831 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5839 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5840 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5844 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
5846 mod_btf->btf = btf;
5847 mod_btf->id = id;
5848 mod_btf->fd = fd;
5849 mod_btf->name = strdup(name);
5850 if (!mod_btf->name) {
5851 err = -ENOMEM;
5879 return ERR_PTR(-EINVAL);
5881 local_name = btf__name_by_offset(local_btf, local_t->name_off);
5883 return ERR_PTR(-EINVAL);
5888 return ERR_PTR(-ENOMEM);
5891 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
5897 if (cands->len)
5901 if (obj->btf_vmlinux_override)
5909 for (i = 0; i < obj->btf_module_cnt; i++) {
5911 obj->btf_modules[i].btf,
5912 obj->btf_modules[i].name,
5913 btf__type_cnt(obj->btf_vmlinux),
5926 * type-based CO-RE relocations and follow slightly different rules than
5927 * field-based relocations. This function assumes that root types were already
5928 * checked for name match. Beyond that initial root-level name check, names
5930 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5933 * - for ENUMs, the size is ignored;
5934 * - for INT, size and signedness are ignored;
5935 * - for ARRAY, dimensionality is ignored, element types are checked for
5937 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
5938 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5939 * - FUNC_PROTOs are compatible if they have compatible signature: same
5942 * more experience with using BPF CO-RE relocations.
5971 relos = libbpf_reallocarray(prog->reloc_desc,
5972 prog->nr_reloc + 1, sizeof(*relos));
5974 return -ENOMEM;
5975 relo = &relos[prog->nr_reloc];
5976 relo->type = RELO_CORE;
5977 relo->insn_idx = insn_idx;
5978 relo->core_relo = core_relo;
5979 prog->reloc_desc = relos;
5980 prog->nr_reloc++;
5989 for (i = 0; i < prog->nr_reloc; i++) {
5990 relo = &prog->reloc_desc[i];
5991 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)
5994 return relo->core_relo;
6009 const char *prog_name = prog->name;
6012 __u32 local_id = relo->type_id;
6017 return -EINVAL;
6019 local_name = btf__name_by_offset(local_btf, local_type->name_off);
6021 return -EINVAL;
6023 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
6025 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
6057 if (obj->btf_ext->core_relo_info.len == 0)
6062 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
6064 err = libbpf_get_error(obj->btf_vmlinux_override);
6077 seg = &obj->btf_ext->core_relo_info;
6080 sec_idx = seg->sec_idxs[sec_num];
6083 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6085 err = -EINVAL;
6089 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);
6092 if (rec->insn_off % BPF_INSN_SZ)
6093 return -EINVAL;
6094 insn_idx = rec->insn_off / BPF_INSN_SZ;
6101 * This is similar to what x86-64 linker does for relocations.
6105 …pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subpro…
6109 /* no need to apply CO-RE relocation if the program is
6112 if (!prog->autoload)
6116 * program's frame of reference; (sub-)program code is not yet
6117 * relocated, so it's enough to just subtract in-section offset
6119 insn_idx = insn_idx - prog->sec_insn_off;
6120 if (insn_idx >= prog->insns_cnt)
6121 return -EINVAL;
6122 insn = &prog->insns[insn_idx];
6127 prog->name, i, err);
6131 if (prog->obj->gen_loader)
6134 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
6137 prog->name, i, err);
6141 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
6144 prog->name, i, insn_idx, err);
6151 /* obj->btf_vmlinux and module BTFs are freed after object load */
6152 btf__free(obj->btf_vmlinux_override);
6153 obj->btf_vmlinux_override = NULL;
6157 bpf_core_free_cands(entry->pvalue);
6175 prog->name, relo_idx, insn_idx, map_idx, map->name);
6179 insn->code = BPF_JMP | BPF_CALL;
6180 insn->dst_reg = 0;
6181 insn->src_reg = 0;
6182 insn->off = 0;
6185 * invalid func unknown#2001000123
6186 * where lower 123 is map index into obj->maps[] array
6188 insn->imm = MAP_LDIMM64_POISON_BASE + map_idx;
6195 * - map references;
6196 * - global variable references;
6197 * - extern references.
6204 for (i = 0; i < prog->nr_reloc; i++) {
6205 struct reloc_desc *relo = &prog->reloc_desc[i];
6206 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6210 switch (relo->type) {
6212 map = &obj->maps[relo->map_idx];
6213 if (obj->gen_loader) {
6215 insn[0].imm = relo->map_idx;
6216 } else if (map->autocreate) {
6218 insn[0].imm = map->fd;
6220 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6221 relo->map_idx, map);
6225 map = &obj->maps[relo->map_idx];
6226 insn[1].imm = insn[0].imm + relo->sym_off;
6227 if (obj->gen_loader) {
6229 insn[0].imm = relo->map_idx;
6230 } else if (map->autocreate) {
6232 insn[0].imm = map->fd;
6234 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6235 relo->map_idx, map);
6239 ext = &obj->externs[relo->sym_off];
6240 if (ext->type == EXT_KCFG) {
6241 if (obj->gen_loader) {
6243 insn[0].imm = obj->kconfig_map_idx;
6246 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6248 insn[1].imm = ext->kcfg.data_off;
6250 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
6252 insn[0].imm = ext->ksym.kernel_btf_id;
6253 insn[1].imm = ext->ksym.kernel_btf_obj_fd;
6255 insn[0].imm = (__u32)ext->ksym.addr;
6256 insn[1].imm = ext->ksym.addr >> 32;
6261 ext = &obj->externs[relo->sym_off];
6263 if (ext->is_set) {
6264 insn[0].imm = ext->ksym.kernel_btf_id;
6265 insn[0].off = ext->ksym.btf_fd_idx;
6274 prog->name, i);
6275 return -EINVAL;
6287 prog->name, i, relo->type);
6288 return -EINVAL;
6309 sec_idx = ext_info->sec_idxs[sec_num];
6311 if (prog->sec_idx != sec_idx)
6317 if (insn_off < prog->sec_insn_off)
6319 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6324 copy_end = rec + ext_info->rec_size;
6328 return -ENOENT;
6330 /* append func/line info of a given (sub-)program to the main
6333 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
6334 new_sz = old_sz + (copy_end - copy_start);
6337 return -ENOMEM;
6339 *prog_rec_cnt = new_sz / ext_info->rec_size;
6340 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6342 /* Kernel instruction offsets are in units of 8-byte
6348 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6351 for (; rec < rec_end; rec += ext_info->rec_size) {
6356 *prog_rec_sz = ext_info->rec_size;
6360 return -ENOENT;
6373 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
6379 if (main_prog != prog && !main_prog->func_info)
6382 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6383 &main_prog->func_info,
6384 &main_prog->func_info_cnt,
6385 &main_prog->func_info_rec_size);
6387 if (err != -ENOENT) {
6389 prog->name, err);
6392 if (main_prog->func_info) {
6397 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6402 prog->name);
6407 if (main_prog != prog && !main_prog->line_info)
6410 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6411 &main_prog->line_info,
6412 &main_prog->line_info_cnt,
6413 &main_prog->line_info_rec_size);
6415 if (err != -ENOENT) {
6417 prog->name, err);
6420 if (main_prog->line_info) {
6425 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6430 prog->name);
6440 if (insn_idx == relo->insn_idx)
6442 return insn_idx < relo->insn_idx ? -1 : 1;
6447 if (!prog->nr_reloc)
6449 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6450 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6455 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
6461 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
6463 return -ENOMEM;
6464 if (subprog->nr_reloc)
6465 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
6466 sizeof(*relos) * subprog->nr_reloc);
6468 for (i = main_prog->nr_reloc; i < new_cnt; i++)
6469 relos[i].insn_idx += subprog->sub_insn_off;
6473 main_prog->reloc_desc = relos;
6474 main_prog->nr_reloc = new_cnt;
6492 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6493 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6498 if (relo && relo->type == RELO_EXTERN_FUNC)
6503 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6505 prog->name, insn_idx, relo->type);
6506 return -LIBBPF_ERRNO__RELOC;
6509 /* sub-program instruction index is a combination of
6512 * call always has imm = -1, but for static functions
6513 * relocation is against STT_SECTION and insn->imm
6516 * for subprog addr relocation, the relo->sym_off + insn->imm is
6519 if (relo->type == RELO_CALL)
6520 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6522 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6529 prog->name, insn_idx);
6530 return -LIBBPF_ERRNO__RELOC;
6535 * offset necessary, insns->imm is relative to
6538 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6541 /* we enforce that sub-programs should be in .text section */
6542 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6544 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6545 prog->name);
6546 return -LIBBPF_ERRNO__RELOC;
6552 * - append it at the end of main program's instructions blog;
6553 * - process is recursively, while current program is put on hold;
6554 * - if that subprogram calls some other not yet processes
6559 if (subprog->sub_insn_off == 0) {
6560 subprog->sub_insn_off = main_prog->insns_cnt;
6562 new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6563 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6565 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6566 return -ENOMEM;
6568 main_prog->insns = insns;
6569 main_prog->insns_cnt = new_cnt;
6571 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6572 subprog->insns_cnt * sizeof(*insns));
6574 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6575 main_prog->name, subprog->insns_cnt, subprog->name);
6586 /* main_prog->insns memory could have been re-allocated, so
6589 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6596 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6599 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6606 * Relocate sub-program calls.
6608 * Algorithm operates as follows. Each entry-point BPF program (referred to as
6609 * main prog) is processed separately. For each subprog (non-entry functions,
6618 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6634 * subprog->sub_insn_off as zero at all times and won't be appended to current
6643 * +--------+ +-------+
6645 * +--+---+ +--+-+-+ +---+--+
6647 * +--+---+ +------+ +---+--+
6650 * +---+-------+ +------+----+
6652 * +-----------+ +-----------+
6657 * +-----------+------+
6659 * +-----------+------+
6664 * +-----------+------+------+
6666 * +-----------+------+------+
6675 * +-----------+------+
6677 * +-----------+------+
6680 * +-----------+------+------+
6682 * +-----------+------+------+
6695 for (i = 0; i < obj->nr_programs; i++) {
6696 subprog = &obj->programs[i];
6700 subprog->sub_insn_off = 0;
6717 for (i = 0; i < obj->nr_programs; i++) {
6718 prog = &obj->programs[i];
6719 zfree(&prog->reloc_desc);
6720 prog->nr_reloc = 0;
6729 if (a->insn_idx != b->insn_idx)
6730 return a->insn_idx < b->insn_idx ? -1 : 1;
6733 if (a->type != b->type)
6734 return a->type < b->type ? -1 : 1;
6743 for (i = 0; i < obj->nr_programs; i++) {
6744 struct bpf_program *p = &obj->programs[i];
6746 if (!p->nr_reloc)
6749 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6760 if (obj->btf_ext) {
6763 pr_warn("failed to perform CO-RE relocations: %d\n",
6770 /* Before relocating calls pre-process relocations and mark
6777 for (i = 0; i < obj->nr_programs; i++) {
6778 prog = &obj->programs[i];
6779 for (j = 0; j < prog->nr_reloc; j++) {
6780 struct reloc_desc *relo = &prog->reloc_desc[j];
6781 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6784 if (relo->type == RELO_SUBPROG_ADDR)
6796 for (i = 0; i < obj->nr_programs; i++) {
6797 prog = &obj->programs[i];
6798 /* sub-program's sub-calls are relocated within the context of
6803 if (!prog->autoload)
6809 prog->name, err);
6814 for (i = 0; i < obj->nr_programs; i++) {
6815 prog = &obj->programs[i];
6818 if (!prog->autoload)
6823 prog->name, err);
6851 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
6852 return -EINVAL;
6853 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
6855 return -EINVAL;
6857 nrels = shdr->sh_size / shdr->sh_entsize;
6862 return -LIBBPF_ERRNO__FORMAT;
6865 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
6868 i, (size_t)ELF64_R_SYM(rel->r_info));
6869 return -LIBBPF_ERRNO__FORMAT;
6871 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
6873 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",
6874 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
6875 (size_t)rel->r_offset, sym->st_name, name);
6877 for (j = 0; j < obj->nr_maps; j++) {
6878 map = &obj->maps[j];
6879 if (map->sec_idx != obj->efile.btf_maps_shndx)
6882 vi = btf_var_secinfos(sec) + map->btf_var_idx;
6883 if (vi->offset <= rel->r_offset &&
6884 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
6887 if (j == obj->nr_maps) {
6888 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
6889 i, name, (size_t)rel->r_offset);
6890 return -EINVAL;
6893 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
6894 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
6897 if (sym->st_shndx != obj->efile.btf_maps_shndx) {
6898 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
6900 return -LIBBPF_ERRNO__RELOC;
6902 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
6903 map->def.key_size != sizeof(int)) {
6904 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
6905 i, map->name, sizeof(int));
6906 return -EINVAL;
6912 return -ESRCH;
6919 return -ESRCH;
6921 if (targ_prog->sec_idx != sym->st_shndx ||
6922 targ_prog->sec_insn_off * 8 != sym->st_value ||
6924 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
6926 return -LIBBPF_ERRNO__RELOC;
6929 return -EINVAL;
6932 var = btf__type_by_id(obj->btf, vi->type);
6933 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
6935 return -EINVAL;
6936 member = btf_members(def) + btf_vlen(def) - 1;
6937 mname = btf__name_by_offset(obj->btf, member->name_off);
6939 return -EINVAL;
6941 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
6942 if (rel->r_offset - vi->offset < moff)
6943 return -EINVAL;
6945 moff = rel->r_offset - vi->offset - moff;
6950 return -EINVAL;
6952 if (moff >= map->init_slots_sz) {
6954 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
6956 return -ENOMEM;
6957 map->init_slots = tmp;
6958 memset(map->init_slots + map->init_slots_sz, 0,
6959 (new_sz - map->init_slots_sz) * host_ptr_sz);
6960 map->init_slots_sz = new_sz;
6962 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
6965 i, map->name, moff, type, name);
6975 for (i = 0; i < obj->efile.sec_cnt; i++) {
6976 struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
6982 if (sec_desc->sec_type != SEC_RELO)
6986 shdr = sec_desc->shdr;
6990 data = sec_desc->data;
6991 idx = shdr->sh_info;
6993 if (shdr->sh_type != SHT_REL) {
6995 return -LIBBPF_ERRNO__INTERNAL;
6998 if (idx == obj->efile.st_ops_shndx)
7000 else if (idx == obj->efile.btf_maps_shndx)
7014 if (BPF_CLASS(insn->code) == BPF_JMP &&
7015 BPF_OP(insn->code) == BPF_CALL &&
7016 BPF_SRC(insn->code) == BPF_K &&
7017 insn->src_reg == 0 &&
7018 insn->dst_reg == 0) {
7019 *func_id = insn->imm;
7027 struct bpf_insn *insn = prog->insns;
7031 if (obj->gen_loader)
7034 for (i = 0; i < prog->insns_cnt; i++, insn++) {
7046 insn->imm = BPF_FUNC_probe_read;
7051 insn->imm = BPF_FUNC_probe_read_str;
7063 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
7070 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
7071 opts->expected_attach_type = 0;
7074 opts->prog_flags |= BPF_F_SLEEPABLE;
7076 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
7077 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
7079 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
7083 attach_name = strchr(prog->sec_name, '/');
7094 …pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\…
7095 prog->name);
7096 return -EINVAL;
7105 prog->attach_btf_obj_fd = btf_obj_fd;
7106 prog->attach_btf_id = btf_type_id;
7109 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
7113 opts->attach_btf_obj_fd = btf_obj_fd;
7114 opts->attach_btf_id = btf_type_id;
7132 __u32 log_level = prog->log_level;
7134 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
7140 prog->name, prog->sec_name);
7141 return -EINVAL;
7145 return -EINVAL;
7147 load_attr.expected_attach_type = prog->expected_attach_type;
7149 prog_name = prog->name;
7150 load_attr.attach_prog_fd = prog->attach_prog_fd;
7151 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
7152 load_attr.attach_btf_id = prog->attach_btf_id;
7154 load_attr.prog_ifindex = prog->prog_ifindex;
7160 load_attr.func_info = prog->func_info;
7161 load_attr.func_info_rec_size = prog->func_info_rec_size;
7162 load_attr.func_info_cnt = prog->func_info_cnt;
7163 load_attr.line_info = prog->line_info;
7164 load_attr.line_info_rec_size = prog->line_info_rec_size;
7165 load_attr.line_info_cnt = prog->line_info_cnt;
7168 load_attr.prog_flags = prog->prog_flags;
7169 load_attr.fd_array = obj->fd_array;
7172 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
7173 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
7176 prog->name, err);
7179 insns = prog->insns;
7180 insns_cnt = prog->insns_cnt;
7183 if (obj->gen_loader) {
7184 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
7186 prog - obj->programs);
7187 *prog_fd = -1;
7198 if (prog->log_buf) {
7199 log_buf = prog->log_buf;
7200 log_buf_size = prog->log_size;
7202 } else if (obj->log_buf) {
7203 log_buf = obj->log_buf;
7204 log_buf_size = obj->log_size;
7210 ret = -ENOMEM;
7223 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
7226 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7227 prog->name, log_buf);
7230 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
7234 for (i = 0; i < obj->nr_maps; i++) {
7235 map = &prog->obj->maps[i];
7236 if (map->libbpf_type != LIBBPF_MAP_RODATA)
7242 prog->name, map->real_name, cp);
7262 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
7267 ret = -errno;
7269 /* post-process verifier log to improve error descriptions */
7273 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
7277 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7278 prog->name, log_buf);
7294 p = cur - 1;
7295 while (p - 1 >= buf && *(p - 1) != '\n')
7296 p--;
7304 /* size of the remaining log content to the right from the to-be-replaced part */
7305 size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
7310 * shift log contents by (patch_sz - orig_sz) bytes to the right
7311 * starting from after to-be-replaced part of the log.
7314 * shift log contents by (orig_sz - patch_sz) bytes to the left
7315 * starting from after to-be-replaced part of the log
7324 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
7326 } else if (patch_sz - orig_sz > buf_sz - log_sz) {
7328 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
7342 /* Expected log for failed and not properly guarded CO-RE relocation:
7343 * line1 -> 123: (85) call unknown#195896080
7344 * line2 -> invalid func unknown#195896080
7345 * line3 -> <anything else or end of buffer>
7348 * instruction index to find corresponding CO-RE relocation and
7350 * failed CO-RE relocation.
7357 if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1)
7364 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
7370 "%d: <invalid CO-RE relocation>\n"
7371 "failed to resolve CO-RE relocation %s%s\n",
7374 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7381 /* Expected log for failed and not properly guarded CO-RE relocation:
7382 * line1 -> 123: (85) call unknown#2001000345
7383 * line2 -> invalid func unknown#2001000345
7384 * line3 -> <anything else or end of buffer>
7387 * "345" in "2001000345" are map index in obj->maps to fetch map name.
7389 struct bpf_object *obj = prog->obj;
7394 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2)
7397 map_idx -= MAP_LDIMM64_POISON_BASE;
7398 if (map_idx < 0 || map_idx >= obj->nr_maps)
7400 map = &obj->maps[map_idx];
7405 insn_idx, map->name);
7407 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7422 next_line = buf + log_sz - 1;
7429 /* failed CO-RE relocation case */
7430 if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) {
7438 } else if (str_has_pfx(cur_line, "invalid func unknown#"MAP_LDIMM64_POISON_PFX)) {
7452 struct bpf_object *obj = prog->obj;
7455 for (i = 0; i < prog->nr_reloc; i++) {
7456 struct reloc_desc *relo = &prog->reloc_desc[i];
7457 struct extern_desc *ext = &obj->externs[relo->sym_off];
7459 switch (relo->type) {
7461 if (ext->type != EXT_KSYM)
7463 bpf_gen__record_extern(obj->gen_loader, ext->name,
7464 ext->is_weak, !ext->ksym.type_id,
7465 BTF_KIND_VAR, relo->insn_idx);
7468 bpf_gen__record_extern(obj->gen_loader, ext->name,
7469 ext->is_weak, false, BTF_KIND_FUNC,
7470 relo->insn_idx);
7474 .insn_off = relo->insn_idx * 8,
7475 .type_id = relo->core_relo->type_id,
7476 .access_str_off = relo->core_relo->access_str_off,
7477 .kind = relo->core_relo->kind,
7480 bpf_gen__record_relo_core(obj->gen_loader, &cr);
7497 for (i = 0; i < obj->nr_programs; i++) {
7498 prog = &obj->programs[i];
7504 for (i = 0; i < obj->nr_programs; i++) {
7505 prog = &obj->programs[i];
7508 if (!prog->autoload) {
7509 pr_debug("prog '%s': skipped loading\n", prog->name);
7512 prog->log_level |= log_level;
7514 if (obj->gen_loader)
7517 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7518 obj->license, obj->kern_version, &prog->fd);
7520 pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
7537 prog->sec_def = find_sec_def(prog->sec_name);
7538 if (!prog->sec_def) {
7541 prog->name, prog->sec_name);
7545 prog->type = prog->sec_def->prog_type;
7546 prog->expected_attach_type = prog->sec_def->expected_attach_type;
7551 if (prog->sec_def->prog_setup_fn) {
7552 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
7555 prog->name, err);
7579 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7584 return ERR_PTR(-EINVAL);
7589 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
7602 return ERR_PTR(-EINVAL);
7604 return ERR_PTR(-EINVAL);
7610 obj->log_buf = log_buf;
7611 obj->log_size = log_size;
7612 obj->log_level = log_level;
7617 err = -ENAMETOOLONG;
7620 obj->btf_custom_path = strdup(btf_tmp_path);
7621 if (!obj->btf_custom_path) {
7622 err = -ENOMEM;
7629 obj->kconfig = strdup(kconfig);
7630 if (!obj->kconfig) {
7631 err = -ENOMEM;
7659 return libbpf_err_ptr(-EINVAL);
7676 return libbpf_err_ptr(-EINVAL);
7686 return libbpf_err(-EINVAL);
7688 for (i = 0; i < obj->nr_maps; i++) {
7689 zclose(obj->maps[i].fd);
7690 if (obj->maps[i].st_ops)
7691 zfree(&obj->maps[i].st_ops->kern_vdata);
7694 for (i = 0; i < obj->nr_programs; i++)
7695 bpf_program__unload(&obj->programs[i]);
7708 m->def.map_flags ^= BPF_F_MMAPABLE;
7723 err = -errno;
7735 err = -EINVAL;
7756 if (!ext || ext->type != EXT_KSYM)
7759 t = btf__type_by_id(obj->btf, ext->btf_id);
7763 if (ext->is_set && ext->ksym.addr != sym_addr) {
7765 sym_name, ext->ksym.addr, sym_addr);
7766 return -EINVAL;
7768 if (!ext->is_set) {
7769 ext->is_set = true;
7770 ext->ksym.addr = sym_addr;
7789 btf = obj->btf_vmlinux;
7793 if (id == -ENOENT) {
7798 for (i = 0; i < obj->btf_module_cnt; i++) {
7800 mod_btf = &obj->btf_modules[i];
7801 btf = mod_btf->btf;
7803 if (id != -ENOENT)
7808 return -ESRCH;
7825 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
7827 if (id == -ESRCH && ext->is_weak)
7830 ext->name);
7835 local_type_id = ext->ksym.type_id;
7839 targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
7840 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
7842 err = bpf_core_types_are_compat(obj->btf, local_type_id,
7848 local_type = btf__type_by_id(obj->btf, local_type_id);
7849 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
7850 targ_name = btf__name_by_offset(btf, targ_type->name_off);
7853 ext->name, local_type_id,
7856 return -EINVAL;
7859 ext->is_set = true;
7860 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
7861 ext->ksym.kernel_btf_id = id;
7863 ext->name, id, btf_kind_str(targ_var), targ_var_name);
7877 local_func_proto_id = ext->ksym.type_id;
7879 kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC, &kern_btf, &mod_btf);
7881 if (kfunc_id == -ESRCH && ext->is_weak)
7884 ext->name);
7889 kfunc_proto_id = kern_func->type;
7891 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
7895 ext->name, local_func_proto_id, kfunc_proto_id);
7896 return -EINVAL;
7900 if (mod_btf && !mod_btf->fd_array_idx) {
7901 /* insn->off is s16 */
7902 if (obj->fd_array_cnt == INT16_MAX) {
7904 ext->name, mod_btf->fd_array_idx);
7905 return -E2BIG;
7908 if (!obj->fd_array_cnt)
7909 obj->fd_array_cnt = 1;
7911 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
7912 obj->fd_array_cnt + 1);
7915 mod_btf->fd_array_idx = obj->fd_array_cnt;
7917 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
7920 ext->is_set = true;
7921 ext->ksym.kernel_btf_id = kfunc_id;
7922 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
7924 ext->name, kfunc_id);
7935 for (i = 0; i < obj->nr_extern; i++) {
7936 ext = &obj->externs[i];
7937 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
7940 if (obj->gen_loader) {
7941 ext->is_set = true;
7942 ext->ksym.kernel_btf_obj_fd = 0;
7943 ext->ksym.kernel_btf_id = 0;
7946 t = btf__type_by_id(obj->btf, ext->btf_id);
7966 if (obj->nr_extern == 0)
7969 if (obj->kconfig_map_idx >= 0)
7970 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
7972 for (i = 0; i < obj->nr_extern; i++) {
7973 ext = &obj->externs[i];
7975 if (ext->type == EXT_KSYM) {
7976 if (ext->ksym.type_id)
7981 } else if (ext->type == EXT_KCFG) {
7982 void *ext_ptr = kcfg_data + ext->kcfg.data_off;
7986 if (str_has_pfx(ext->name, "CONFIG_")) {
7992 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
7995 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
7996 return -EINVAL;
7998 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
8000 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
8002 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
8010 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
8011 return -EINVAL;
8018 ext->name, (long long)value);
8020 pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
8021 return -EINVAL;
8027 return -EINVAL;
8029 for (i = 0; i < obj->nr_extern; i++) {
8030 ext = &obj->externs[i];
8031 if (ext->type == EXT_KCFG && !ext->is_set) {
8040 return -EINVAL;
8045 return -EINVAL;
8050 return -EINVAL;
8052 for (i = 0; i < obj->nr_extern; i++) {
8053 ext = &obj->externs[i];
8055 if (!ext->is_set && !ext->is_weak) {
8056 pr_warn("extern '%s' (strong): not resolved\n", ext->name);
8057 return -ESRCH;
8058 } else if (!ext->is_set) {
8060 ext->name);
8072 return libbpf_err(-EINVAL);
8074 if (obj->loaded) {
8075 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
8076 return libbpf_err(-EINVAL);
8079 if (obj->gen_loader)
8080 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
8084 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
8089 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
8093 if (obj->gen_loader) {
8095 if (obj->btf)
8096 btf__set_fd(obj->btf, -1);
8097 for (i = 0; i < obj->nr_maps; i++)
8098 obj->maps[i].fd = -1;
8100 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
8104 zfree(&obj->fd_array);
8107 for (i = 0; i < obj->btf_module_cnt; i++) {
8108 close(obj->btf_modules[i].fd);
8109 btf__free(obj->btf_modules[i].btf);
8110 free(obj->btf_modules[i].name);
8112 free(obj->btf_modules);
8115 btf__free(obj->btf_vmlinux);
8116 obj->btf_vmlinux = NULL;
8118 obj->loaded = true; /* doesn't matter if successfully or not */
8125 /* unpin any maps that were auto-pinned during load */
8126 for (i = 0; i < obj->nr_maps; i++)
8127 if (obj->maps[i].pinned && !obj->maps[i].reused)
8128 bpf_map__unpin(&obj->maps[i], NULL);
8131 pr_warn("failed to load object '%s'\n", obj->path);
8148 return -ENOMEM;
8152 err = -errno;
8156 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8170 return -EINVAL;
8174 return -ENOMEM;
8180 err = -errno;
8186 err = -EINVAL;
8197 if (prog->fd < 0) {
8198 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
8199 return libbpf_err(-EINVAL);
8210 if (bpf_obj_pin(prog->fd, path)) {
8211 err = -errno;
8213 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
8217 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
8225 if (prog->fd < 0) {
8226 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
8227 return libbpf_err(-EINVAL);
8236 return libbpf_err(-errno);
8238 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
8249 return libbpf_err(-EINVAL);
8252 if (map->pin_path) {
8253 if (path && strcmp(path, map->pin_path)) {
8255 bpf_map__name(map), map->pin_path, path);
8256 return libbpf_err(-EINVAL);
8257 } else if (map->pinned) {
8258 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
8259 bpf_map__name(map), map->pin_path);
8266 return libbpf_err(-EINVAL);
8267 } else if (map->pinned) {
8269 return libbpf_err(-EEXIST);
8272 map->pin_path = strdup(path);
8273 if (!map->pin_path) {
8274 err = -errno;
8279 err = make_parent_dir(map->pin_path);
8283 err = check_path(map->pin_path);
8287 if (bpf_obj_pin(map->fd, map->pin_path)) {
8288 err = -errno;
8292 map->pinned = true;
8293 pr_debug("pinned map '%s'\n", map->pin_path);
8298 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8309 return libbpf_err(-EINVAL);
8312 if (map->pin_path) {
8313 if (path && strcmp(path, map->pin_path)) {
8315 bpf_map__name(map), map->pin_path, path);
8316 return libbpf_err(-EINVAL);
8318 path = map->pin_path;
8322 return libbpf_err(-EINVAL);
8331 return libbpf_err(-errno);
8333 map->pinned = false;
8346 return libbpf_err(-errno);
8349 free(map->pin_path);
8350 map->pin_path = new;
8359 return map->pin_path;
8364 return map->pinned;
8383 return libbpf_err(-ENOENT);
8385 if (!obj->loaded) {
8387 return libbpf_err(-ENOENT);
8394 if (!map->autocreate)
8403 } else if (!map->pin_path) {
8416 if (!map->pin_path)
8431 return libbpf_err(-ENOENT);
8443 } else if (!map->pin_path) {
8462 return libbpf_err(-ENOENT);
8464 if (!obj->loaded) {
8466 return libbpf_err(-ENOENT);
8470 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8483 if (pathname_concat(buf, sizeof(buf), path, prog->name))
8498 return libbpf_err(-ENOENT);
8503 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8534 if (map->inner_map) {
8535 bpf_map__destroy(map->inner_map);
8536 zfree(&map->inner_map);
8539 zfree(&map->init_slots);
8540 map->init_slots_sz = 0;
8542 if (map->mmaped) {
8543 munmap(map->mmaped, bpf_map_mmap_sz(map));
8544 map->mmaped = NULL;
8547 if (map->st_ops) {
8548 zfree(&map->st_ops->data);
8549 zfree(&map->st_ops->progs);
8550 zfree(&map->st_ops->kern_func_off);
8551 zfree(&map->st_ops);
8554 zfree(&map->name);
8555 zfree(&map->real_name);
8556 zfree(&map->pin_path);
8558 if (map->fd >= 0)
8559 zclose(map->fd);
8569 usdt_manager_free(obj->usdt_man);
8570 obj->usdt_man = NULL;
8572 bpf_gen__free(obj->gen_loader);
8575 btf__free(obj->btf);
8576 btf_ext__free(obj->btf_ext);
8578 for (i = 0; i < obj->nr_maps; i++)
8579 bpf_map__destroy(&obj->maps[i]);
8581 zfree(&obj->btf_custom_path);
8582 zfree(&obj->kconfig);
8583 zfree(&obj->externs);
8584 obj->nr_extern = 0;
8586 zfree(&obj->maps);
8587 obj->nr_maps = 0;
8589 if (obj->programs && obj->nr_programs) {
8590 for (i = 0; i < obj->nr_programs; i++)
8591 bpf_program__exit(&obj->programs[i]);
8593 zfree(&obj->programs);
8600 return obj ? obj->name : libbpf_err_ptr(-EINVAL);
8605 return obj ? obj->kern_version : 0;
8610 return obj ? obj->btf : NULL;
8615 return obj->btf ? btf__fd(obj->btf) : -1;
8620 if (obj->loaded)
8621 return libbpf_err(-EINVAL);
8623 obj->kern_version = kern_version;
8633 return -EFAULT;
8635 return -EINVAL;
8638 return -ENOMEM;
8639 gen->opts = opts;
8640 obj->gen_loader = gen;
8648 size_t nr_programs = obj->nr_programs;
8656 return forward ? &obj->programs[0] :
8657 &obj->programs[nr_programs - 1];
8659 if (p->obj != obj) {
8664 idx = (p - obj->programs) + (forward ? 1 : -1);
8665 if (idx >= obj->nr_programs || idx < 0)
8667 return &obj->programs[idx];
8696 prog->prog_ifindex = ifindex;
8701 return prog->name;
8706 return prog->sec_name;
8711 return prog->autoload;
8716 if (prog->obj->loaded)
8717 return libbpf_err(-EINVAL);
8719 prog->autoload = autoload;
8725 return prog->autoattach;
8730 prog->autoattach = autoattach;
8735 return prog->insns;
8740 return prog->insns_cnt;
8748 if (prog->obj->loaded)
8749 return -EBUSY;
8751 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
8753 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
8754 return -ENOMEM;
8758 prog->insns = insns;
8759 prog->insns_cnt = new_insn_cnt;
8766 return libbpf_err(-EINVAL);
8768 if (prog->fd < 0)
8769 return libbpf_err(-ENOENT);
8771 return prog->fd;
8779 return prog->type;
8784 if (prog->obj->loaded)
8785 return libbpf_err(-EBUSY);
8787 prog->type = type;
8796 return prog->expected_attach_type;
8802 if (prog->obj->loaded)
8803 return libbpf_err(-EBUSY);
8805 prog->expected_attach_type = type;
8811 return prog->prog_flags;
8816 if (prog->obj->loaded)
8817 return libbpf_err(-EBUSY);
8819 prog->prog_flags = flags;
8825 return prog->log_level;
8830 if (prog->obj->loaded)
8831 return libbpf_err(-EBUSY);
8833 prog->log_level = log_level;
8839 *log_size = prog->log_size;
8840 return prog->log_buf;
8846 return -EINVAL;
8847 if (prog->log_size > UINT_MAX)
8848 return -EINVAL;
8849 if (prog->obj->loaded)
8850 return -EBUSY;
8852 prog->log_buf = log_buf;
8853 prog->log_size = log_size;
8980 return libbpf_err(-EINVAL);
8983 return libbpf_err(-E2BIG);
8989 return libbpf_err(-ENOMEM);
8995 return libbpf_err(-EBUSY);
9000 sec_def->sec = sec ? strdup(sec) : NULL;
9001 if (sec && !sec_def->sec)
9002 return libbpf_err(-ENOMEM);
9004 sec_def->prog_type = prog_type;
9005 sec_def->expected_attach_type = exp_attach_type;
9006 sec_def->cookie = OPTS_GET(opts, cookie, 0);
9008 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
9009 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
9010 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
9012 sec_def->handler_id = ++last_custom_sec_def_handler_id;
9019 return sec_def->handler_id;
9028 return libbpf_err(-EINVAL);
9042 return libbpf_err(-ENOENT);
9046 custom_sec_defs[i - 1] = custom_sec_defs[i];
9047 custom_sec_def_cnt--;
9059 size_t len = strlen(sec_def->sec);
9062 if (sec_def->sec[len - 1] == '/') {
9063 if (str_has_pfx(sec_name, sec_def->sec))
9069 * well-formed SEC("type/extras") with proper '/' separator
9071 if (sec_def->sec[len - 1] == '+') {
9072 len--;
9074 if (strncmp(sec_name, sec_def->sec, len) != 0)
9082 return strcmp(sec_name, sec_def->sec) == 0;
9127 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9130 if (!(sec_def->cookie & SEC_ATTACHABLE))
9152 return libbpf_err(-EINVAL);
9156 *prog_type = sec_def->prog_type;
9157 *expected_attach_type = sec_def->expected_attach_type;
9168 return libbpf_err(-ESRCH);
9209 for (i = 0; i < obj->nr_maps; i++) {
9210 map = &obj->maps[i];
9213 if (map->sec_offset <= offset &&
9214 offset - map->sec_offset < map->def.value_size)
9221 /* Collect the reloc from ELF and populate the st_ops->progs[] */
9238 btf = obj->btf;
9239 nrels = shdr->sh_size / shdr->sh_entsize;
9244 return -LIBBPF_ERRNO__FORMAT;
9247 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
9250 (size_t)ELF64_R_SYM(rel->r_info));
9251 return -LIBBPF_ERRNO__FORMAT;
9254 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
9255 map = find_struct_ops_map_by_offset(obj, rel->r_offset);
9257 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
9258 (size_t)rel->r_offset);
9259 return -EINVAL;
9262 moff = rel->r_offset - map->sec_offset;
9263 shdr_idx = sym->st_shndx;
9264 st_ops = map->st_ops;
9265 …pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %…
9266 map->name,
9267 (long long)(rel->r_info >> 32),
9268 (long long)sym->st_value,
9269 shdr_idx, (size_t)rel->r_offset,
9270 map->sec_offset, sym->st_name, name);
9273 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
9274 map->name, (size_t)rel->r_offset, shdr_idx);
9275 return -LIBBPF_ERRNO__RELOC;
9277 if (sym->st_value % BPF_INSN_SZ) {
9279 map->name, (unsigned long long)sym->st_value);
9280 return -LIBBPF_ERRNO__FORMAT;
9282 insn_idx = sym->st_value / BPF_INSN_SZ;
9284 member = find_member_by_offset(st_ops->type, moff * 8);
9287 map->name, moff);
9288 return -EINVAL;
9290 member_idx = member - btf_members(st_ops->type);
9291 name = btf__name_by_offset(btf, member->name_off);
9293 if (!resolve_func_ptr(btf, member->type, NULL)) {
9295 map->name, name);
9296 return -EINVAL;
9302 map->name, shdr_idx, name);
9303 return -EINVAL;
9307 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
9309 map->name, prog->name);
9310 return -EINVAL;
9316 if (!prog->attach_btf_id) {
9317 prog->attach_btf_id = st_ops->type_id;
9318 prog->expected_attach_type = member_idx;
9321 /* struct_ops BPF prog can be re-used between multiple
9325 if (prog->attach_btf_id != st_ops->type_id ||
9326 prog->expected_attach_type != member_idx) {
9328 map->name, prog->name, prog->sec_name, prog->type,
9329 prog->attach_btf_id, prog->expected_attach_type, name);
9330 return -EINVAL;
9333 st_ops->progs[member_idx] = prog;
9380 return -ENAMETOOLONG;
9430 err = -EINVAL;
9457 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
9463 if (ret != -ENOENT)
9470 for (i = 0; i < obj->btf_module_cnt; i++) {
9471 const struct module_btf *mod = &obj->btf_modules[i];
9473 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
9475 *btf_obj_fd = mod->fd;
9479 if (ret == -ENOENT)
9485 return -ESRCH;
9491 enum bpf_attach_type attach_type = prog->expected_attach_type;
9492 __u32 attach_prog_fd = prog->attach_prog_fd;
9496 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
9498 pr_warn("prog '%s': attach program FD is not set\n", prog->name);
9499 return -EINVAL;
9504 prog->name, attach_prog_fd, attach_name, err);
9513 if (prog->obj->gen_loader) {
9514 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
9518 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9522 prog->name, attach_name, err);
9535 return libbpf_err(-EINVAL);
9546 return libbpf_err(-EINVAL);
9549 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9550 return libbpf_err(-EINVAL);
9551 if (!(sec_def->cookie & SEC_ATTACHABLE))
9552 return libbpf_err(-EINVAL);
9554 *attach_type = sec_def->expected_attach_type;
9560 return map ? map->fd : libbpf_err(-EINVAL);
9566 * their user-visible name differs from kernel-visible name. Users see
9571 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
9573 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
9584 return map->real_name;
9586 return map->name;
9591 return map->def.type;
9596 if (map->fd >= 0)
9597 return libbpf_err(-EBUSY);
9598 map->def.type = type;
9604 return map->def.map_flags;
9609 if (map->fd >= 0)
9610 return libbpf_err(-EBUSY);
9611 map->def.map_flags = flags;
9617 return map->map_extra;
9622 if (map->fd >= 0)
9623 return libbpf_err(-EBUSY);
9624 map->map_extra = map_extra;
9630 return map->numa_node;
9635 if (map->fd >= 0)
9636 return libbpf_err(-EBUSY);
9637 map->numa_node = numa_node;
9643 return map->def.key_size;
9648 if (map->fd >= 0)
9649 return libbpf_err(-EBUSY);
9650 map->def.key_size = size;
9656 return map->def.value_size;
9661 if (map->fd >= 0)
9662 return libbpf_err(-EBUSY);
9663 map->def.value_size = size;
9669 return map ? map->btf_key_type_id : 0;
9674 return map ? map->btf_value_type_id : 0;
9680 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
9681 size != map->def.value_size || map->fd >= 0)
9682 return libbpf_err(-EINVAL);
9684 memcpy(map->mmaped, data, size);
9690 if (!map->mmaped)
9692 *psize = map->def.value_size;
9693 return map->mmaped;
9698 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
9703 return map->map_ifindex;
9708 if (map->fd >= 0)
9709 return libbpf_err(-EBUSY);
9710 map->map_ifindex = ifindex;
9716 if (!bpf_map_type__is_map_in_map(map->def.type)) {
9718 return libbpf_err(-EINVAL);
9720 if (map->inner_map_fd != -1) {
9722 return libbpf_err(-EINVAL);
9724 if (map->inner_map) {
9725 bpf_map__destroy(map->inner_map);
9726 zfree(&map->inner_map);
9728 map->inner_map_fd = fd;
9738 if (!obj || !obj->maps)
9741 s = obj->maps;
9742 e = obj->maps + obj->nr_maps;
9750 idx = (m - obj->maps) + i;
9751 if (idx >= obj->nr_maps || idx < 0)
9753 return &obj->maps[idx];
9760 return obj->maps;
9769 if (!obj->nr_maps)
9771 return obj->maps + obj->nr_maps - 1;
9774 return __bpf_map__iter(next, obj, -1);
9788 if (pos->real_name && strcmp(pos->real_name, name) == 0)
9794 if (strcmp(pos->real_name, name) == 0)
9798 if (strcmp(pos->name, name) == 0)
9813 if (map->fd <= 0)
9814 return -ENOENT;
9816 if (map->def.key_size != key_sz) {
9818 map->name, key_sz, map->def.key_size);
9819 return -EINVAL;
9825 switch (map->def.type) {
9831 size_t elem_sz = roundup(map->def.value_size, 8);
9834 … pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",
9835 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
9836 return -EINVAL;
9841 if (map->def.value_size != value_sz) {
9843 map->name, value_sz, map->def.value_size);
9844 return -EINVAL;
9861 return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
9874 return bpf_map_update_elem(map->fd, key, value, flags);
9886 return bpf_map_delete_elem_flags(map->fd, key, flags);
9899 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
9911 return bpf_map_get_next_key(map->fd, cur_key, next_key);
9920 errno = -PTR_ERR(ptr);
9927 return -errno;
9951 link->disconnected = true;
9961 if (!link->disconnected && link->detach)
9962 err = link->detach(link);
9963 if (link->pin_path)
9964 free(link->pin_path);
9965 if (link->dealloc)
9966 link->dealloc(link);
9975 return link->fd;
9980 return link->pin_path;
9985 return libbpf_err_errno(close(link->fd));
9995 fd = -errno;
10003 return libbpf_err_ptr(-ENOMEM);
10005 link->detach = &bpf_link__detach_fd;
10006 link->fd = fd;
10008 link->pin_path = strdup(path);
10009 if (!link->pin_path) {
10011 return libbpf_err_ptr(-ENOMEM);
10019 return bpf_link_detach(link->fd) ? -errno : 0;
10026 if (link->pin_path)
10027 return libbpf_err(-EBUSY);
10035 link->pin_path = strdup(path);
10036 if (!link->pin_path)
10037 return libbpf_err(-ENOMEM);
10039 if (bpf_obj_pin(link->fd, link->pin_path)) {
10040 err = -errno;
10041 zfree(&link->pin_path);
10045 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
10053 if (!link->pin_path)
10054 return libbpf_err(-EINVAL);
10056 err = unlink(link->pin_path);
10058 return -errno;
10060 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
10061 zfree(&link->pin_path);
10082 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
10083 err = -errno;
10085 if (perf_link->perf_event_fd != link->fd)
10086 close(perf_link->perf_event_fd);
10087 close(link->fd);
10090 if (perf_link->legacy_probe_name) {
10091 if (perf_link->legacy_is_kprobe) {
10092 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
10093 perf_link->legacy_is_retprobe);
10095 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
10096 perf_link->legacy_is_retprobe);
10107 free(perf_link->legacy_probe_name);
10116 int prog_fd, link_fd = -1, err;
10119 return libbpf_err_ptr(-EINVAL);
10123 prog->name, pfd);
10124 return libbpf_err_ptr(-EINVAL);
10129 prog->name);
10130 return libbpf_err_ptr(-EINVAL);
10135 return libbpf_err_ptr(-ENOMEM);
10136 link->link.detach = &bpf_link_perf_detach;
10137 link->link.dealloc = &bpf_link_perf_dealloc;
10138 link->perf_event_fd = pfd;
10140 if (kernel_supports(prog->obj, FEAT_PERF_LINK)) {
10146 err = -errno;
10148 prog->name, pfd,
10152 link->link.fd = link_fd;
10155 pr_warn("prog '%s': user context value is not supported\n", prog->name);
10156 err = -EOPNOTSUPP;
10161 err = -errno;
10163 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10164 if (err == -EPROTO)
10166 prog->name, pfd);
10169 link->link.fd = pfd;
10172 err = -errno;
10174 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10178 return &link->link;
10192 * this function is expected to parse integer in the range of [0, 2^31-1] from
10204 err = -errno;
10211 err = err == EOF ? -EIO : -errno;
10261 return -EINVAL;
10293 pid < 0 ? -1 : pid /* pid */,
10294 pid == -1 ? 0 : -1 /* cpu */,
10295 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10296 return pfd >= 0 ? pfd : -errno;
10306 return -errno;
10313 err = -errno;
10324 static int has_debugfs = -1;
10367 return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
10411 pid < 0 ? -1 : pid, /* pid */
10412 pid == -1 ? 0 : -1, /* cpu */
10413 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10415 err = -errno;
10500 return libbpf_err_ptr(-EINVAL);
10510 -1 /* pid */, 0 /* ref_ctr_off */);
10519 return libbpf_err_ptr(-ENOMEM);
10522 offset, -1 /* pid */);
10525 err = -errno;
10527 prog->name, retprobe ? "kretprobe" : "kprobe",
10537 prog->name, retprobe ? "kretprobe" : "kprobe",
10545 perf_link->legacy_probe_name = legacy_probe;
10546 perf_link->legacy_is_kprobe = true;
10547 perf_link->legacy_is_retprobe = retprobe;
10579 return libbpf_err_ptr(-EINVAL);
10581 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
10640 if (!glob_match(sym_name, res->pattern))
10643 err = libbpf_ensure_mem((void **) &res->addrs, &res->cap, sizeof(unsigned long),
10644 res->cnt + 1);
10648 res->addrs[res->cnt++] = (unsigned long) sym_addr;
10671 return libbpf_err_ptr(-EINVAL);
10679 return libbpf_err_ptr(-EINVAL);
10681 return libbpf_err_ptr(-EINVAL);
10683 return libbpf_err_ptr(-EINVAL);
10685 return libbpf_err_ptr(-EINVAL);
10692 err = -ENOENT;
10709 err = -ENOMEM;
10712 link->detach = &bpf_link__detach_fd;
10717 err = -errno;
10719 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10722 link->fd = link_fd;
10742 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
10743 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
10746 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
10748 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
10750 func_name = prog->sec_name + sizeof("kprobe/") - 1;
10752 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
10755 return -EINVAL;
10760 return -EINVAL;
10776 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
10777 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
10780 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
10782 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
10784 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
10787 return *link ? 0 : -errno;
10799 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
10800 if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
10801 strcmp(prog->sec_name, "kretprobe.multi") == 0)
10804 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
10806 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
10808 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
10810 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
10813 return -EINVAL;
10846 return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
10887 pid < 0 ? -1 : pid, /* pid */
10888 pid == -1 ? 0 : -1, /* cpu */
10889 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10891 err = -errno;
10949 long ret = -ENOENT;
10964 ret = -errno;
10972 pr_warn("elf: could not read elf from %s: %s\n", binary_path, elf_errmsg(-1));
10974 return -LIBBPF_ERRNO__FORMAT;
10977 pr_warn("elf: failed to get ehdr from %s: %s\n", binary_path, elf_errmsg(-1));
10978 ret = -LIBBPF_ERRNO__FORMAT;
10992 * a binary is stripped, it may only have SHT_DYNSYM, and a fully-statically
11006 int last_bind = -1;
11033 binary_path, elf_errmsg(-1));
11034 ret = -LIBBPF_ERRNO__FORMAT;
11037 nr_syms = symbols->d_size / sh.sh_entsize;
11043 binary_path, elf_errmsg(-1));
11044 ret = -LIBBPF_ERRNO__FORMAT;
11052 binary_path, elf_errmsg(-1));
11053 ret = -LIBBPF_ERRNO__FORMAT;
11105 /* Only accept one non-weak bind. */
11108 ret = -LIBBPF_ERRNO__FORMAT;
11111 /* already have a non-weak bind, and
11136 ret = sym.st_value - sym_sh.sh_addr + sym_sh.sh_offset;
11144 ret = st_value - sh_addr + sh_offset;
11163 ret = -ENOENT;
11189 return "/lib/x86_64-linux-gnu";
11191 return "/lib/i386-linux-gnu";
11193 return "/lib/s390x-linux-gnu";
11195 return "/lib/s390-linux-gnu";
11197 return "/lib/arm-linux-gnueabi";
11199 return "/lib/arm-linux-gnueabihf";
11201 return "/lib/aarch64-linux-gnu";
11203 return "/lib/mips64el-linux-gnuabi64";
11205 return "/lib/mipsel-linux-gnu";
11207 return "/lib/powerpc64le-linux-gnu";
11209 return "/lib/sparc64-linux-gnu";
11211 return "/lib/riscv64-linux-gnu";
11246 seg_len = next_path ? next_path - s : strlen(s);
11257 return -ENOENT;
11275 return libbpf_err_ptr(-EINVAL);
11282 return libbpf_err_ptr(-EINVAL);
11289 prog->name, binary_path, err);
11312 return libbpf_err_ptr(-EINVAL);
11319 return libbpf_err_ptr(-ENOMEM);
11325 err = -errno;
11327 prog->name, retprobe ? "uretprobe" : "uprobe",
11338 prog->name, retprobe ? "uretprobe" : "uprobe",
11346 perf_link->legacy_probe_name = legacy_probe;
11347 perf_link->legacy_is_kprobe = false;
11348 perf_link->legacy_is_retprobe = retprobe;
11360 /* Format of u[ret]probe section definition supporting auto-attach:
11367 * specified (and auto-attach is not possible) or the above format is specified for
11368 * auto-attach.
11374 int n, ret = -EINVAL;
11379 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[a-zA-Z0-9_.]+%li",
11383 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
11388 prog->name, prog->sec_name);
11396 prog->name);
11400 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
11404 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
11405 prog->sec_name);
11432 struct bpf_object *obj = prog->obj;
11438 return libbpf_err_ptr(-EINVAL);
11442 prog->name);
11443 return libbpf_err_ptr(-EINVAL);
11447 return libbpf_err_ptr(-EINVAL);
11453 prog->name, binary_path, err);
11462 if (IS_ERR(obj->usdt_man))
11463 return libbpf_ptr(obj->usdt_man);
11464 if (!obj->usdt_man) {
11465 obj->usdt_man = usdt_manager_new(obj);
11466 if (IS_ERR(obj->usdt_man))
11467 return libbpf_ptr(obj->usdt_man);
11471 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
11489 /* no auto-attach for just SEC("usdt") */
11498 err = -EINVAL;
11500 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
11520 return -errno;
11524 return -E2BIG;
11550 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
11551 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11553 err = -errno;
11573 return libbpf_err_ptr(-EINVAL);
11580 prog->name, tp_category, tp_name,
11589 prog->name, tp_category, tp_name,
11609 /* no auto-attach for SEC("tp") or SEC("tracepoint") */
11610 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
11613 sec_name = strdup(prog->sec_name);
11615 return -ENOMEM;
11618 if (str_has_pfx(prog->sec_name, "tp/"))
11619 tp_cat = sec_name + sizeof("tp/") - 1;
11621 tp_cat = sec_name + sizeof("tracepoint/") - 1;
11625 return -EINVAL;
11644 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11645 return libbpf_err_ptr(-EINVAL);
11650 return libbpf_err_ptr(-ENOMEM);
11651 link->detach = &bpf_link__detach_fd;
11655 pfd = -errno;
11658 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11661 link->fd = pfd;
11681 if (!str_has_pfx(prog->sec_name, prefixes[i]))
11685 /* no auto-attach case of, e.g., SEC("raw_tp") */
11686 if (prog->sec_name[pfx_len] == '\0')
11689 if (prog->sec_name[pfx_len] != '/')
11692 tp_name = prog->sec_name + pfx_len + 1;
11698 prog->name, prog->sec_name);
11699 return -EINVAL;
11716 return libbpf_err_ptr(-EINVAL);
11720 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11721 return libbpf_err_ptr(-EINVAL);
11726 return libbpf_err_ptr(-ENOMEM);
11727 link->detach = &bpf_link__detach_fd;
11733 pfd = -errno;
11736 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11739 link->fd = pfd;
11784 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11785 return libbpf_err_ptr(-EINVAL);
11790 return libbpf_err_ptr(-ENOMEM);
11791 link->detach = &bpf_link__detach_fd;
11796 link_fd = -errno;
11799 prog->name, target_name,
11803 link->fd = link_fd;
11833 prog->name);
11834 return libbpf_err_ptr(-EINVAL);
11837 if (prog->type != BPF_PROG_TYPE_EXT) {
11839 prog->name);
11840 return libbpf_err_ptr(-EINVAL);
11868 return libbpf_err_ptr(-EINVAL);
11875 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11876 return libbpf_err_ptr(-EINVAL);
11881 return libbpf_err_ptr(-ENOMEM);
11882 link->detach = &bpf_link__detach_fd;
11887 link_fd = -errno;
11890 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
11893 link->fd = link_fd;
11908 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
11909 return libbpf_err_ptr(-EOPNOTSUPP);
11911 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
11915 /* When calling bpf_program__attach() explicitly, auto-attach support
11921 return libbpf_err_ptr(-EOPNOTSUPP);
11930 if (bpf_map_delete_elem(link->fd, &zero))
11931 return -errno;
11943 if (!bpf_map__is_struct_ops(map) || map->fd == -1)
11944 return libbpf_err_ptr(-EINVAL);
11948 return libbpf_err_ptr(-EINVAL);
11950 st_ops = map->st_ops;
11951 for (i = 0; i < btf_vlen(st_ops->type); i++) {
11952 struct bpf_program *prog = st_ops->progs[i];
11960 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
11964 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
11966 err = -errno;
11971 link->detach = bpf_link__detach_struct_ops;
11972 link->fd = map->fd;
11987 __u64 data_tail = header->data_tail;
11994 ehdr = base + (data_tail & (mmap_size - 1));
11995 ehdr_size = ehdr->size;
11999 size_t len_first = base + mmap_size - copy_start;
12000 size_t len_secnd = ehdr_size - len_first;
12034 /* sample_cb and lost_cb are higher-level common-case callbacks */
12073 if (cpu_buf->base &&
12074 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
12075 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
12076 if (cpu_buf->fd >= 0) {
12077 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
12078 close(cpu_buf->fd);
12080 free(cpu_buf->buf);
12090 if (pb->cpu_bufs) {
12091 for (i = 0; i < pb->cpu_cnt; i++) {
12092 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
12097 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
12100 free(pb->cpu_bufs);
12102 if (pb->epoll_fd >= 0)
12103 close(pb->epoll_fd);
12104 free(pb->events);
12118 return ERR_PTR(-ENOMEM);
12120 cpu_buf->pb = pb;
12121 cpu_buf->cpu = cpu;
12122 cpu_buf->map_key = map_key;
12124 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
12125 -1, PERF_FLAG_FD_CLOEXEC);
12126 if (cpu_buf->fd < 0) {
12127 err = -errno;
12133 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
12135 cpu_buf->fd, 0);
12136 if (cpu_buf->base == MAP_FAILED) {
12137 cpu_buf->base = NULL;
12138 err = -errno;
12144 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
12145 err = -errno;
12172 return libbpf_err_ptr(-EINVAL);
12198 return libbpf_err_ptr(-EINVAL);
12201 return libbpf_err_ptr(-EINVAL);
12224 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {
12227 return ERR_PTR(-EINVAL);
12230 /* best-effort sanity checks */
12235 err = -errno;
12237 * -EBADFD, -EFAULT, or -E2BIG on real error
12239 if (err != -EINVAL) {
12250 return ERR_PTR(-EINVAL);
12256 return ERR_PTR(-ENOMEM);
12258 pb->event_cb = p->event_cb;
12259 pb->sample_cb = p->sample_cb;
12260 pb->lost_cb = p->lost_cb;
12261 pb->ctx = p->ctx;
12263 pb->page_size = getpagesize();
12264 pb->mmap_size = pb->page_size * page_cnt;
12265 pb->map_fd = map_fd;
12267 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
12268 if (pb->epoll_fd < 0) {
12269 err = -errno;
12275 if (p->cpu_cnt > 0) {
12276 pb->cpu_cnt = p->cpu_cnt;
12278 pb->cpu_cnt = libbpf_num_possible_cpus();
12279 if (pb->cpu_cnt < 0) {
12280 err = pb->cpu_cnt;
12283 if (map.max_entries && map.max_entries < pb->cpu_cnt)
12284 pb->cpu_cnt = map.max_entries;
12287 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
12288 if (!pb->events) {
12289 err = -ENOMEM;
12293 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
12294 if (!pb->cpu_bufs) {
12295 err = -ENOMEM;
12306 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
12310 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
12311 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
12316 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
12319 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
12325 pb->cpu_bufs[j] = cpu_buf;
12327 err = bpf_map_update_elem(pb->map_fd, &map_key,
12328 &cpu_buf->fd, 0);
12330 err = -errno;
12331 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
12332 cpu, map_key, cpu_buf->fd,
12337 pb->events[j].events = EPOLLIN;
12338 pb->events[j].data.ptr = cpu_buf;
12339 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
12340 &pb->events[j]) < 0) {
12341 err = -errno;
12343 cpu, cpu_buf->fd,
12349 pb->cpu_cnt = j;
12378 struct perf_buffer *pb = cpu_buf->pb;
12382 if (pb->event_cb)
12383 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
12385 switch (e->type) {
12389 if (pb->sample_cb)
12390 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
12396 if (pb->lost_cb)
12397 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
12401 pr_warn("unknown perf sample type %d\n", e->type);
12412 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
12413 pb->page_size, &cpu_buf->buf,
12414 &cpu_buf->buf_size,
12423 return pb->epoll_fd;
12430 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
12432 return -errno;
12435 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
12451 return pb->cpu_cnt;
12457 * select()/poll()/epoll() Linux syscalls.
12463 if (buf_idx >= pb->cpu_cnt)
12464 return libbpf_err(-EINVAL);
12466 cpu_buf = pb->cpu_bufs[buf_idx];
12468 return libbpf_err(-ENOENT);
12470 return cpu_buf->fd;
12477 if (buf_idx >= pb->cpu_cnt)
12478 return libbpf_err(-EINVAL);
12480 cpu_buf = pb->cpu_bufs[buf_idx];
12482 return libbpf_err(-ENOENT);
12484 *buf = cpu_buf->base;
12485 *buf_size = pb->mmap_size;
12494 * - 0 on success;
12495 * - <0 on failure.
12501 if (buf_idx >= pb->cpu_cnt)
12502 return libbpf_err(-EINVAL);
12504 cpu_buf = pb->cpu_bufs[buf_idx];
12506 return libbpf_err(-ENOENT);
12515 for (i = 0; i < pb->cpu_cnt; i++) {
12516 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
12537 return libbpf_err(-EINVAL);
12539 if (prog->obj->loaded)
12540 return libbpf_err(-EINVAL);
12546 prog->attach_prog_fd = attach_prog_fd;
12557 return libbpf_err(-EINVAL);
12560 err = bpf_object__load_vmlinux_btf(prog->obj, true);
12563 err = find_kernel_btf_id(prog->obj, attach_func_name,
12564 prog->expected_attach_type,
12570 prog->attach_btf_id = btf_id;
12571 prog->attach_btf_obj_fd = btf_obj_fd;
12572 prog->attach_prog_fd = attach_prog_fd;
12578 int err = 0, n, len, start, end = -1;
12584 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
12590 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
12593 err = -EINVAL;
12601 err = -EINVAL;
12606 err = -ENOMEM;
12610 memset(tmp + *mask_sz, 0, start - *mask_sz);
12611 memset(tmp + start, 1, end - start + 1);
12617 return -EINVAL;
12633 err = -errno;
12640 err = len ? -errno : -EINVAL;
12646 return -E2BIG;
12693 return -ESRCH;
12696 /* externs shouldn't be pre-setup from user code */
12697 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
12698 *mmaped = (*map)->mmaped;
12716 return -ESRCH;
12726 .object_name = s->name,
12731 /* Attempt to preserve opts->object_name, unless overriden by user
12739 if (!opts->object_name)
12740 skel_opts.object_name = s->name;
12743 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
12747 s->name, err);
12751 *s->obj = obj;
12752 err = populate_skeleton_maps(obj, s->maps, s->map_cnt);
12754 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);
12758 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt);
12760 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);
12778 if (!s->obj)
12779 return libbpf_err(-EINVAL);
12781 btf = bpf_object__btf(s->obj);
12784 bpf_object__name(s->obj));
12785 return libbpf_err(-errno);
12788 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt);
12794 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt);
12800 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
12801 var_skel = &s->vars[var_idx];
12802 map = *var_skel->map;
12810 return libbpf_err(-EINVAL);
12816 var_type = btf__type_by_id(btf, var->type);
12817 var_name = btf__name_by_offset(btf, var_type->name_off);
12818 if (strcmp(var_name, var_skel->name) == 0) {
12819 *var_skel->addr = map->mmaped + var->offset;
12831 free(s->maps);
12832 free(s->progs);
12833 free(s->vars);
12841 err = bpf_object__load(*s->obj);
12843 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
12847 for (i = 0; i < s->map_cnt; i++) {
12848 struct bpf_map *map = *s->maps[i].map;
12851 void **mmaped = s->maps[i].mmaped;
12856 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
12861 if (map->def.map_flags & BPF_F_RDONLY_PROG)
12866 /* Remap anonymous mmap()-ed "map initialization image" as
12867 * a BPF map-backed mmap()-ed memory, but preserving the same
12876 *mmaped = mmap(map->mmaped, mmap_sz, prot,
12879 err = -errno;
12881 pr_warn("failed to re-mmap() map '%s': %d\n",
12894 for (i = 0; i < s->prog_cnt; i++) {
12895 struct bpf_program *prog = *s->progs[i].prog;
12896 struct bpf_link **link = s->progs[i].link;
12898 if (!prog->autoload || !prog->autoattach)
12901 /* auto-attaching not supported for this program */
12902 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
12905 /* if user already set the link manually, don't attempt auto-attach */
12909 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
12911 pr_warn("prog '%s': failed to auto-attach: %d\n",
12916 /* It's possible that for some SEC() definitions auto-attach
12921 * auto-attached. But if not, it shouldn't trigger skeleton's
12935 for (i = 0; i < s->prog_cnt; i++) {
12936 struct bpf_link **link = s->progs[i].link;
12948 if (s->progs)
12950 if (s->obj)
12951 bpf_object__close(*s->obj);
12952 free(s->maps);
12953 free(s->progs);