Lines Matching +full:kconfig +full:- +full:ext
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
67 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
207 [BPF_PROG_TYPE_EXT] = "ext",
259 if (err != -EPERM || geteuid() != 0) in pr_perm_msg()
276 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", in pr_perm_msg()
292 fd = -1; \
303 /* as of v1.0 libbpf_set_strict_mode() is a no-op */ in libbpf_set_strict_mode()
349 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
368 /* BPF program support non-linear XDP buffer */
406 * program. For the entry-point (main) BPF program, this is always
407 * zero. For a sub-program, this gets reset before each of main BPF
409 * whether sub-program was already appended to the main program, and
421 * entry-point BPF programs this includes the size of main program
422 * itself plus all the used sub-programs, appended at the end
473 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
484 #define KCONFIG_SEC ".kconfig"
509 * .data*, .bss, .kconfig) and preserves their original ELF section
579 /* BTF fd index to be patched in for insn->off, this is
580 * 0 for vmlinux BTF, index in obj->fd_array for module
644 char *kconfig; member
665 /* Path to the custom BTF to be used for BPF CO-RE relocations as an
669 /* vmlinux BTF override for CO-RE relocations */
706 zclose(prog->fd); in bpf_program__unload()
708 zfree(&prog->func_info); in bpf_program__unload()
709 zfree(&prog->line_info); in bpf_program__unload()
718 zfree(&prog->name); in bpf_program__exit()
719 zfree(&prog->sec_name); in bpf_program__exit()
720 zfree(&prog->insns); in bpf_program__exit()
721 zfree(&prog->reloc_desc); in bpf_program__exit()
723 prog->nr_reloc = 0; in bpf_program__exit()
724 prog->insns_cnt = 0; in bpf_program__exit()
725 prog->sec_idx = -1; in bpf_program__exit()
730 return BPF_CLASS(insn->code) == BPF_JMP && in insn_is_subprog_call()
731 BPF_OP(insn->code) == BPF_CALL && in insn_is_subprog_call()
732 BPF_SRC(insn->code) == BPF_K && in insn_is_subprog_call()
733 insn->src_reg == BPF_PSEUDO_CALL && in insn_is_subprog_call()
734 insn->dst_reg == 0 && in insn_is_subprog_call()
735 insn->off == 0; in insn_is_subprog_call()
740 return insn->code == (BPF_JMP | BPF_CALL); in is_call_insn()
745 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; in insn_is_pseudo_func()
756 return -EINVAL; in bpf_object__init_prog()
760 prog->obj = obj; in bpf_object__init_prog()
762 prog->sec_idx = sec_idx; in bpf_object__init_prog()
763 prog->sec_insn_off = sec_off / BPF_INSN_SZ; in bpf_object__init_prog()
764 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; in bpf_object__init_prog()
766 prog->insns_cnt = prog->sec_insn_cnt; in bpf_object__init_prog()
768 prog->type = BPF_PROG_TYPE_UNSPEC; in bpf_object__init_prog()
769 prog->fd = -1; in bpf_object__init_prog()
770 prog->exception_cb_idx = -1; in bpf_object__init_prog()
777 prog->autoload = false; in bpf_object__init_prog()
781 prog->autoload = true; in bpf_object__init_prog()
784 prog->autoattach = true; in bpf_object__init_prog()
787 prog->log_level = obj->log_level; in bpf_object__init_prog()
789 prog->sec_name = strdup(sec_name); in bpf_object__init_prog()
790 if (!prog->sec_name) in bpf_object__init_prog()
793 prog->name = strdup(name); in bpf_object__init_prog()
794 if (!prog->name) in bpf_object__init_prog()
797 prog->insns = malloc(insn_data_sz); in bpf_object__init_prog()
798 if (!prog->insns) in bpf_object__init_prog()
800 memcpy(prog->insns, insn_data, insn_data_sz); in bpf_object__init_prog()
806 return -ENOMEM; in bpf_object__init_prog()
813 Elf_Data *symbols = obj->efile.symbols; in bpf_object__add_programs()
815 void *data = sec_data->d_buf; in bpf_object__add_programs()
816 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; in bpf_object__add_programs()
821 progs = obj->programs; in bpf_object__add_programs()
822 nr_progs = obj->nr_programs; in bpf_object__add_programs()
823 nr_syms = symbols->d_size / sizeof(Elf64_Sym); in bpf_object__add_programs()
828 if (sym->st_shndx != sec_idx) in bpf_object__add_programs()
830 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) in bpf_object__add_programs()
833 prog_sz = sym->st_size; in bpf_object__add_programs()
834 sec_off = sym->st_value; in bpf_object__add_programs()
836 name = elf_sym_str(obj, sym->st_name); in bpf_object__add_programs()
840 return -LIBBPF_ERRNO__FORMAT; in bpf_object__add_programs()
846 return -LIBBPF_ERRNO__FORMAT; in bpf_object__add_programs()
849 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { in bpf_object__add_programs()
851 return -ENOTSUP; in bpf_object__add_programs()
860 * In this case the original obj->programs in bpf_object__add_programs()
866 return -ENOMEM; in bpf_object__add_programs()
868 obj->programs = progs; in bpf_object__add_programs()
877 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL) in bpf_object__add_programs()
878 prog->sym_global = true; in bpf_object__add_programs()
885 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN in bpf_object__add_programs()
886 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) in bpf_object__add_programs()
887 prog->mark_btf_static = true; in bpf_object__add_programs()
890 obj->nr_programs = nr_progs; in bpf_object__add_programs()
918 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) in find_member_by_name()
970 if (kern_data_member->type == kern_type_id) in find_struct_ops_kern_types()
976 return -EINVAL; in find_struct_ops_kern_types()
990 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; in bpf_map__is_struct_ops()
1006 st_ops = map->st_ops; in bpf_map__init_kern_struct_ops()
1007 type = st_ops->type; in bpf_map__init_kern_struct_ops()
1008 tname = st_ops->tname; in bpf_map__init_kern_struct_ops()
1017 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); in bpf_map__init_kern_struct_ops()
1019 map->def.value_size = kern_vtype->size; in bpf_map__init_kern_struct_ops()
1020 map->btf_vmlinux_value_type_id = kern_vtype_id; in bpf_map__init_kern_struct_ops()
1022 st_ops->kern_vdata = calloc(1, kern_vtype->size); in bpf_map__init_kern_struct_ops()
1023 if (!st_ops->kern_vdata) in bpf_map__init_kern_struct_ops()
1024 return -ENOMEM; in bpf_map__init_kern_struct_ops()
1026 data = st_ops->data; in bpf_map__init_kern_struct_ops()
1027 kern_data_off = kern_data_member->offset / 8; in bpf_map__init_kern_struct_ops()
1028 kern_data = st_ops->kern_vdata + kern_data_off; in bpf_map__init_kern_struct_ops()
1040 mname = btf__name_by_offset(btf, member->name_off); in bpf_map__init_kern_struct_ops()
1044 map->name, mname); in bpf_map__init_kern_struct_ops()
1045 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1048 kern_member_idx = kern_member - btf_members(kern_type); in bpf_map__init_kern_struct_ops()
1052 map->name, mname); in bpf_map__init_kern_struct_ops()
1053 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1056 moff = member->offset / 8; in bpf_map__init_kern_struct_ops()
1057 kern_moff = kern_member->offset / 8; in bpf_map__init_kern_struct_ops()
1062 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); in bpf_map__init_kern_struct_ops()
1063 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, in bpf_map__init_kern_struct_ops()
1065 if (BTF_INFO_KIND(mtype->info) != in bpf_map__init_kern_struct_ops()
1066 BTF_INFO_KIND(kern_mtype->info)) { in bpf_map__init_kern_struct_ops()
1068 map->name, mname, BTF_INFO_KIND(mtype->info), in bpf_map__init_kern_struct_ops()
1069 BTF_INFO_KIND(kern_mtype->info)); in bpf_map__init_kern_struct_ops()
1070 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1076 prog = st_ops->progs[i]; in bpf_map__init_kern_struct_ops()
1081 kern_mtype->type, in bpf_map__init_kern_struct_ops()
1084 /* mtype->type must be a func_proto which was in bpf_map__init_kern_struct_ops()
1090 map->name, mname); in bpf_map__init_kern_struct_ops()
1091 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1094 prog->attach_btf_id = kern_type_id; in bpf_map__init_kern_struct_ops()
1095 prog->expected_attach_type = kern_member_idx; in bpf_map__init_kern_struct_ops()
1097 st_ops->kern_func_off[i] = kern_data_off + kern_moff; in bpf_map__init_kern_struct_ops()
1100 map->name, mname, prog->name, moff, in bpf_map__init_kern_struct_ops()
1110 map->name, mname, (ssize_t)msize, in bpf_map__init_kern_struct_ops()
1112 return -ENOTSUP; in bpf_map__init_kern_struct_ops()
1116 map->name, mname, (unsigned int)msize, in bpf_map__init_kern_struct_ops()
1130 for (i = 0; i < obj->nr_maps; i++) { in bpf_object__init_kern_struct_ops_maps()
1131 map = &obj->maps[i]; in bpf_object__init_kern_struct_ops_maps()
1136 err = bpf_map__init_kern_struct_ops(map, obj->btf, in bpf_object__init_kern_struct_ops_maps()
1137 obj->btf_vmlinux); in bpf_object__init_kern_struct_ops_maps()
1157 if (shndx == -1) in init_struct_ops_maps()
1160 btf = obj->btf; in init_struct_ops_maps()
1166 return -EINVAL; in init_struct_ops_maps()
1172 type = btf__type_by_id(obj->btf, vsi->type); in init_struct_ops_maps()
1173 var_name = btf__name_by_offset(obj->btf, type->name_off); in init_struct_ops_maps()
1175 type_id = btf__resolve_type(obj->btf, vsi->type); in init_struct_ops_maps()
1178 vsi->type, sec_name); in init_struct_ops_maps()
1179 return -EINVAL; in init_struct_ops_maps()
1182 type = btf__type_by_id(obj->btf, type_id); in init_struct_ops_maps()
1183 tname = btf__name_by_offset(obj->btf, type->name_off); in init_struct_ops_maps()
1186 return -ENOTSUP; in init_struct_ops_maps()
1190 return -EINVAL; in init_struct_ops_maps()
1197 map->sec_idx = shndx; in init_struct_ops_maps()
1198 map->sec_offset = vsi->offset; in init_struct_ops_maps()
1199 map->name = strdup(var_name); in init_struct_ops_maps()
1200 if (!map->name) in init_struct_ops_maps()
1201 return -ENOMEM; in init_struct_ops_maps()
1203 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; in init_struct_ops_maps()
1204 map->def.key_size = sizeof(int); in init_struct_ops_maps()
1205 map->def.value_size = type->size; in init_struct_ops_maps()
1206 map->def.max_entries = 1; in init_struct_ops_maps()
1207 map->def.map_flags = map_flags; in init_struct_ops_maps()
1209 map->st_ops = calloc(1, sizeof(*map->st_ops)); in init_struct_ops_maps()
1210 if (!map->st_ops) in init_struct_ops_maps()
1211 return -ENOMEM; in init_struct_ops_maps()
1212 st_ops = map->st_ops; in init_struct_ops_maps()
1213 st_ops->data = malloc(type->size); in init_struct_ops_maps()
1214 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); in init_struct_ops_maps()
1215 st_ops->kern_func_off = malloc(btf_vlen(type) * in init_struct_ops_maps()
1216 sizeof(*st_ops->kern_func_off)); in init_struct_ops_maps()
1217 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) in init_struct_ops_maps()
1218 return -ENOMEM; in init_struct_ops_maps()
1220 if (vsi->offset + type->size > data->d_size) { in init_struct_ops_maps()
1223 return -EINVAL; in init_struct_ops_maps()
1226 memcpy(st_ops->data, in init_struct_ops_maps()
1227 data->d_buf + vsi->offset, in init_struct_ops_maps()
1228 type->size); in init_struct_ops_maps()
1229 st_ops->tname = tname; in init_struct_ops_maps()
1230 st_ops->type = type; in init_struct_ops_maps()
1231 st_ops->type_id = type_id; in init_struct_ops_maps()
1234 tname, type_id, var_name, vsi->offset); in init_struct_ops_maps()
1244 err = init_struct_ops_maps(obj, STRUCT_OPS_SEC, obj->efile.st_ops_shndx, in bpf_object_init_struct_ops()
1245 obj->efile.st_ops_data, 0); in bpf_object_init_struct_ops()
1247 obj->efile.st_ops_link_shndx, in bpf_object_init_struct_ops()
1248 obj->efile.st_ops_link_data, in bpf_object_init_struct_ops()
1264 return ERR_PTR(-ENOMEM); in bpf_object__new()
1267 strcpy(obj->path, path); in bpf_object__new()
1269 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); in bpf_object__new()
1272 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); in bpf_object__new()
1273 end = strchr(obj->name, '.'); in bpf_object__new()
1278 obj->efile.fd = -1; in bpf_object__new()
1285 obj->efile.obj_buf = obj_buf; in bpf_object__new()
1286 obj->efile.obj_buf_sz = obj_buf_sz; in bpf_object__new()
1287 obj->efile.btf_maps_shndx = -1; in bpf_object__new()
1288 obj->efile.st_ops_shndx = -1; in bpf_object__new()
1289 obj->efile.st_ops_link_shndx = -1; in bpf_object__new()
1290 obj->kconfig_map_idx = -1; in bpf_object__new()
1292 obj->kern_version = get_kernel_version(); in bpf_object__new()
1293 obj->loaded = false; in bpf_object__new()
1300 if (!obj->efile.elf) in bpf_object__elf_finish()
1303 elf_end(obj->efile.elf); in bpf_object__elf_finish()
1304 obj->efile.elf = NULL; in bpf_object__elf_finish()
1305 obj->efile.symbols = NULL; in bpf_object__elf_finish()
1306 obj->efile.st_ops_data = NULL; in bpf_object__elf_finish()
1307 obj->efile.st_ops_link_data = NULL; in bpf_object__elf_finish()
1309 zfree(&obj->efile.secs); in bpf_object__elf_finish()
1310 obj->efile.sec_cnt = 0; in bpf_object__elf_finish()
1311 zclose(obj->efile.fd); in bpf_object__elf_finish()
1312 obj->efile.obj_buf = NULL; in bpf_object__elf_finish()
1313 obj->efile.obj_buf_sz = 0; in bpf_object__elf_finish()
1322 if (obj->efile.elf) { in bpf_object__elf_init()
1324 return -LIBBPF_ERRNO__LIBELF; in bpf_object__elf_init()
1327 if (obj->efile.obj_buf_sz > 0) { in bpf_object__elf_init()
1329 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz); in bpf_object__elf_init()
1331 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC); in bpf_object__elf_init()
1332 if (obj->efile.fd < 0) { in bpf_object__elf_init()
1335 err = -errno; in bpf_object__elf_init()
1337 pr_warn("elf: failed to open %s: %s\n", obj->path, cp); in bpf_object__elf_init()
1341 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL); in bpf_object__elf_init()
1345 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1)); in bpf_object__elf_init()
1346 err = -LIBBPF_ERRNO__LIBELF; in bpf_object__elf_init()
1350 obj->efile.elf = elf; in bpf_object__elf_init()
1353 err = -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_init()
1354 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path); in bpf_object__elf_init()
1359 err = -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_init()
1360 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path); in bpf_object__elf_init()
1364 obj->efile.ehdr = ehdr = elf64_getehdr(elf); in bpf_object__elf_init()
1365 if (!obj->efile.ehdr) { in bpf_object__elf_init()
1366 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1)); in bpf_object__elf_init()
1367 err = -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_init()
1371 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) { in bpf_object__elf_init()
1373 obj->path, elf_errmsg(-1)); in bpf_object__elf_init()
1374 err = -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_init()
1379 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) { in bpf_object__elf_init()
1381 obj->path, elf_errmsg(-1)); in bpf_object__elf_init()
1382 err = -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_init()
1387 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) { in bpf_object__elf_init()
1388 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path); in bpf_object__elf_init()
1389 err = -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_init()
1402 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB) in bpf_object__check_endianness()
1405 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB) in bpf_object__check_endianness()
1410 pr_warn("elf: endianness mismatch in %s.\n", obj->path); in bpf_object__check_endianness()
1411 return -LIBBPF_ERRNO__ENDIAN; in bpf_object__check_endianness()
1418 pr_warn("invalid license section in %s\n", obj->path); in bpf_object__init_license()
1419 return -LIBBPF_ERRNO__FORMAT; in bpf_object__init_license()
1421 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't in bpf_object__init_license()
1424 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license))); in bpf_object__init_license()
1425 pr_debug("license of %s is %s\n", obj->path, obj->license); in bpf_object__init_license()
1435 pr_warn("invalid kver section in %s\n", obj->path); in bpf_object__init_kversion()
1436 return -LIBBPF_ERRNO__FORMAT; in bpf_object__init_kversion()
1439 obj->kern_version = kver; in bpf_object__init_kversion()
1440 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); in bpf_object__init_kversion()
1458 return -EINVAL; in find_elf_sec_sz()
1463 *size = data->d_size; in find_elf_sec_sz()
1467 return -ENOENT; in find_elf_sec_sz()
1472 Elf_Data *symbols = obj->efile.symbols; in find_elf_var_sym()
1476 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) { in find_elf_var_sym()
1479 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT) in find_elf_var_sym()
1482 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL && in find_elf_var_sym()
1483 ELF64_ST_BIND(sym->st_info) != STB_WEAK) in find_elf_var_sym()
1486 sname = elf_sym_str(obj, sym->st_name); in find_elf_var_sym()
1489 return ERR_PTR(-EIO); in find_elf_var_sym()
1495 return ERR_PTR(-ENOENT); in find_elf_var_sym()
1503 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap, in bpf_object__add_map()
1504 sizeof(*obj->maps), obj->nr_maps + 1); in bpf_object__add_map()
1508 map = &obj->maps[obj->nr_maps++]; in bpf_object__add_map()
1509 map->obj = obj; in bpf_object__add_map()
1510 map->fd = -1; in bpf_object__add_map()
1511 map->inner_map_fd = -1; in bpf_object__add_map()
1512 map->autocreate = true; in bpf_object__add_map()
1531 if (!map->mmaped) in bpf_map_mmap_resize()
1532 return -EINVAL; in bpf_map_mmap_resize()
1537 mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); in bpf_map_mmap_resize()
1539 return -errno; in bpf_map_mmap_resize()
1541 memcpy(mmaped, map->mmaped, min(old_sz, new_sz)); in bpf_map_mmap_resize()
1542 munmap(map->mmaped, old_sz); in bpf_map_mmap_resize()
1543 map->mmaped = mmaped; in bpf_map_mmap_resize()
1568 * name is '.kconfig', we'll still have entirety of '.kconfig' added in internal_map_name()
1571 * name will be "my_obje.kconfig". in internal_map_name()
1577 * '.rodata.abracad' kernel and user-visible name. in internal_map_name()
1586 sfx_len = BPF_OBJ_NAME_LEN - 1; in internal_map_name()
1592 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name)); in internal_map_name()
1594 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, in internal_map_name()
1619 if (!map->btf_value_type_id) in map_is_mmapable()
1622 t = btf__type_by_id(obj->btf, map->btf_value_type_id); in map_is_mmapable()
1628 vt = btf__type_by_id(obj->btf, vsi->type); in map_is_mmapable()
1632 if (btf_var(vt)->linkage != BTF_VAR_STATIC) in map_is_mmapable()
1652 map->libbpf_type = type; in bpf_object__init_internal_map()
1653 map->sec_idx = sec_idx; in bpf_object__init_internal_map()
1654 map->sec_offset = 0; in bpf_object__init_internal_map()
1655 map->real_name = strdup(real_name); in bpf_object__init_internal_map()
1656 map->name = internal_map_name(obj, real_name); in bpf_object__init_internal_map()
1657 if (!map->real_name || !map->name) { in bpf_object__init_internal_map()
1658 zfree(&map->real_name); in bpf_object__init_internal_map()
1659 zfree(&map->name); in bpf_object__init_internal_map()
1660 return -ENOMEM; in bpf_object__init_internal_map()
1663 def = &map->def; in bpf_object__init_internal_map()
1664 def->type = BPF_MAP_TYPE_ARRAY; in bpf_object__init_internal_map()
1665 def->key_size = sizeof(int); in bpf_object__init_internal_map()
1666 def->value_size = data_sz; in bpf_object__init_internal_map()
1667 def->max_entries = 1; in bpf_object__init_internal_map()
1668 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG in bpf_object__init_internal_map()
1675 def->map_flags |= BPF_F_MMAPABLE; in bpf_object__init_internal_map()
1678 map->name, map->sec_idx, map->sec_offset, def->map_flags); in bpf_object__init_internal_map()
1680 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); in bpf_object__init_internal_map()
1681 map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, in bpf_object__init_internal_map()
1682 MAP_SHARED | MAP_ANONYMOUS, -1, 0); in bpf_object__init_internal_map()
1683 if (map->mmaped == MAP_FAILED) { in bpf_object__init_internal_map()
1684 err = -errno; in bpf_object__init_internal_map()
1685 map->mmaped = NULL; in bpf_object__init_internal_map()
1687 map->name, err); in bpf_object__init_internal_map()
1688 zfree(&map->real_name); in bpf_object__init_internal_map()
1689 zfree(&map->name); in bpf_object__init_internal_map()
1694 memcpy(map->mmaped, data, data_sz); in bpf_object__init_internal_map()
1696 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); in bpf_object__init_internal_map()
1707 * Populate obj->maps with libbpf internal maps. in bpf_object__init_global_data_maps()
1709 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) { in bpf_object__init_global_data_maps()
1710 sec_desc = &obj->efile.secs[sec_idx]; in bpf_object__init_global_data_maps()
1713 if (!sec_desc->data || sec_desc->data->d_size == 0) in bpf_object__init_global_data_maps()
1716 switch (sec_desc->sec_type) { in bpf_object__init_global_data_maps()
1721 sec_desc->data->d_buf, in bpf_object__init_global_data_maps()
1722 sec_desc->data->d_size); in bpf_object__init_global_data_maps()
1725 obj->has_rodata = true; in bpf_object__init_global_data_maps()
1729 sec_desc->data->d_buf, in bpf_object__init_global_data_maps()
1730 sec_desc->data->d_size); in bpf_object__init_global_data_maps()
1737 sec_desc->data->d_size); in bpf_object__init_global_data_maps()
1755 for (i = 0; i < obj->nr_extern; i++) { in find_extern_by_name()
1756 if (strcmp(obj->externs[i].name, name) == 0) in find_extern_by_name()
1757 return &obj->externs[i]; in find_extern_by_name()
1762 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, in set_kcfg_value_tri() argument
1765 switch (ext->kcfg.type) { in set_kcfg_value_tri()
1769 ext->name, value); in set_kcfg_value_tri()
1770 return -EINVAL; in set_kcfg_value_tri()
1790 ext->name, value); in set_kcfg_value_tri()
1791 return -EINVAL; in set_kcfg_value_tri()
1793 ext->is_set = true; in set_kcfg_value_tri()
1797 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, in set_kcfg_value_str() argument
1802 if (ext->kcfg.type != KCFG_CHAR_ARR) { in set_kcfg_value_str()
1804 ext->name, value); in set_kcfg_value_str()
1805 return -EINVAL; in set_kcfg_value_str()
1809 if (value[len - 1] != '"') { in set_kcfg_value_str()
1811 ext->name, value); in set_kcfg_value_str()
1812 return -EINVAL; in set_kcfg_value_str()
1816 len -= 2; in set_kcfg_value_str()
1817 if (len >= ext->kcfg.sz) { in set_kcfg_value_str()
1819 ext->name, value, len, ext->kcfg.sz - 1); in set_kcfg_value_str()
1820 len = ext->kcfg.sz - 1; in set_kcfg_value_str()
1824 ext->is_set = true; in set_kcfg_value_str()
1836 err = -errno; in parse_u64()
1842 return -EINVAL; in parse_u64()
1847 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) in is_kcfg_value_in_range() argument
1849 int bit_sz = ext->kcfg.sz * 8; in is_kcfg_value_in_range()
1851 if (ext->kcfg.sz == 8) in is_kcfg_value_in_range()
1854 /* Validate that value stored in u64 fits in integer of `ext->sz` in is_kcfg_value_in_range()
1859 * -2^(Y-1) <= X <= 2^(Y-1) - 1 in is_kcfg_value_in_range()
1860 * 0 <= X + 2^(Y-1) <= 2^Y - 1 in is_kcfg_value_in_range()
1861 * 0 <= X + 2^(Y-1) < 2^Y in is_kcfg_value_in_range()
1863 * For unsigned target integer, check that all the (64 - Y) bits are in is_kcfg_value_in_range()
1866 if (ext->kcfg.is_signed) in is_kcfg_value_in_range()
1867 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); in is_kcfg_value_in_range()
1872 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, in set_kcfg_value_num() argument
1875 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR && in set_kcfg_value_num()
1876 ext->kcfg.type != KCFG_BOOL) { in set_kcfg_value_num()
1878 ext->name, (unsigned long long)value); in set_kcfg_value_num()
1879 return -EINVAL; in set_kcfg_value_num()
1881 if (ext->kcfg.type == KCFG_BOOL && value > 1) { in set_kcfg_value_num()
1883 ext->name, (unsigned long long)value); in set_kcfg_value_num()
1884 return -EINVAL; in set_kcfg_value_num()
1887 if (!is_kcfg_value_in_range(ext, value)) { in set_kcfg_value_num()
1889 ext->name, (unsigned long long)value, ext->kcfg.sz); in set_kcfg_value_num()
1890 return -ERANGE; in set_kcfg_value_num()
1892 switch (ext->kcfg.sz) { in set_kcfg_value_num()
1906 return -EINVAL; in set_kcfg_value_num()
1908 ext->is_set = true; in set_kcfg_value_num()
1915 struct extern_desc *ext; in bpf_object__process_kconfig_line() local
1927 return -EINVAL; in bpf_object__process_kconfig_line()
1932 if (buf[len - 1] == '\n') in bpf_object__process_kconfig_line()
1933 buf[len - 1] = '\0'; in bpf_object__process_kconfig_line()
1939 return -EINVAL; in bpf_object__process_kconfig_line()
1942 ext = find_extern_by_name(obj, buf); in bpf_object__process_kconfig_line()
1943 if (!ext || ext->is_set) in bpf_object__process_kconfig_line()
1946 ext_val = data + ext->kcfg.data_off; in bpf_object__process_kconfig_line()
1951 err = set_kcfg_value_tri(ext, ext_val, *value); in bpf_object__process_kconfig_line()
1954 err = set_kcfg_value_str(ext, ext_val, value); in bpf_object__process_kconfig_line()
1960 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value); in bpf_object__process_kconfig_line()
1963 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { in bpf_object__process_kconfig_line()
1964 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value); in bpf_object__process_kconfig_line()
1965 return -EINVAL; in bpf_object__process_kconfig_line()
1967 err = set_kcfg_value_num(ext, ext_val, num); in bpf_object__process_kconfig_line()
1972 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value); in bpf_object__process_kconfig_line()
1984 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); in bpf_object__read_kconfig_file()
1986 return -EINVAL; in bpf_object__read_kconfig_file()
1988 return -ENAMETOOLONG; in bpf_object__read_kconfig_file()
1996 pr_warn("failed to open system Kconfig\n"); in bpf_object__read_kconfig_file()
1997 return -ENOENT; in bpf_object__read_kconfig_file()
2003 pr_warn("error parsing system Kconfig line '%s': %d\n", in bpf_object__read_kconfig_file()
2023 err = -errno; in bpf_object__read_kconfig_mem()
2024 pr_warn("failed to open in-memory Kconfig: %d\n", err); in bpf_object__read_kconfig_mem()
2031 pr_warn("error parsing in-memory Kconfig line '%s': %d\n", in bpf_object__read_kconfig_mem()
2043 struct extern_desc *last_ext = NULL, *ext; in bpf_object__init_kconfig_map() local
2047 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__init_kconfig_map()
2048 ext = &obj->externs[i]; in bpf_object__init_kconfig_map()
2049 if (ext->type == EXT_KCFG) in bpf_object__init_kconfig_map()
2050 last_ext = ext; in bpf_object__init_kconfig_map()
2056 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz; in bpf_object__init_kconfig_map()
2058 ".kconfig", obj->efile.symbols_shndx, in bpf_object__init_kconfig_map()
2063 obj->kconfig_map_idx = obj->nr_maps - 1; in bpf_object__init_kconfig_map()
2078 *res_id = t->type; in skip_mods_and_typedefs()
2079 t = btf__type_by_id(btf, t->type); in skip_mods_and_typedefs()
2094 t = skip_mods_and_typedefs(btf, t->type, res_id); in resolve_func_ptr()
2141 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); in get_map_field_int()
2142 const char *name = btf__name_by_offset(btf, m->name_off); in get_map_field_int()
2152 arr_t = btf__type_by_id(btf, t->type); in get_map_field_int()
2155 map_name, name, t->type); in get_map_field_int()
2164 *res = arr_info->nelems; in get_map_field_int()
2174 return -EINVAL; in pathname_concat()
2176 return -ENAMETOOLONG; in pathname_concat()
2215 const char *name = btf__name_by_offset(btf, m->name_off); in parse_btf_map_def()
2219 return -EINVAL; in parse_btf_map_def()
2222 if (!get_map_field_int(map_name, btf, m, &map_def->map_type)) in parse_btf_map_def()
2223 return -EINVAL; in parse_btf_map_def()
2224 map_def->parts |= MAP_DEF_MAP_TYPE; in parse_btf_map_def()
2226 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries)) in parse_btf_map_def()
2227 return -EINVAL; in parse_btf_map_def()
2228 map_def->parts |= MAP_DEF_MAX_ENTRIES; in parse_btf_map_def()
2230 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags)) in parse_btf_map_def()
2231 return -EINVAL; in parse_btf_map_def()
2232 map_def->parts |= MAP_DEF_MAP_FLAGS; in parse_btf_map_def()
2234 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node)) in parse_btf_map_def()
2235 return -EINVAL; in parse_btf_map_def()
2236 map_def->parts |= MAP_DEF_NUMA_NODE; in parse_btf_map_def()
2241 return -EINVAL; in parse_btf_map_def()
2242 if (map_def->key_size && map_def->key_size != sz) { in parse_btf_map_def()
2244 map_name, map_def->key_size, sz); in parse_btf_map_def()
2245 return -EINVAL; in parse_btf_map_def()
2247 map_def->key_size = sz; in parse_btf_map_def()
2248 map_def->parts |= MAP_DEF_KEY_SIZE; in parse_btf_map_def()
2252 t = btf__type_by_id(btf, m->type); in parse_btf_map_def()
2255 map_name, m->type); in parse_btf_map_def()
2256 return -EINVAL; in parse_btf_map_def()
2261 return -EINVAL; in parse_btf_map_def()
2263 sz = btf__resolve_size(btf, t->type); in parse_btf_map_def()
2266 map_name, t->type, (ssize_t)sz); in parse_btf_map_def()
2269 if (map_def->key_size && map_def->key_size != sz) { in parse_btf_map_def()
2271 map_name, map_def->key_size, (ssize_t)sz); in parse_btf_map_def()
2272 return -EINVAL; in parse_btf_map_def()
2274 map_def->key_size = sz; in parse_btf_map_def()
2275 map_def->key_type_id = t->type; in parse_btf_map_def()
2276 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE; in parse_btf_map_def()
2281 return -EINVAL; in parse_btf_map_def()
2282 if (map_def->value_size && map_def->value_size != sz) { in parse_btf_map_def()
2284 map_name, map_def->value_size, sz); in parse_btf_map_def()
2285 return -EINVAL; in parse_btf_map_def()
2287 map_def->value_size = sz; in parse_btf_map_def()
2288 map_def->parts |= MAP_DEF_VALUE_SIZE; in parse_btf_map_def()
2292 t = btf__type_by_id(btf, m->type); in parse_btf_map_def()
2295 map_name, m->type); in parse_btf_map_def()
2296 return -EINVAL; in parse_btf_map_def()
2301 return -EINVAL; in parse_btf_map_def()
2303 sz = btf__resolve_size(btf, t->type); in parse_btf_map_def()
2306 map_name, t->type, (ssize_t)sz); in parse_btf_map_def()
2309 if (map_def->value_size && map_def->value_size != sz) { in parse_btf_map_def()
2311 map_name, map_def->value_size, (ssize_t)sz); in parse_btf_map_def()
2312 return -EINVAL; in parse_btf_map_def()
2314 map_def->value_size = sz; in parse_btf_map_def()
2315 map_def->value_type_id = t->type; in parse_btf_map_def()
2316 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE; in parse_btf_map_def()
2319 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type); in parse_btf_map_def()
2320 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY; in parse_btf_map_def()
2321 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value"; in parse_btf_map_def()
2326 pr_warn("map '%s': multi-level inner maps not supported.\n", in parse_btf_map_def()
2328 return -ENOTSUP; in parse_btf_map_def()
2330 if (i != vlen - 1) { in parse_btf_map_def()
2333 return -EINVAL; in parse_btf_map_def()
2336 pr_warn("map '%s': should be map-in-map or prog-array.\n", in parse_btf_map_def()
2338 return -ENOTSUP; in parse_btf_map_def()
2340 if (map_def->value_size && map_def->value_size != 4) { in parse_btf_map_def()
2342 map_name, map_def->value_size); in parse_btf_map_def()
2343 return -EINVAL; in parse_btf_map_def()
2345 map_def->value_size = 4; in parse_btf_map_def()
2346 t = btf__type_by_id(btf, m->type); in parse_btf_map_def()
2349 map_name, desc, m->type); in parse_btf_map_def()
2350 return -EINVAL; in parse_btf_map_def()
2352 if (!btf_is_array(t) || btf_array(t)->nelems) { in parse_btf_map_def()
2353 pr_warn("map '%s': %s spec is not a zero-sized array.\n", in parse_btf_map_def()
2355 return -EINVAL; in parse_btf_map_def()
2357 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL); in parse_btf_map_def()
2361 return -EINVAL; in parse_btf_map_def()
2363 t = skip_mods_and_typedefs(btf, t->type, NULL); in parse_btf_map_def()
2366 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n", in parse_btf_map_def()
2368 return -EINVAL; in parse_btf_map_def()
2373 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", in parse_btf_map_def()
2375 return -EINVAL; in parse_btf_map_def()
2383 map_def->parts |= MAP_DEF_INNER_MAP; in parse_btf_map_def()
2389 return -EINVAL; in parse_btf_map_def()
2392 return -EINVAL; in parse_btf_map_def()
2396 return -EINVAL; in parse_btf_map_def()
2398 map_def->pinning = val; in parse_btf_map_def()
2399 map_def->parts |= MAP_DEF_PINNING; in parse_btf_map_def()
2404 return -EINVAL; in parse_btf_map_def()
2405 map_def->map_extra = map_extra; in parse_btf_map_def()
2406 map_def->parts |= MAP_DEF_MAP_EXTRA; in parse_btf_map_def()
2410 return -ENOTSUP; in parse_btf_map_def()
2416 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) { in parse_btf_map_def()
2418 return -EINVAL; in parse_btf_map_def()
2433 * a power-of-2 multiple of kernel's page size. If user diligently in adjust_ringbuf_sz()
2440 * user-set size to satisfy both user size request and kernel in adjust_ringbuf_sz()
2449 * very close to UINT_MAX but is not a power-of-2 multiple of in adjust_ringbuf_sz()
2457 return map->def.type == BPF_MAP_TYPE_RINGBUF || in map_is_ringbuf()
2458 map->def.type == BPF_MAP_TYPE_USER_RINGBUF; in map_is_ringbuf()
2463 map->def.type = def->map_type; in fill_map_from_def()
2464 map->def.key_size = def->key_size; in fill_map_from_def()
2465 map->def.value_size = def->value_size; in fill_map_from_def()
2466 map->def.max_entries = def->max_entries; in fill_map_from_def()
2467 map->def.map_flags = def->map_flags; in fill_map_from_def()
2468 map->map_extra = def->map_extra; in fill_map_from_def()
2470 map->numa_node = def->numa_node; in fill_map_from_def()
2471 map->btf_key_type_id = def->key_type_id; in fill_map_from_def()
2472 map->btf_value_type_id = def->value_type_id; in fill_map_from_def()
2474 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ in fill_map_from_def()
2476 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); in fill_map_from_def()
2478 if (def->parts & MAP_DEF_MAP_TYPE) in fill_map_from_def()
2479 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); in fill_map_from_def()
2481 if (def->parts & MAP_DEF_KEY_TYPE) in fill_map_from_def()
2483 map->name, def->key_type_id, def->key_size); in fill_map_from_def()
2484 else if (def->parts & MAP_DEF_KEY_SIZE) in fill_map_from_def()
2485 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size); in fill_map_from_def()
2487 if (def->parts & MAP_DEF_VALUE_TYPE) in fill_map_from_def()
2489 map->name, def->value_type_id, def->value_size); in fill_map_from_def()
2490 else if (def->parts & MAP_DEF_VALUE_SIZE) in fill_map_from_def()
2491 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size); in fill_map_from_def()
2493 if (def->parts & MAP_DEF_MAX_ENTRIES) in fill_map_from_def()
2494 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries); in fill_map_from_def()
2495 if (def->parts & MAP_DEF_MAP_FLAGS) in fill_map_from_def()
2496 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags); in fill_map_from_def()
2497 if (def->parts & MAP_DEF_MAP_EXTRA) in fill_map_from_def()
2498 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name, in fill_map_from_def()
2499 (unsigned long long)def->map_extra); in fill_map_from_def()
2500 if (def->parts & MAP_DEF_PINNING) in fill_map_from_def()
2501 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning); in fill_map_from_def()
2502 if (def->parts & MAP_DEF_NUMA_NODE) in fill_map_from_def()
2503 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node); in fill_map_from_def()
2505 if (def->parts & MAP_DEF_INNER_MAP) in fill_map_from_def()
2506 pr_debug("map '%s': found inner map definition.\n", map->name); in fill_map_from_def()
2534 var = btf__type_by_id(obj->btf, vi->type); in bpf_object__init_user_btf_map()
2536 map_name = btf__name_by_offset(obj->btf, var->name_off); in bpf_object__init_user_btf_map()
2540 return -EINVAL; in bpf_object__init_user_btf_map()
2542 if ((__u64)vi->offset + vi->size > data->d_size) { in bpf_object__init_user_btf_map()
2544 return -EINVAL; in bpf_object__init_user_btf_map()
2549 return -EINVAL; in bpf_object__init_user_btf_map()
2551 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) { in bpf_object__init_user_btf_map()
2553 map_name, btf_var_linkage_str(var_extra->linkage)); in bpf_object__init_user_btf_map()
2554 return -EOPNOTSUPP; in bpf_object__init_user_btf_map()
2557 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); in bpf_object__init_user_btf_map()
2561 return -EINVAL; in bpf_object__init_user_btf_map()
2563 if (def->size > vi->size) { in bpf_object__init_user_btf_map()
2565 return -EINVAL; in bpf_object__init_user_btf_map()
2571 map->name = strdup(map_name); in bpf_object__init_user_btf_map()
2572 if (!map->name) { in bpf_object__init_user_btf_map()
2574 return -ENOMEM; in bpf_object__init_user_btf_map()
2576 map->libbpf_type = LIBBPF_MAP_UNSPEC; in bpf_object__init_user_btf_map()
2577 map->def.type = BPF_MAP_TYPE_UNSPEC; in bpf_object__init_user_btf_map()
2578 map->sec_idx = sec_idx; in bpf_object__init_user_btf_map()
2579 map->sec_offset = vi->offset; in bpf_object__init_user_btf_map()
2580 map->btf_var_idx = var_idx; in bpf_object__init_user_btf_map()
2582 map_name, map->sec_idx, map->sec_offset); in bpf_object__init_user_btf_map()
2584 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def); in bpf_object__init_user_btf_map()
2593 pr_warn("map '%s': couldn't build pin path.\n", map->name); in bpf_object__init_user_btf_map()
2599 map->inner_map = calloc(1, sizeof(*map->inner_map)); in bpf_object__init_user_btf_map()
2600 if (!map->inner_map) in bpf_object__init_user_btf_map()
2601 return -ENOMEM; in bpf_object__init_user_btf_map()
2602 map->inner_map->fd = -1; in bpf_object__init_user_btf_map()
2603 map->inner_map->sec_idx = sec_idx; in bpf_object__init_user_btf_map()
2604 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1); in bpf_object__init_user_btf_map()
2605 if (!map->inner_map->name) in bpf_object__init_user_btf_map()
2606 return -ENOMEM; in bpf_object__init_user_btf_map()
2607 sprintf(map->inner_map->name, "%s.inner", map_name); in bpf_object__init_user_btf_map()
2609 fill_map_from_def(map->inner_map, &inner_def); in bpf_object__init_user_btf_map()
2629 if (obj->efile.btf_maps_shndx < 0) in bpf_object__init_user_btf_maps()
2632 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); in bpf_object__init_user_btf_maps()
2636 MAPS_ELF_SEC, obj->path); in bpf_object__init_user_btf_maps()
2637 return -EINVAL; in bpf_object__init_user_btf_maps()
2640 nr_types = btf__type_cnt(obj->btf); in bpf_object__init_user_btf_maps()
2642 t = btf__type_by_id(obj->btf, i); in bpf_object__init_user_btf_maps()
2645 name = btf__name_by_offset(obj->btf, t->name_off); in bpf_object__init_user_btf_maps()
2648 obj->efile.btf_maps_sec_btf_id = i; in bpf_object__init_user_btf_maps()
2655 return -ENOENT; in bpf_object__init_user_btf_maps()
2661 obj->efile.btf_maps_shndx, in bpf_object__init_user_btf_maps()
2697 return sh->sh_flags & SHF_EXECINSTR; in section_have_execinstr()
2732 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); in bpf_object__sanitize_btf()
2738 t->size = 1; in bpf_object__sanitize_btf()
2747 name = (char *)btf__name_by_offset(btf, t->name_off); in bpf_object__sanitize_btf()
2755 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); in bpf_object__sanitize_btf()
2758 m->offset = v->offset * 8; in bpf_object__sanitize_btf()
2759 m->type = v->type; in bpf_object__sanitize_btf()
2761 vt = (void *)btf__type_by_id(btf, v->type); in bpf_object__sanitize_btf()
2762 m->name_off = vt->name_off; in bpf_object__sanitize_btf()
2767 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); in bpf_object__sanitize_btf()
2768 t->size = sizeof(__u32); /* kernel enforced */ in bpf_object__sanitize_btf()
2771 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); in bpf_object__sanitize_btf()
2774 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); in bpf_object__sanitize_btf()
2776 /* replace FLOAT with an equally-sized empty STRUCT; in bpf_object__sanitize_btf()
2780 t->name_off = 0; in bpf_object__sanitize_btf()
2781 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); in bpf_object__sanitize_btf()
2784 t->name_off = 0; in bpf_object__sanitize_btf()
2785 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0); in bpf_object__sanitize_btf()
2788 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false); in bpf_object__sanitize_btf()
2803 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen); in bpf_object__sanitize_btf()
2805 m->type = enum64_placeholder_id; in bpf_object__sanitize_btf()
2806 m->offset = 0; in bpf_object__sanitize_btf()
2816 return obj->efile.btf_maps_shndx >= 0 || in libbpf_needs_btf()
2817 obj->efile.st_ops_shndx >= 0 || in libbpf_needs_btf()
2818 obj->efile.st_ops_link_shndx >= 0 || in libbpf_needs_btf()
2819 obj->nr_extern > 0; in libbpf_needs_btf()
2824 return obj->efile.st_ops_shndx >= 0 || obj->efile.st_ops_link_shndx >= 0; in kernel_needs_btf()
2831 int err = -ENOENT; in bpf_object__init_btf()
2834 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); in bpf_object__init_btf()
2835 err = libbpf_get_error(obj->btf); in bpf_object__init_btf()
2837 obj->btf = NULL; in bpf_object__init_btf()
2841 /* enforce 8-byte pointers for BPF-targeted BTFs */ in bpf_object__init_btf()
2842 btf__set_pointer_size(obj->btf, 8); in bpf_object__init_btf()
2848 if (!obj->btf) { in bpf_object__init_btf()
2853 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); in bpf_object__init_btf()
2854 err = libbpf_get_error(obj->btf_ext); in bpf_object__init_btf()
2858 obj->btf_ext = NULL; in bpf_object__init_btf()
2862 /* setup .BTF.ext to ELF section mapping */ in bpf_object__init_btf()
2863 ext_segs[0] = &obj->btf_ext->func_info; in bpf_object__init_btf()
2864 ext_segs[1] = &obj->btf_ext->line_info; in bpf_object__init_btf()
2865 ext_segs[2] = &obj->btf_ext->core_relo_info; in bpf_object__init_btf()
2872 if (seg->sec_cnt == 0) in bpf_object__init_btf()
2875 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs)); in bpf_object__init_btf()
2876 if (!seg->sec_idxs) { in bpf_object__init_btf()
2877 err = -ENOMEM; in bpf_object__init_btf()
2888 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); in bpf_object__init_btf()
2895 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn); in bpf_object__init_btf()
2912 return a->offset - b->offset; in compare_vsi_off()
2919 const char *sec_name = btf__name_by_offset(btf, t->name_off); in btf_fixup_datasec()
2926 return -ENOENT; in btf_fixup_datasec()
2929 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and in btf_fixup_datasec()
2941 * to be optional. But the STV_HIDDEN handling is non-optional for any in btf_fixup_datasec()
2942 * non-extern DATASEC, so the variable fixup loop below handles both in btf_fixup_datasec()
2943 * functions at the same time, paying the cost of BTF VAR <-> ELF in btf_fixup_datasec()
2946 if (t->size == 0) { in btf_fixup_datasec()
2951 return -ENOENT; in btf_fixup_datasec()
2954 t->size = size; in btf_fixup_datasec()
2964 t_var = btf__type_by_id(btf, vsi->type); in btf_fixup_datasec()
2966 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name); in btf_fixup_datasec()
2967 return -EINVAL; in btf_fixup_datasec()
2971 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN) in btf_fixup_datasec()
2974 var_name = btf__name_by_offset(btf, t_var->name_off); in btf_fixup_datasec()
2978 return -ENOENT; in btf_fixup_datasec()
2985 return -ENOENT; in btf_fixup_datasec()
2989 vsi->offset = sym->st_value; in btf_fixup_datasec()
2998 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN in btf_fixup_datasec()
2999 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL) in btf_fixup_datasec()
3000 var->linkage = BTF_VAR_STATIC; in btf_fixup_datasec()
3012 if (!obj->btf) in bpf_object_fixup_btf()
3015 n = btf__type_cnt(obj->btf); in bpf_object_fixup_btf()
3017 struct btf_type *t = btf_type_by_id(obj->btf, i); in bpf_object_fixup_btf()
3025 err = btf_fixup_datasec(obj, obj->btf, t); in bpf_object_fixup_btf()
3036 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || in prog_needs_vmlinux_btf()
3037 prog->type == BPF_PROG_TYPE_LSM) in prog_needs_vmlinux_btf()
3043 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) in prog_needs_vmlinux_btf()
3054 /* CO-RE relocations need kernel BTF, only when btf_custom_path in obj_needs_vmlinux_btf()
3057 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path) in obj_needs_vmlinux_btf()
3061 for (i = 0; i < obj->nr_extern; i++) { in obj_needs_vmlinux_btf()
3062 const struct extern_desc *ext; in obj_needs_vmlinux_btf() local
3064 ext = &obj->externs[i]; in obj_needs_vmlinux_btf()
3065 if (ext->type == EXT_KSYM && ext->ksym.type_id) in obj_needs_vmlinux_btf()
3070 if (!prog->autoload) in obj_needs_vmlinux_btf()
3084 if (obj->btf_vmlinux || obj->gen_loader) in bpf_object__load_vmlinux_btf()
3090 obj->btf_vmlinux = btf__load_vmlinux_btf(); in bpf_object__load_vmlinux_btf()
3091 err = libbpf_get_error(obj->btf_vmlinux); in bpf_object__load_vmlinux_btf()
3094 obj->btf_vmlinux = NULL; in bpf_object__load_vmlinux_btf()
3102 struct btf *kern_btf = obj->btf; in bpf_object__sanitize_and_load_btf()
3106 if (!obj->btf) in bpf_object__sanitize_and_load_btf()
3111 err = -EOPNOTSUPP; in bpf_object__sanitize_and_load_btf()
3126 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__sanitize_and_load_btf()
3127 struct bpf_program *prog = &obj->programs[i]; in bpf_object__sanitize_and_load_btf()
3132 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) in bpf_object__sanitize_and_load_btf()
3135 n = btf__type_cnt(obj->btf); in bpf_object__sanitize_and_load_btf()
3137 t = btf_type_by_id(obj->btf, j); in bpf_object__sanitize_and_load_btf()
3141 name = btf__str_by_offset(obj->btf, t->name_off); in bpf_object__sanitize_and_load_btf()
3142 if (strcmp(name, prog->name) != 0) in bpf_object__sanitize_and_load_btf()
3145 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0); in bpf_object__sanitize_and_load_btf()
3152 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__sanitize_and_load_btf()
3153 struct bpf_program *prog = &obj->programs[i]; in bpf_object__sanitize_and_load_btf()
3158 n = btf__type_cnt(obj->btf); in bpf_object__sanitize_and_load_btf()
3164 t = btf_type_by_id(obj->btf, j); in bpf_object__sanitize_and_load_btf()
3165 if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1) in bpf_object__sanitize_and_load_btf()
3168 name = btf__str_by_offset(obj->btf, t->name_off); in bpf_object__sanitize_and_load_btf()
3172 t = btf_type_by_id(obj->btf, t->type); in bpf_object__sanitize_and_load_btf()
3175 prog->name); in bpf_object__sanitize_and_load_btf()
3176 return -EINVAL; in bpf_object__sanitize_and_load_btf()
3178 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off))) in bpf_object__sanitize_and_load_btf()
3184 if (prog->exception_cb_idx >= 0) { in bpf_object__sanitize_and_load_btf()
3185 prog->exception_cb_idx = -1; in bpf_object__sanitize_and_load_btf()
3192 prog->name); in bpf_object__sanitize_and_load_btf()
3193 return -EINVAL; in bpf_object__sanitize_and_load_btf()
3196 for (k = 0; k < obj->nr_programs; k++) { in bpf_object__sanitize_and_load_btf()
3197 struct bpf_program *subprog = &obj->programs[k]; in bpf_object__sanitize_and_load_btf()
3201 if (strcmp(name, subprog->name)) in bpf_object__sanitize_and_load_btf()
3203 /* Enforce non-hidden, as from verifier point of in bpf_object__sanitize_and_load_btf()
3207 if (!subprog->sym_global || subprog->mark_btf_static) { in bpf_object__sanitize_and_load_btf()
3208 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n", in bpf_object__sanitize_and_load_btf()
3209 prog->name, subprog->name); in bpf_object__sanitize_and_load_btf()
3210 return -EINVAL; in bpf_object__sanitize_and_load_btf()
3213 if (prog->exception_cb_idx >= 0) { in bpf_object__sanitize_and_load_btf()
3215 prog->name, subprog->name); in bpf_object__sanitize_and_load_btf()
3216 return -EINVAL; in bpf_object__sanitize_and_load_btf()
3218 prog->exception_cb_idx = k; in bpf_object__sanitize_and_load_btf()
3222 if (prog->exception_cb_idx >= 0) in bpf_object__sanitize_and_load_btf()
3224 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name); in bpf_object__sanitize_and_load_btf()
3225 return -ENOENT; in bpf_object__sanitize_and_load_btf()
3236 raw_data = btf__raw_data(obj->btf, &sz); in bpf_object__sanitize_and_load_btf()
3242 /* enforce 8-byte pointers for BPF-targeted BTFs */ in bpf_object__sanitize_and_load_btf()
3243 btf__set_pointer_size(obj->btf, 8); in bpf_object__sanitize_and_load_btf()
3249 if (obj->gen_loader) { in bpf_object__sanitize_and_load_btf()
3254 return -ENOMEM; in bpf_object__sanitize_and_load_btf()
3255 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size); in bpf_object__sanitize_and_load_btf()
3257 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. in bpf_object__sanitize_and_load_btf()
3262 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size, in bpf_object__sanitize_and_load_btf()
3263 obj->log_level ? 1 : 0); in bpf_object__sanitize_and_load_btf()
3268 btf__set_fd(obj->btf, btf__fd(kern_btf)); in bpf_object__sanitize_and_load_btf()
3269 btf__set_fd(kern_btf, -1); in bpf_object__sanitize_and_load_btf()
3289 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off); in elf_sym_str()
3292 off, obj->path, elf_errmsg(-1)); in elf_sym_str()
3303 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off); in elf_sec_str()
3306 off, obj->path, elf_errmsg(-1)); in elf_sec_str()
3317 scn = elf_getscn(obj->efile.elf, idx); in elf_sec_by_idx()
3320 idx, obj->path, elf_errmsg(-1)); in elf_sec_by_idx()
3329 Elf *elf = obj->efile.elf; in elf_sec_by_name()
3355 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); in elf_sec_hdr()
3374 name = elf_sec_str(obj, sh->sh_name); in elf_sec_name()
3377 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); in elf_sec_name()
3395 obj->path, elf_errmsg(-1)); in elf_sec_data()
3404 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym)) in elf_sym_by_idx()
3407 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx; in elf_sym_by_idx()
3412 if (idx >= data->d_size / sizeof(Elf64_Rel)) in elf_rel_by_idx()
3415 return (Elf64_Rel *)data->d_buf + idx; in elf_rel_by_idx()
3427 if (hdr->sh_type == SHT_STRTAB) in ignore_elf_section()
3431 if (hdr->sh_type == SHT_LLVM_ADDRSIG) in ignore_elf_section()
3435 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 && in ignore_elf_section()
3444 name += sizeof(".rel") - 1; in ignore_elf_section()
3449 /* .BTF and .BTF.ext don't need relocations */ in ignore_elf_section()
3463 if (a->sec_idx != b->sec_idx) in cmp_progs()
3464 return a->sec_idx < b->sec_idx ? -1 : 1; in cmp_progs()
3467 return a->sec_insn_off < b->sec_insn_off ? -1 : 1; in cmp_progs()
3473 Elf *elf = obj->efile.elf; in bpf_object__elf_collect()
3482 /* ELF section indices are 0-based, but sec #0 is special "invalid" in bpf_object__elf_collect()
3487 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) { in bpf_object__elf_collect()
3489 obj->path, elf_errmsg(-1)); in bpf_object__elf_collect()
3490 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3492 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs)); in bpf_object__elf_collect()
3493 if (!obj->efile.secs) in bpf_object__elf_collect()
3494 return -ENOMEM; in bpf_object__elf_collect()
3503 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3505 if (sh->sh_type == SHT_SYMTAB) { in bpf_object__elf_collect()
3506 if (obj->efile.symbols) { in bpf_object__elf_collect()
3507 pr_warn("elf: multiple symbol tables in %s\n", obj->path); in bpf_object__elf_collect()
3508 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3513 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3517 obj->efile.symbols = data; in bpf_object__elf_collect()
3518 obj->efile.symbols_shndx = idx; in bpf_object__elf_collect()
3519 obj->efile.strtabidx = sh->sh_link; in bpf_object__elf_collect()
3523 if (!obj->efile.symbols) { in bpf_object__elf_collect()
3525 obj->path); in bpf_object__elf_collect()
3526 return -ENOENT; in bpf_object__elf_collect()
3532 sec_desc = &obj->efile.secs[idx]; in bpf_object__elf_collect()
3536 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3538 name = elf_sec_str(obj, sh->sh_name); in bpf_object__elf_collect()
3540 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3547 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3550 idx, name, (unsigned long)data->d_size, in bpf_object__elf_collect()
3551 (int)sh->sh_link, (unsigned long)sh->sh_flags, in bpf_object__elf_collect()
3552 (int)sh->sh_type); in bpf_object__elf_collect()
3555 err = bpf_object__init_license(obj, data->d_buf, data->d_size); in bpf_object__elf_collect()
3559 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); in bpf_object__elf_collect()
3564 return -ENOTSUP; in bpf_object__elf_collect()
3566 obj->efile.btf_maps_shndx = idx; in bpf_object__elf_collect()
3568 if (sh->sh_type != SHT_PROGBITS) in bpf_object__elf_collect()
3569 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3572 if (sh->sh_type != SHT_PROGBITS) in bpf_object__elf_collect()
3573 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3575 } else if (sh->sh_type == SHT_SYMTAB) { in bpf_object__elf_collect()
3577 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) { in bpf_object__elf_collect()
3578 if (sh->sh_flags & SHF_EXECINSTR) { in bpf_object__elf_collect()
3580 obj->efile.text_shndx = idx; in bpf_object__elf_collect()
3586 sec_desc->sec_type = SEC_DATA; in bpf_object__elf_collect()
3587 sec_desc->shdr = sh; in bpf_object__elf_collect()
3588 sec_desc->data = data; in bpf_object__elf_collect()
3591 sec_desc->sec_type = SEC_RODATA; in bpf_object__elf_collect()
3592 sec_desc->shdr = sh; in bpf_object__elf_collect()
3593 sec_desc->data = data; in bpf_object__elf_collect()
3595 obj->efile.st_ops_data = data; in bpf_object__elf_collect()
3596 obj->efile.st_ops_shndx = idx; in bpf_object__elf_collect()
3598 obj->efile.st_ops_link_data = data; in bpf_object__elf_collect()
3599 obj->efile.st_ops_link_shndx = idx; in bpf_object__elf_collect()
3604 } else if (sh->sh_type == SHT_REL) { in bpf_object__elf_collect()
3605 int targ_sec_idx = sh->sh_info; /* points to other section */ in bpf_object__elf_collect()
3607 if (sh->sh_entsize != sizeof(Elf64_Rel) || in bpf_object__elf_collect()
3608 targ_sec_idx >= obj->efile.sec_cnt) in bpf_object__elf_collect()
3609 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3622 sec_desc->sec_type = SEC_RELO; in bpf_object__elf_collect()
3623 sec_desc->shdr = sh; in bpf_object__elf_collect()
3624 sec_desc->data = data; in bpf_object__elf_collect()
3625 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 || in bpf_object__elf_collect()
3627 sec_desc->sec_type = SEC_BSS; in bpf_object__elf_collect()
3628 sec_desc->shdr = sh; in bpf_object__elf_collect()
3629 sec_desc->data = data; in bpf_object__elf_collect()
3632 (size_t)sh->sh_size); in bpf_object__elf_collect()
3636 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { in bpf_object__elf_collect()
3637 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path); in bpf_object__elf_collect()
3638 return -LIBBPF_ERRNO__FORMAT; in bpf_object__elf_collect()
3641 /* sort BPF programs by section name and in-section instruction offset in bpf_object__elf_collect()
3644 if (obj->nr_programs) in bpf_object__elf_collect()
3645 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs); in bpf_object__elf_collect()
3652 int bind = ELF64_ST_BIND(sym->st_info); in sym_is_extern()
3654 return sym->st_shndx == SHN_UNDEF && in sym_is_extern()
3656 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE; in sym_is_extern()
3661 int bind = ELF64_ST_BIND(sym->st_info); in sym_is_subprog()
3662 int type = ELF64_ST_TYPE(sym->st_info); in sym_is_subprog()
3665 if (sym->st_shndx != text_shndx) in sym_is_subprog()
3683 return -ESRCH; in find_extern_btf_id()
3692 tname = btf__name_by_offset(btf, t->name_off); in find_extern_btf_id()
3697 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) in find_extern_btf_id()
3698 return -EINVAL; in find_extern_btf_id()
3701 return -EINVAL; in find_extern_btf_id()
3706 return -ENOENT; in find_extern_btf_id()
3715 return -ESRCH; in find_extern_sec_btf_id()
3726 if (vs->type == ext_btf_id) in find_extern_sec_btf_id()
3731 return -ENOENT; in find_extern_sec_btf_id()
3741 name = btf__name_by_offset(btf, t->name_off); in find_kcfg_type()
3750 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN; in find_kcfg_type()
3753 if (t->size == 1) in find_kcfg_type()
3755 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) in find_kcfg_type()
3760 if (t->size != 4) in find_kcfg_type()
3770 if (btf_array(t)->nelems == 0) in find_kcfg_type()
3772 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR) in find_kcfg_type()
3785 if (a->type != b->type) in cmp_externs()
3786 return a->type < b->type ? -1 : 1; in cmp_externs()
3788 if (a->type == EXT_KCFG) { in cmp_externs()
3790 if (a->kcfg.align != b->kcfg.align) in cmp_externs()
3791 return a->kcfg.align > b->kcfg.align ? -1 : 1; in cmp_externs()
3793 if (a->kcfg.sz != b->kcfg.sz) in cmp_externs()
3794 return a->kcfg.sz < b->kcfg.sz ? -1 : 1; in cmp_externs()
3798 return strcmp(a->name, b->name); in cmp_externs()
3836 vt = btf__type_by_id(btf, vs->type); in add_dummy_ksym_var()
3860 struct extern_desc *ext; in bpf_object__collect_externs() local
3867 if (!obj->efile.symbols) in bpf_object__collect_externs()
3870 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); in bpf_object__collect_externs()
3872 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym)) in bpf_object__collect_externs()
3873 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_externs()
3875 dummy_var_btf_id = add_dummy_ksym_var(obj->btf); in bpf_object__collect_externs()
3879 n = sh->sh_size / sh->sh_entsize; in bpf_object__collect_externs()
3886 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_externs()
3889 ext_name = elf_sym_str(obj, sym->st_name); in bpf_object__collect_externs()
3893 ext = obj->externs; in bpf_object__collect_externs()
3894 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); in bpf_object__collect_externs()
3895 if (!ext) in bpf_object__collect_externs()
3896 return -ENOMEM; in bpf_object__collect_externs()
3897 obj->externs = ext; in bpf_object__collect_externs()
3898 ext = &ext[obj->nr_extern]; in bpf_object__collect_externs()
3899 memset(ext, 0, sizeof(*ext)); in bpf_object__collect_externs()
3900 obj->nr_extern++; in bpf_object__collect_externs()
3902 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); in bpf_object__collect_externs()
3903 if (ext->btf_id <= 0) { in bpf_object__collect_externs()
3905 ext_name, ext->btf_id); in bpf_object__collect_externs()
3906 return ext->btf_id; in bpf_object__collect_externs()
3908 t = btf__type_by_id(obj->btf, ext->btf_id); in bpf_object__collect_externs()
3909 ext->name = btf__name_by_offset(obj->btf, t->name_off); in bpf_object__collect_externs()
3910 ext->sym_idx = i; in bpf_object__collect_externs()
3911 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK; in bpf_object__collect_externs()
3913 ext_essent_len = bpf_core_essential_name_len(ext->name); in bpf_object__collect_externs()
3914 ext->essent_name = NULL; in bpf_object__collect_externs()
3915 if (ext_essent_len != strlen(ext->name)) { in bpf_object__collect_externs()
3916 ext->essent_name = strndup(ext->name, ext_essent_len); in bpf_object__collect_externs()
3917 if (!ext->essent_name) in bpf_object__collect_externs()
3918 return -ENOMEM; in bpf_object__collect_externs()
3921 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); in bpf_object__collect_externs()
3922 if (ext->sec_btf_id <= 0) { in bpf_object__collect_externs()
3924 ext_name, ext->btf_id, ext->sec_btf_id); in bpf_object__collect_externs()
3925 return ext->sec_btf_id; in bpf_object__collect_externs()
3927 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); in bpf_object__collect_externs()
3928 sec_name = btf__name_by_offset(obj->btf, sec->name_off); in bpf_object__collect_externs()
3933 ext->name, KCONFIG_SEC); in bpf_object__collect_externs()
3934 return -ENOTSUP; in bpf_object__collect_externs()
3937 ext->type = EXT_KCFG; in bpf_object__collect_externs()
3938 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); in bpf_object__collect_externs()
3939 if (ext->kcfg.sz <= 0) { in bpf_object__collect_externs()
3941 ext_name, ext->kcfg.sz); in bpf_object__collect_externs()
3942 return ext->kcfg.sz; in bpf_object__collect_externs()
3944 ext->kcfg.align = btf__align_of(obj->btf, t->type); in bpf_object__collect_externs()
3945 if (ext->kcfg.align <= 0) { in bpf_object__collect_externs()
3947 ext_name, ext->kcfg.align); in bpf_object__collect_externs()
3948 return -EINVAL; in bpf_object__collect_externs()
3950 ext->kcfg.type = find_kcfg_type(obj->btf, t->type, in bpf_object__collect_externs()
3951 &ext->kcfg.is_signed); in bpf_object__collect_externs()
3952 if (ext->kcfg.type == KCFG_UNKNOWN) { in bpf_object__collect_externs()
3954 return -ENOTSUP; in bpf_object__collect_externs()
3958 ext->type = EXT_KSYM; in bpf_object__collect_externs()
3959 skip_mods_and_typedefs(obj->btf, t->type, in bpf_object__collect_externs()
3960 &ext->ksym.type_id); in bpf_object__collect_externs()
3963 return -ENOTSUP; in bpf_object__collect_externs()
3966 pr_debug("collected %d externs total\n", obj->nr_extern); in bpf_object__collect_externs()
3968 if (!obj->nr_extern) in bpf_object__collect_externs()
3972 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); in bpf_object__collect_externs()
3976 * pretending that each extern is a 8-byte variable in bpf_object__collect_externs()
3979 /* find existing 4-byte integer type in BTF to use for fake in bpf_object__collect_externs()
3982 int int_btf_id = find_int_btf_id(obj->btf); in bpf_object__collect_externs()
3984 * will be used to replace the vs->type and in bpf_object__collect_externs()
3990 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id); in bpf_object__collect_externs()
3991 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__collect_externs()
3992 ext = &obj->externs[i]; in bpf_object__collect_externs()
3993 if (ext->type != EXT_KSYM) in bpf_object__collect_externs()
3996 i, ext->sym_idx, ext->name); in bpf_object__collect_externs()
4005 vt = (void *)btf__type_by_id(obj->btf, vs->type); in bpf_object__collect_externs()
4006 ext_name = btf__name_by_offset(obj->btf, vt->name_off); in bpf_object__collect_externs()
4007 ext = find_extern_by_name(obj, ext_name); in bpf_object__collect_externs()
4008 if (!ext) { in bpf_object__collect_externs()
4011 return -ESRCH; in bpf_object__collect_externs()
4018 func_proto = btf__type_by_id(obj->btf, in bpf_object__collect_externs()
4019 vt->type); in bpf_object__collect_externs()
4027 dummy_var->name_off; in bpf_object__collect_externs()
4028 vs->type = dummy_var_btf_id; in bpf_object__collect_externs()
4029 vt->info &= ~0xffff; in bpf_object__collect_externs()
4030 vt->info |= BTF_FUNC_GLOBAL; in bpf_object__collect_externs()
4032 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED; in bpf_object__collect_externs()
4033 vt->type = int_btf_id; in bpf_object__collect_externs()
4035 vs->offset = off; in bpf_object__collect_externs()
4036 vs->size = sizeof(int); in bpf_object__collect_externs()
4038 sec->size = off; in bpf_object__collect_externs()
4043 /* for kcfg externs calculate their offsets within a .kconfig map */ in bpf_object__collect_externs()
4045 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__collect_externs()
4046 ext = &obj->externs[i]; in bpf_object__collect_externs()
4047 if (ext->type != EXT_KCFG) in bpf_object__collect_externs()
4050 ext->kcfg.data_off = roundup(off, ext->kcfg.align); in bpf_object__collect_externs()
4051 off = ext->kcfg.data_off + ext->kcfg.sz; in bpf_object__collect_externs()
4053 i, ext->sym_idx, ext->kcfg.data_off, ext->name); in bpf_object__collect_externs()
4055 sec->size = off; in bpf_object__collect_externs()
4060 t = btf__type_by_id(obj->btf, vs->type); in bpf_object__collect_externs()
4061 ext_name = btf__name_by_offset(obj->btf, t->name_off); in bpf_object__collect_externs()
4062 ext = find_extern_by_name(obj, ext_name); in bpf_object__collect_externs()
4063 if (!ext) { in bpf_object__collect_externs()
4066 return -ESRCH; in bpf_object__collect_externs()
4068 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; in bpf_object__collect_externs()
4069 vs->offset = ext->kcfg.data_off; in bpf_object__collect_externs()
4077 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1; in prog_is_subprog()
4089 if (!strcmp(prog->name, name)) in bpf_object__find_program_by_name()
4098 switch (obj->efile.secs[shndx].sec_type) { in bpf_object__shndx_is_data()
4111 return shndx == obj->efile.btf_maps_shndx; in bpf_object__shndx_is_maps()
4117 if (shndx == obj->efile.symbols_shndx) in bpf_object__section_to_libbpf_map_type()
4120 switch (obj->efile.secs[shndx].sec_type) { in bpf_object__section_to_libbpf_map_type()
4137 struct bpf_insn *insn = &prog->insns[insn_idx]; in bpf_program__record_reloc()
4138 size_t map_idx, nr_maps = prog->obj->nr_maps; in bpf_program__record_reloc()
4139 struct bpf_object *obj = prog->obj; in bpf_program__record_reloc()
4140 __u32 shdr_idx = sym->st_shndx; in bpf_program__record_reloc()
4147 prog->name, sym_name, insn_idx, insn->code); in bpf_program__record_reloc()
4148 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4152 int sym_idx = ELF64_R_SYM(rel->r_info); in bpf_program__record_reloc()
4153 int i, n = obj->nr_extern; in bpf_program__record_reloc()
4154 struct extern_desc *ext; in bpf_program__record_reloc() local
4157 ext = &obj->externs[i]; in bpf_program__record_reloc()
4158 if (ext->sym_idx == sym_idx) in bpf_program__record_reloc()
4163 prog->name, sym_name, sym_idx); in bpf_program__record_reloc()
4164 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4167 prog->name, i, ext->name, ext->sym_idx, insn_idx); in bpf_program__record_reloc()
4168 if (insn->code == (BPF_JMP | BPF_CALL)) in bpf_program__record_reloc()
4169 reloc_desc->type = RELO_EXTERN_CALL; in bpf_program__record_reloc()
4171 reloc_desc->type = RELO_EXTERN_LD64; in bpf_program__record_reloc()
4172 reloc_desc->insn_idx = insn_idx; in bpf_program__record_reloc()
4173 reloc_desc->ext_idx = i; in bpf_program__record_reloc()
4177 /* sub-program call relocation */ in bpf_program__record_reloc()
4179 if (insn->src_reg != BPF_PSEUDO_CALL) { in bpf_program__record_reloc()
4180 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); in bpf_program__record_reloc()
4181 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4184 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { in bpf_program__record_reloc()
4187 prog->name, sym_name, sym_sec_name); in bpf_program__record_reloc()
4188 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4190 if (sym->st_value % BPF_INSN_SZ) { in bpf_program__record_reloc()
4192 prog->name, sym_name, (size_t)sym->st_value); in bpf_program__record_reloc()
4193 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4195 reloc_desc->type = RELO_CALL; in bpf_program__record_reloc()
4196 reloc_desc->insn_idx = insn_idx; in bpf_program__record_reloc()
4197 reloc_desc->sym_off = sym->st_value; in bpf_program__record_reloc()
4203 prog->name, sym_name, shdr_idx); in bpf_program__record_reloc()
4204 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4208 if (sym_is_subprog(sym, obj->efile.text_shndx)) { in bpf_program__record_reloc()
4209 /* global_func: sym->st_value = offset in the section, insn->imm = 0. in bpf_program__record_reloc()
4210 * local_func: sym->st_value = 0, insn->imm = offset in the section. in bpf_program__record_reloc()
4212 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) { in bpf_program__record_reloc()
4214 prog->name, sym_name, (size_t)sym->st_value, insn->imm); in bpf_program__record_reloc()
4215 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4218 reloc_desc->type = RELO_SUBPROG_ADDR; in bpf_program__record_reloc()
4219 reloc_desc->insn_idx = insn_idx; in bpf_program__record_reloc()
4220 reloc_desc->sym_off = sym->st_value; in bpf_program__record_reloc()
4231 prog->name, sym_name, sym_sec_name); in bpf_program__record_reloc()
4232 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4235 map = &obj->maps[map_idx]; in bpf_program__record_reloc()
4236 if (map->libbpf_type != type || in bpf_program__record_reloc()
4237 map->sec_idx != sym->st_shndx || in bpf_program__record_reloc()
4238 map->sec_offset != sym->st_value) in bpf_program__record_reloc()
4241 prog->name, map_idx, map->name, map->sec_idx, in bpf_program__record_reloc()
4242 map->sec_offset, insn_idx); in bpf_program__record_reloc()
4247 prog->name, sym_sec_name, (size_t)sym->st_value); in bpf_program__record_reloc()
4248 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4250 reloc_desc->type = RELO_LD64; in bpf_program__record_reloc()
4251 reloc_desc->insn_idx = insn_idx; in bpf_program__record_reloc()
4252 reloc_desc->map_idx = map_idx; in bpf_program__record_reloc()
4253 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ in bpf_program__record_reloc()
4260 prog->name, sym_sec_name); in bpf_program__record_reloc()
4261 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4264 map = &obj->maps[map_idx]; in bpf_program__record_reloc()
4265 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx) in bpf_program__record_reloc()
4268 prog->name, map_idx, map->name, map->sec_idx, in bpf_program__record_reloc()
4269 map->sec_offset, insn_idx); in bpf_program__record_reloc()
4274 prog->name, sym_sec_name); in bpf_program__record_reloc()
4275 return -LIBBPF_ERRNO__RELOC; in bpf_program__record_reloc()
4278 reloc_desc->type = RELO_DATA; in bpf_program__record_reloc()
4279 reloc_desc->insn_idx = insn_idx; in bpf_program__record_reloc()
4280 reloc_desc->map_idx = map_idx; in bpf_program__record_reloc()
4281 reloc_desc->sym_off = sym->st_value; in bpf_program__record_reloc()
4287 return insn_idx >= prog->sec_insn_off && in prog_contains_insn()
4288 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; in prog_contains_insn()
4294 int l = 0, r = obj->nr_programs - 1, m; in find_prog_by_sec_insn()
4297 if (!obj->nr_programs) in find_prog_by_sec_insn()
4301 m = l + (r - l + 1) / 2; in find_prog_by_sec_insn()
4302 prog = &obj->programs[m]; in find_prog_by_sec_insn()
4304 if (prog->sec_idx < sec_idx || in find_prog_by_sec_insn()
4305 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) in find_prog_by_sec_insn()
4308 r = m - 1; in find_prog_by_sec_insn()
4313 prog = &obj->programs[l]; in find_prog_by_sec_insn()
4314 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) in find_prog_by_sec_insn()
4323 size_t sec_idx = shdr->sh_info, sym_idx; in bpf_object__collect_prog_relos()
4334 if (sec_idx >= obj->efile.sec_cnt) in bpf_object__collect_prog_relos()
4335 return -EINVAL; in bpf_object__collect_prog_relos()
4340 relo_sec_name = elf_sec_str(obj, shdr->sh_name); in bpf_object__collect_prog_relos()
4343 return -EINVAL; in bpf_object__collect_prog_relos()
4347 nrels = shdr->sh_size / shdr->sh_entsize; in bpf_object__collect_prog_relos()
4353 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_prog_relos()
4356 sym_idx = ELF64_R_SYM(rel->r_info); in bpf_object__collect_prog_relos()
4361 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_prog_relos()
4364 if (sym->st_shndx >= obj->efile.sec_cnt) { in bpf_object__collect_prog_relos()
4366 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i); in bpf_object__collect_prog_relos()
4367 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_prog_relos()
4370 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) { in bpf_object__collect_prog_relos()
4372 relo_sec_name, (size_t)rel->r_offset, i); in bpf_object__collect_prog_relos()
4373 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_prog_relos()
4376 insn_idx = rel->r_offset / BPF_INSN_SZ; in bpf_object__collect_prog_relos()
4383 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0) in bpf_object__collect_prog_relos()
4384 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)); in bpf_object__collect_prog_relos()
4386 sym_name = elf_sym_str(obj, sym->st_name); in bpf_object__collect_prog_relos()
4399 relos = libbpf_reallocarray(prog->reloc_desc, in bpf_object__collect_prog_relos()
4400 prog->nr_reloc + 1, sizeof(*relos)); in bpf_object__collect_prog_relos()
4402 return -ENOMEM; in bpf_object__collect_prog_relos()
4403 prog->reloc_desc = relos; in bpf_object__collect_prog_relos()
4406 insn_idx -= prog->sec_insn_off; in bpf_object__collect_prog_relos()
4407 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], in bpf_object__collect_prog_relos()
4412 prog->nr_reloc++; in bpf_object__collect_prog_relos()
4421 if (!obj->btf) in map_fill_btf_type_info()
4422 return -ENOENT; in map_fill_btf_type_info()
4424 /* if it's BTF-defined map, we don't need to search for type IDs. in map_fill_btf_type_info()
4428 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map)) in map_fill_btf_type_info()
4436 return -ENOENT; in map_fill_btf_type_info()
4438 id = btf__find_by_name(obj->btf, map->real_name); in map_fill_btf_type_info()
4442 map->btf_key_type_id = 0; in map_fill_btf_type_info()
4443 map->btf_value_type_id = id; in map_fill_btf_type_info()
4459 err = -errno; in bpf_get_map_info_from_fdinfo()
4467 info->type = val; in bpf_get_map_info_from_fdinfo()
4469 info->key_size = val; in bpf_get_map_info_from_fdinfo()
4471 info->value_size = val; in bpf_get_map_info_from_fdinfo()
4473 info->max_entries = val; in bpf_get_map_info_from_fdinfo()
4475 info->map_flags = val; in bpf_get_map_info_from_fdinfo()
4485 return map->autocreate; in bpf_map__autocreate()
4490 if (map->obj->loaded) in bpf_map__set_autocreate()
4491 return libbpf_err(-EBUSY); in bpf_map__set_autocreate()
4493 map->autocreate = autocreate; in bpf_map__set_autocreate()
4512 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0) in bpf_map__reuse_fd()
4513 new_name = strdup(map->name); in bpf_map__reuse_fd()
4518 return libbpf_err(-errno); in bpf_map__reuse_fd()
4527 err = -errno; in bpf_map__reuse_fd()
4531 err = zclose(map->fd); in bpf_map__reuse_fd()
4533 err = -errno; in bpf_map__reuse_fd()
4536 free(map->name); in bpf_map__reuse_fd()
4538 map->fd = new_fd; in bpf_map__reuse_fd()
4539 map->name = new_name; in bpf_map__reuse_fd()
4540 map->def.type = info.type; in bpf_map__reuse_fd()
4541 map->def.key_size = info.key_size; in bpf_map__reuse_fd()
4542 map->def.value_size = info.value_size; in bpf_map__reuse_fd()
4543 map->def.max_entries = info.max_entries; in bpf_map__reuse_fd()
4544 map->def.map_flags = info.map_flags; in bpf_map__reuse_fd()
4545 map->btf_key_type_id = info.btf_key_type_id; in bpf_map__reuse_fd()
4546 map->btf_value_type_id = info.btf_value_type_id; in bpf_map__reuse_fd()
4547 map->reused = true; in bpf_map__reuse_fd()
4548 map->map_extra = info.map_extra; in bpf_map__reuse_fd()
4561 return map->def.max_entries; in bpf_map__max_entries()
4566 if (!bpf_map_type__is_map_in_map(map->def.type)) in bpf_map__inner_map()
4569 return map->inner_map; in bpf_map__inner_map()
4574 if (map->obj->loaded) in bpf_map__set_max_entries()
4575 return libbpf_err(-EBUSY); in bpf_map__set_max_entries()
4577 map->def.max_entries = max_entries; in bpf_map__set_max_entries()
4579 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ in bpf_map__set_max_entries()
4581 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); in bpf_map__set_max_entries()
4596 if (obj->gen_loader) in bpf_object__probe_loading()
4614 return -ret; in bpf_object__probe_loading()
4663 ret = -errno; in probe_kern_global_data()
4666 __func__, cp, -ret); in probe_kern_global_data()
4766 BTF_TYPE_DECL_TAG_ENC(1, 2, -1), in probe_kern_btf_decl_tag()
4808 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) in probe_kern_exp_attach_type()
4820 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ in probe_kern_probe_read_kernel()
4843 ret = -errno; in probe_prog_bind_map()
4846 __func__, cp, -ret); in probe_prog_bind_map()
4904 return -errno; in probe_perf_link()
4909 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL); in probe_perf_link()
4910 err = -errno; /* close() can clobber errno */ in probe_perf_link()
4916 return link_fd < 0 && err == -EBADF; in probe_perf_link()
4935 return -errno; in probe_uprobe_multi_link()
4937 /* Creating uprobe in '/' binary should fail with -EBADF. */ in probe_uprobe_multi_link()
4942 link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts); in probe_uprobe_multi_link()
4943 err = -errno; /* close() can clobber errno */ in probe_uprobe_multi_link()
4949 return link_fd < 0 && err == -EBADF; in probe_uprobe_multi_link()
5037 "memcg-based memory accounting", probe_memcg_account,
5049 "BPF multi-uprobe link support", probe_uprobe_multi_link,
5058 if (obj && obj->gen_loader) in kernel_supports()
5064 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) { in kernel_supports()
5065 ret = feat->probe(); in kernel_supports()
5067 WRITE_ONCE(feat->res, FEAT_SUPPORTED); in kernel_supports()
5069 WRITE_ONCE(feat->res, FEAT_MISSING); in kernel_supports()
5071 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret); in kernel_supports()
5072 WRITE_ONCE(feat->res, FEAT_MISSING); in kernel_supports()
5076 return READ_ONCE(feat->res) == FEAT_SUPPORTED; in kernel_supports()
5096 return (map_info.type == map->def.type && in map_is_reuse_compat()
5097 map_info.key_size == map->def.key_size && in map_is_reuse_compat()
5098 map_info.value_size == map->def.value_size && in map_is_reuse_compat()
5099 map_info.max_entries == map->def.max_entries && in map_is_reuse_compat()
5100 map_info.map_flags == map->def.map_flags && in map_is_reuse_compat()
5101 map_info.map_extra == map->map_extra); in map_is_reuse_compat()
5110 pin_fd = bpf_obj_get(map->pin_path); in bpf_object__reuse_map()
5112 err = -errno; in bpf_object__reuse_map()
5113 if (err == -ENOENT) { in bpf_object__reuse_map()
5115 map->pin_path); in bpf_object__reuse_map()
5119 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); in bpf_object__reuse_map()
5121 map->pin_path, cp); in bpf_object__reuse_map()
5127 map->pin_path); in bpf_object__reuse_map()
5129 return -EINVAL; in bpf_object__reuse_map()
5137 map->pinned = true; in bpf_object__reuse_map()
5138 pr_debug("reused pinned map at '%s'\n", map->pin_path); in bpf_object__reuse_map()
5146 enum libbpf_map_type map_type = map->libbpf_type; in bpf_object__populate_internal_map()
5150 if (obj->gen_loader) { in bpf_object__populate_internal_map()
5151 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, in bpf_object__populate_internal_map()
5152 map->mmaped, map->def.value_size); in bpf_object__populate_internal_map()
5154 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); in bpf_object__populate_internal_map()
5157 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); in bpf_object__populate_internal_map()
5159 err = -errno; in bpf_object__populate_internal_map()
5162 map->name, cp); in bpf_object__populate_internal_map()
5166 /* Freeze .rodata and .kconfig map as read-only from syscall side. */ in bpf_object__populate_internal_map()
5168 err = bpf_map_freeze(map->fd); in bpf_object__populate_internal_map()
5170 err = -errno; in bpf_object__populate_internal_map()
5172 pr_warn("Error freezing map(%s) as read-only: %s\n", in bpf_object__populate_internal_map()
5173 map->name, cp); in bpf_object__populate_internal_map()
5185 struct bpf_map_def *def = &map->def; in bpf_object__create_map()
5190 map_name = map->name; in bpf_object__create_map()
5191 create_attr.map_ifindex = map->map_ifindex; in bpf_object__create_map()
5192 create_attr.map_flags = def->map_flags; in bpf_object__create_map()
5193 create_attr.numa_node = map->numa_node; in bpf_object__create_map()
5194 create_attr.map_extra = map->map_extra; in bpf_object__create_map()
5197 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; in bpf_object__create_map()
5199 if (obj->btf && btf__fd(obj->btf) >= 0) { in bpf_object__create_map()
5200 create_attr.btf_fd = btf__fd(obj->btf); in bpf_object__create_map()
5201 create_attr.btf_key_type_id = map->btf_key_type_id; in bpf_object__create_map()
5202 create_attr.btf_value_type_id = map->btf_value_type_id; in bpf_object__create_map()
5205 if (bpf_map_type__is_map_in_map(def->type)) { in bpf_object__create_map()
5206 if (map->inner_map) { in bpf_object__create_map()
5207 err = bpf_object__create_map(obj, map->inner_map, true); in bpf_object__create_map()
5210 map->name, err); in bpf_object__create_map()
5213 map->inner_map_fd = bpf_map__fd(map->inner_map); in bpf_object__create_map()
5215 if (map->inner_map_fd >= 0) in bpf_object__create_map()
5216 create_attr.inner_map_fd = map->inner_map_fd; in bpf_object__create_map()
5219 switch (def->type) { in bpf_object__create_map()
5236 map->btf_key_type_id = 0; in bpf_object__create_map()
5237 map->btf_value_type_id = 0; in bpf_object__create_map()
5242 if (obj->gen_loader) { in bpf_object__create_map()
5243 bpf_gen__map_create(obj->gen_loader, def->type, map_name, in bpf_object__create_map()
5244 def->key_size, def->value_size, def->max_entries, in bpf_object__create_map()
5245 &create_attr, is_inner ? -1 : map - obj->maps); in bpf_object__create_map()
5247 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. in bpf_object__create_map()
5249 map->fd = 0; in bpf_object__create_map()
5251 map->fd = bpf_map_create(def->type, map_name, in bpf_object__create_map()
5252 def->key_size, def->value_size, in bpf_object__create_map()
5253 def->max_entries, &create_attr); in bpf_object__create_map()
5255 if (map->fd < 0 && (create_attr.btf_key_type_id || in bpf_object__create_map()
5259 err = -errno; in bpf_object__create_map()
5262 map->name, cp, err); in bpf_object__create_map()
5266 map->btf_key_type_id = 0; in bpf_object__create_map()
5267 map->btf_value_type_id = 0; in bpf_object__create_map()
5268 map->fd = bpf_map_create(def->type, map_name, in bpf_object__create_map()
5269 def->key_size, def->value_size, in bpf_object__create_map()
5270 def->max_entries, &create_attr); in bpf_object__create_map()
5273 err = map->fd < 0 ? -errno : 0; in bpf_object__create_map()
5275 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { in bpf_object__create_map()
5276 if (obj->gen_loader) in bpf_object__create_map()
5277 map->inner_map->fd = -1; in bpf_object__create_map()
5278 bpf_map__destroy(map->inner_map); in bpf_object__create_map()
5279 zfree(&map->inner_map); in bpf_object__create_map()
5291 for (i = 0; i < map->init_slots_sz; i++) { in init_map_in_map_slots()
5292 if (!map->init_slots[i]) in init_map_in_map_slots()
5295 targ_map = map->init_slots[i]; in init_map_in_map_slots()
5298 if (obj->gen_loader) { in init_map_in_map_slots()
5299 bpf_gen__populate_outer_map(obj->gen_loader, in init_map_in_map_slots()
5300 map - obj->maps, i, in init_map_in_map_slots()
5301 targ_map - obj->maps); in init_map_in_map_slots()
5303 err = bpf_map_update_elem(map->fd, &i, &fd, 0); in init_map_in_map_slots()
5306 err = -errno; in init_map_in_map_slots()
5308 map->name, i, targ_map->name, fd, err); in init_map_in_map_slots()
5312 map->name, i, targ_map->name, fd); in init_map_in_map_slots()
5315 zfree(&map->init_slots); in init_map_in_map_slots()
5316 map->init_slots_sz = 0; in init_map_in_map_slots()
5327 if (obj->gen_loader) in init_prog_array_slots()
5328 return -ENOTSUP; in init_prog_array_slots()
5330 for (i = 0; i < map->init_slots_sz; i++) { in init_prog_array_slots()
5331 if (!map->init_slots[i]) in init_prog_array_slots()
5334 targ_prog = map->init_slots[i]; in init_prog_array_slots()
5337 err = bpf_map_update_elem(map->fd, &i, &fd, 0); in init_prog_array_slots()
5339 err = -errno; in init_prog_array_slots()
5341 map->name, i, targ_prog->name, fd, err); in init_prog_array_slots()
5345 map->name, i, targ_prog->name, fd); in init_prog_array_slots()
5348 zfree(&map->init_slots); in init_prog_array_slots()
5349 map->init_slots_sz = 0; in init_prog_array_slots()
5359 for (i = 0; i < obj->nr_maps; i++) { in bpf_object_init_prog_arrays()
5360 map = &obj->maps[i]; in bpf_object_init_prog_arrays()
5362 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY) in bpf_object_init_prog_arrays()
5367 zclose(map->fd); in bpf_object_init_prog_arrays()
5376 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) { in map_set_def_max_entries()
5382 map->name, nr_cpus); in map_set_def_max_entries()
5385 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); in map_set_def_max_entries()
5386 map->def.max_entries = nr_cpus; in map_set_def_max_entries()
5401 for (i = 0; i < obj->nr_maps; i++) { in bpf_object__create_maps()
5402 map = &obj->maps[i]; in bpf_object__create_maps()
5405 * (.rodata, .data, .kconfig, etc); later on, during program in bpf_object__create_maps()
5406 * loading, if we detect that at least one of the to-be-loaded in bpf_object__create_maps()
5411 * but also it allows to have CO-RE applications that use in bpf_object__create_maps()
5413 * If those global variable-using programs are not loaded at in bpf_object__create_maps()
5419 map->autocreate = false; in bpf_object__create_maps()
5421 if (!map->autocreate) { in bpf_object__create_maps()
5422 pr_debug("map '%s': skipped auto-creating...\n", map->name); in bpf_object__create_maps()
5432 if (map->pin_path) { in bpf_object__create_maps()
5436 map->name); in bpf_object__create_maps()
5439 if (retried && map->fd < 0) { in bpf_object__create_maps()
5441 map->name); in bpf_object__create_maps()
5442 err = -ENOENT; in bpf_object__create_maps()
5447 if (map->fd >= 0) { in bpf_object__create_maps()
5449 map->name, map->fd); in bpf_object__create_maps()
5456 map->name, map->fd); in bpf_object__create_maps()
5461 zclose(map->fd); in bpf_object__create_maps()
5466 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) { in bpf_object__create_maps()
5469 zclose(map->fd); in bpf_object__create_maps()
5475 if (map->pin_path && !map->pinned) { in bpf_object__create_maps()
5478 zclose(map->fd); in bpf_object__create_maps()
5479 if (!retried && err == -EEXIST) { in bpf_object__create_maps()
5483 pr_warn("map '%s': failed to auto-pin at '%s': %d\n", in bpf_object__create_maps()
5484 map->name, map->pin_path, err); in bpf_object__create_maps()
5494 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err); in bpf_object__create_maps()
5497 zclose(obj->maps[j].fd); in bpf_object__create_maps()
5511 * underscore is ignored by BPF CO-RE relocation during relocation matching.
5518 for (i = n - 5; i >= 0; i--) { in bpf_core_essential_name_len()
5530 free(cands->cands); in bpf_core_free_cands()
5547 local_t = btf__type_by_id(local_cand->btf, local_cand->id); in bpf_core_add_cands()
5548 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off); in bpf_core_add_cands()
5556 targ_name = btf__name_by_offset(targ_btf, t->name_off); in bpf_core_add_cands()
5567 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", in bpf_core_add_cands()
5568 local_cand->id, btf_kind_str(local_t), in bpf_core_add_cands()
5571 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1, in bpf_core_add_cands()
5572 sizeof(*cands->cands)); in bpf_core_add_cands()
5574 return -ENOMEM; in bpf_core_add_cands()
5576 cand = &new_cands[cands->len]; in bpf_core_add_cands()
5577 cand->btf = targ_btf; in bpf_core_add_cands()
5578 cand->id = i; in bpf_core_add_cands()
5580 cands->cands = new_cands; in bpf_core_add_cands()
5581 cands->len++; in bpf_core_add_cands()
5595 if (obj->btf_modules_loaded) in load_module_btfs()
5598 if (obj->gen_loader) in load_module_btfs()
5602 obj->btf_modules_loaded = true; in load_module_btfs()
5617 err = -errno; in load_module_btfs()
5626 err = -errno; in load_module_btfs()
5638 err = -errno; in load_module_btfs()
5643 /* ignore non-module BTFs */ in load_module_btfs()
5649 btf = btf_get_from_fd(fd, obj->btf_vmlinux); in load_module_btfs()
5657 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, in load_module_btfs()
5658 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); in load_module_btfs()
5662 mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; in load_module_btfs()
5664 mod_btf->btf = btf; in load_module_btfs()
5665 mod_btf->id = id; in load_module_btfs()
5666 mod_btf->fd = fd; in load_module_btfs()
5667 mod_btf->name = strdup(name); in load_module_btfs()
5668 if (!mod_btf->name) { in load_module_btfs()
5669 err = -ENOMEM; in load_module_btfs()
5697 return ERR_PTR(-EINVAL); in bpf_core_find_cands()
5699 local_name = btf__name_by_offset(local_btf, local_t->name_off); in bpf_core_find_cands()
5701 return ERR_PTR(-EINVAL); in bpf_core_find_cands()
5706 return ERR_PTR(-ENOMEM); in bpf_core_find_cands()
5709 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; in bpf_core_find_cands()
5715 if (cands->len) in bpf_core_find_cands()
5719 if (obj->btf_vmlinux_override) in bpf_core_find_cands()
5727 for (i = 0; i < obj->btf_module_cnt; i++) { in bpf_core_find_cands()
5729 obj->btf_modules[i].btf, in bpf_core_find_cands()
5730 obj->btf_modules[i].name, in bpf_core_find_cands()
5731 btf__type_cnt(obj->btf_vmlinux), in bpf_core_find_cands()
5744 * type-based CO-RE relocations and follow slightly different rules than
5745 * field-based relocations. This function assumes that root types were already
5746 * checked for name match. Beyond that initial root-level name check, names
5748 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5751 * - for ENUMs, the size is ignored;
5752 * - for INT, size and signedness are ignored;
5753 * - for ARRAY, dimensionality is ignored, element types are checked for
5755 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
5756 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5757 * - FUNC_PROTOs are compatible if they have compatible signature: same
5760 * more experience with using BPF CO-RE relocations.
5789 relos = libbpf_reallocarray(prog->reloc_desc, in record_relo_core()
5790 prog->nr_reloc + 1, sizeof(*relos)); in record_relo_core()
5792 return -ENOMEM; in record_relo_core()
5793 relo = &relos[prog->nr_reloc]; in record_relo_core()
5794 relo->type = RELO_CORE; in record_relo_core()
5795 relo->insn_idx = insn_idx; in record_relo_core()
5796 relo->core_relo = core_relo; in record_relo_core()
5797 prog->reloc_desc = relos; in record_relo_core()
5798 prog->nr_reloc++; in record_relo_core()
5807 for (i = 0; i < prog->nr_reloc; i++) { in find_relo_core()
5808 relo = &prog->reloc_desc[i]; in find_relo_core()
5809 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx) in find_relo_core()
5812 return relo->core_relo; in find_relo_core()
5827 const char *prog_name = prog->name; in bpf_core_resolve_relo()
5830 __u32 local_id = relo->type_id; in bpf_core_resolve_relo()
5835 return -EINVAL; in bpf_core_resolve_relo()
5837 local_name = btf__name_by_offset(local_btf, local_type->name_off); in bpf_core_resolve_relo()
5839 return -EINVAL; in bpf_core_resolve_relo()
5841 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL && in bpf_core_resolve_relo()
5843 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); in bpf_core_resolve_relo()
5875 if (obj->btf_ext->core_relo_info.len == 0) in bpf_object__relocate_core()
5879 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL); in bpf_object__relocate_core()
5880 err = libbpf_get_error(obj->btf_vmlinux_override); in bpf_object__relocate_core()
5893 seg = &obj->btf_ext->core_relo_info; in bpf_object__relocate_core()
5896 sec_idx = seg->sec_idxs[sec_num]; in bpf_object__relocate_core()
5899 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); in bpf_object__relocate_core()
5901 err = -EINVAL; in bpf_object__relocate_core()
5905 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info); in bpf_object__relocate_core()
5908 if (rec->insn_off % BPF_INSN_SZ) in bpf_object__relocate_core()
5909 return -EINVAL; in bpf_object__relocate_core()
5910 insn_idx = rec->insn_off / BPF_INSN_SZ; in bpf_object__relocate_core()
5915 * appends all the .BTF.ext info that used to belong to that in bpf_object__relocate_core()
5917 * This is similar to what x86-64 linker does for relocations. in bpf_object__relocate_core()
5921 …pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subpro… in bpf_object__relocate_core()
5925 /* no need to apply CO-RE relocation if the program is in bpf_object__relocate_core()
5928 if (!prog->autoload) in bpf_object__relocate_core()
5932 * program's frame of reference; (sub-)program code is not yet in bpf_object__relocate_core()
5933 * relocated, so it's enough to just subtract in-section offset in bpf_object__relocate_core()
5935 insn_idx = insn_idx - prog->sec_insn_off; in bpf_object__relocate_core()
5936 if (insn_idx >= prog->insns_cnt) in bpf_object__relocate_core()
5937 return -EINVAL; in bpf_object__relocate_core()
5938 insn = &prog->insns[insn_idx]; in bpf_object__relocate_core()
5943 prog->name, i, err); in bpf_object__relocate_core()
5947 if (prog->obj->gen_loader) in bpf_object__relocate_core()
5950 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res); in bpf_object__relocate_core()
5953 prog->name, i, err); in bpf_object__relocate_core()
5957 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); in bpf_object__relocate_core()
5960 prog->name, i, insn_idx, err); in bpf_object__relocate_core()
5967 /* obj->btf_vmlinux and module BTFs are freed after object load */ in bpf_object__relocate_core()
5968 btf__free(obj->btf_vmlinux_override); in bpf_object__relocate_core()
5969 obj->btf_vmlinux_override = NULL; in bpf_object__relocate_core()
5973 bpf_core_free_cands(entry->pvalue); in bpf_object__relocate_core()
5991 prog->name, relo_idx, insn_idx, map_idx, map->name); in poison_map_ldimm64()
5995 insn->code = BPF_JMP | BPF_CALL; in poison_map_ldimm64()
5996 insn->dst_reg = 0; in poison_map_ldimm64()
5997 insn->src_reg = 0; in poison_map_ldimm64()
5998 insn->off = 0; in poison_map_ldimm64()
6002 * where lower 123 is map index into obj->maps[] array in poison_map_ldimm64()
6004 insn->imm = POISON_LDIMM64_MAP_BASE + map_idx; in poison_map_ldimm64()
6016 int ext_idx, const struct extern_desc *ext) in poison_kfunc_call() argument
6019 prog->name, relo_idx, insn_idx, ext->name); in poison_kfunc_call()
6022 insn->code = BPF_JMP | BPF_CALL; in poison_kfunc_call()
6023 insn->dst_reg = 0; in poison_kfunc_call()
6024 insn->src_reg = 0; in poison_kfunc_call()
6025 insn->off = 0; in poison_kfunc_call()
6029 * where lower 123 is extern index into obj->externs[] array in poison_kfunc_call()
6031 insn->imm = POISON_CALL_KFUNC_BASE + ext_idx; in poison_kfunc_call()
6035 * - map references;
6036 * - global variable references;
6037 * - extern references.
6044 for (i = 0; i < prog->nr_reloc; i++) { in bpf_object__relocate_data()
6045 struct reloc_desc *relo = &prog->reloc_desc[i]; in bpf_object__relocate_data()
6046 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; in bpf_object__relocate_data()
6048 struct extern_desc *ext; in bpf_object__relocate_data() local
6050 switch (relo->type) { in bpf_object__relocate_data()
6052 map = &obj->maps[relo->map_idx]; in bpf_object__relocate_data()
6053 if (obj->gen_loader) { in bpf_object__relocate_data()
6055 insn[0].imm = relo->map_idx; in bpf_object__relocate_data()
6056 } else if (map->autocreate) { in bpf_object__relocate_data()
6058 insn[0].imm = map->fd; in bpf_object__relocate_data()
6060 poison_map_ldimm64(prog, i, relo->insn_idx, insn, in bpf_object__relocate_data()
6061 relo->map_idx, map); in bpf_object__relocate_data()
6065 map = &obj->maps[relo->map_idx]; in bpf_object__relocate_data()
6066 insn[1].imm = insn[0].imm + relo->sym_off; in bpf_object__relocate_data()
6067 if (obj->gen_loader) { in bpf_object__relocate_data()
6069 insn[0].imm = relo->map_idx; in bpf_object__relocate_data()
6070 } else if (map->autocreate) { in bpf_object__relocate_data()
6072 insn[0].imm = map->fd; in bpf_object__relocate_data()
6074 poison_map_ldimm64(prog, i, relo->insn_idx, insn, in bpf_object__relocate_data()
6075 relo->map_idx, map); in bpf_object__relocate_data()
6079 ext = &obj->externs[relo->ext_idx]; in bpf_object__relocate_data()
6080 if (ext->type == EXT_KCFG) { in bpf_object__relocate_data()
6081 if (obj->gen_loader) { in bpf_object__relocate_data()
6083 insn[0].imm = obj->kconfig_map_idx; in bpf_object__relocate_data()
6086 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; in bpf_object__relocate_data()
6088 insn[1].imm = ext->kcfg.data_off; in bpf_object__relocate_data()
6090 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */ in bpf_object__relocate_data()
6092 insn[0].imm = ext->ksym.kernel_btf_id; in bpf_object__relocate_data()
6093 insn[1].imm = ext->ksym.kernel_btf_obj_fd; in bpf_object__relocate_data()
6095 insn[0].imm = (__u32)ext->ksym.addr; in bpf_object__relocate_data()
6096 insn[1].imm = ext->ksym.addr >> 32; in bpf_object__relocate_data()
6101 ext = &obj->externs[relo->ext_idx]; in bpf_object__relocate_data()
6103 if (ext->is_set) { in bpf_object__relocate_data()
6104 insn[0].imm = ext->ksym.kernel_btf_id; in bpf_object__relocate_data()
6105 insn[0].off = ext->ksym.btf_fd_idx; in bpf_object__relocate_data()
6107 poison_kfunc_call(prog, i, relo->insn_idx, insn, in bpf_object__relocate_data()
6108 relo->ext_idx, ext); in bpf_object__relocate_data()
6114 prog->name, i); in bpf_object__relocate_data()
6115 return -EINVAL; in bpf_object__relocate_data()
6127 prog->name, i, relo->type); in bpf_object__relocate_data()
6128 return -EINVAL; in bpf_object__relocate_data()
6149 sec_idx = ext_info->sec_idxs[sec_num]; in adjust_prog_btf_ext_info()
6151 if (prog->sec_idx != sec_idx) in adjust_prog_btf_ext_info()
6157 if (insn_off < prog->sec_insn_off) in adjust_prog_btf_ext_info()
6159 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) in adjust_prog_btf_ext_info()
6164 copy_end = rec + ext_info->rec_size; in adjust_prog_btf_ext_info()
6168 return -ENOENT; in adjust_prog_btf_ext_info()
6170 /* append func/line info of a given (sub-)program to the main in adjust_prog_btf_ext_info()
6173 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size; in adjust_prog_btf_ext_info()
6174 new_sz = old_sz + (copy_end - copy_start); in adjust_prog_btf_ext_info()
6177 return -ENOMEM; in adjust_prog_btf_ext_info()
6179 *prog_rec_cnt = new_sz / ext_info->rec_size; in adjust_prog_btf_ext_info()
6180 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start); in adjust_prog_btf_ext_info()
6182 /* Kernel instruction offsets are in units of 8-byte in adjust_prog_btf_ext_info()
6183 * instructions, while .BTF.ext instruction offsets generated in adjust_prog_btf_ext_info()
6188 off_adj = prog->sub_insn_off - prog->sec_insn_off; in adjust_prog_btf_ext_info()
6191 for (; rec < rec_end; rec += ext_info->rec_size) { in adjust_prog_btf_ext_info()
6196 *prog_rec_sz = ext_info->rec_size; in adjust_prog_btf_ext_info()
6200 return -ENOENT; in adjust_prog_btf_ext_info()
6210 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't in reloc_prog_func_and_line_info()
6213 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC)) in reloc_prog_func_and_line_info()
6219 if (main_prog != prog && !main_prog->func_info) in reloc_prog_func_and_line_info()
6222 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, in reloc_prog_func_and_line_info()
6223 &main_prog->func_info, in reloc_prog_func_and_line_info()
6224 &main_prog->func_info_cnt, in reloc_prog_func_and_line_info()
6225 &main_prog->func_info_rec_size); in reloc_prog_func_and_line_info()
6227 if (err != -ENOENT) { in reloc_prog_func_and_line_info()
6228 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n", in reloc_prog_func_and_line_info()
6229 prog->name, err); in reloc_prog_func_and_line_info()
6232 if (main_prog->func_info) { in reloc_prog_func_and_line_info()
6237 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); in reloc_prog_func_and_line_info()
6241 …pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext … in reloc_prog_func_and_line_info()
6242 prog->name); in reloc_prog_func_and_line_info()
6247 if (main_prog != prog && !main_prog->line_info) in reloc_prog_func_and_line_info()
6250 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, in reloc_prog_func_and_line_info()
6251 &main_prog->line_info, in reloc_prog_func_and_line_info()
6252 &main_prog->line_info_cnt, in reloc_prog_func_and_line_info()
6253 &main_prog->line_info_rec_size); in reloc_prog_func_and_line_info()
6255 if (err != -ENOENT) { in reloc_prog_func_and_line_info()
6256 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n", in reloc_prog_func_and_line_info()
6257 prog->name, err); in reloc_prog_func_and_line_info()
6260 if (main_prog->line_info) { in reloc_prog_func_and_line_info()
6265 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); in reloc_prog_func_and_line_info()
6269 …pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line… in reloc_prog_func_and_line_info()
6270 prog->name); in reloc_prog_func_and_line_info()
6280 if (insn_idx == relo->insn_idx) in cmp_relo_by_insn_idx()
6282 return insn_idx < relo->insn_idx ? -1 : 1; in cmp_relo_by_insn_idx()
6287 if (!prog->nr_reloc) in find_prog_insn_relo()
6289 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, in find_prog_insn_relo()
6290 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); in find_prog_insn_relo()
6295 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc; in append_subprog_relos()
6301 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos)); in append_subprog_relos()
6307 return -ENOMEM; in append_subprog_relos()
6308 if (subprog->nr_reloc) in append_subprog_relos()
6309 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc, in append_subprog_relos()
6310 sizeof(*relos) * subprog->nr_reloc); in append_subprog_relos()
6312 for (i = main_prog->nr_reloc; i < new_cnt; i++) in append_subprog_relos()
6313 relos[i].insn_idx += subprog->sub_insn_off; in append_subprog_relos()
6317 main_prog->reloc_desc = relos; in append_subprog_relos()
6318 main_prog->nr_reloc = new_cnt; in append_subprog_relos()
6330 subprog->sub_insn_off = main_prog->insns_cnt; in bpf_object__append_subprog_code()
6332 new_cnt = main_prog->insns_cnt + subprog->insns_cnt; in bpf_object__append_subprog_code()
6333 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); in bpf_object__append_subprog_code()
6335 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); in bpf_object__append_subprog_code()
6336 return -ENOMEM; in bpf_object__append_subprog_code()
6338 main_prog->insns = insns; in bpf_object__append_subprog_code()
6339 main_prog->insns_cnt = new_cnt; in bpf_object__append_subprog_code()
6341 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, in bpf_object__append_subprog_code()
6342 subprog->insns_cnt * sizeof(*insns)); in bpf_object__append_subprog_code()
6344 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", in bpf_object__append_subprog_code()
6345 main_prog->name, subprog->insns_cnt, subprog->name); in bpf_object__append_subprog_code()
6368 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { in bpf_object__reloc_code()
6369 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; in bpf_object__reloc_code()
6374 if (relo && relo->type == RELO_EXTERN_CALL) in bpf_object__reloc_code()
6379 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { in bpf_object__reloc_code()
6381 prog->name, insn_idx, relo->type); in bpf_object__reloc_code()
6382 return -LIBBPF_ERRNO__RELOC; in bpf_object__reloc_code()
6385 /* sub-program instruction index is a combination of in bpf_object__reloc_code()
6388 * call always has imm = -1, but for static functions in bpf_object__reloc_code()
6389 * relocation is against STT_SECTION and insn->imm in bpf_object__reloc_code()
6392 * for subprog addr relocation, the relo->sym_off + insn->imm is in bpf_object__reloc_code()
6395 if (relo->type == RELO_CALL) in bpf_object__reloc_code()
6396 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; in bpf_object__reloc_code()
6398 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ; in bpf_object__reloc_code()
6405 prog->name, insn_idx); in bpf_object__reloc_code()
6406 return -LIBBPF_ERRNO__RELOC; in bpf_object__reloc_code()
6411 * offset necessary, insns->imm is relative to in bpf_object__reloc_code()
6414 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; in bpf_object__reloc_code()
6417 /* we enforce that sub-programs should be in .text section */ in bpf_object__reloc_code()
6418 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx); in bpf_object__reloc_code()
6420 pr_warn("prog '%s': no .text section found yet sub-program call exists\n", in bpf_object__reloc_code()
6421 prog->name); in bpf_object__reloc_code()
6422 return -LIBBPF_ERRNO__RELOC; in bpf_object__reloc_code()
6428 * - append it at the end of main program's instructions blog; in bpf_object__reloc_code()
6429 * - process is recursively, while current program is put on hold; in bpf_object__reloc_code()
6430 * - if that subprogram calls some other not yet processes in bpf_object__reloc_code()
6435 if (subprog->sub_insn_off == 0) { in bpf_object__reloc_code()
6444 /* main_prog->insns memory could have been re-allocated, so in bpf_object__reloc_code()
6447 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; in bpf_object__reloc_code()
6454 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; in bpf_object__reloc_code()
6457 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); in bpf_object__reloc_code()
6464 * Relocate sub-program calls.
6466 * Algorithm operates as follows. Each entry-point BPF program (referred to as
6467 * main prog) is processed separately. For each subprog (non-entry functions,
6476 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6492 * subprog->sub_insn_off as zero at all times and won't be appended to current
6501 * +--------+ +-------+
6503 * +--+---+ +--+-+-+ +---+--+
6505 * +--+---+ +------+ +---+--+
6508 * +---+-------+ +------+----+
6510 * +-----------+ +-----------+
6515 * +-----------+------+
6517 * +-----------+------+
6522 * +-----------+------+------+
6524 * +-----------+------+------+
6533 * +-----------+------+
6535 * +-----------+------+
6538 * +-----------+------+------+
6540 * +-----------+------+------+
6553 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__relocate_calls()
6554 subprog = &obj->programs[i]; in bpf_object__relocate_calls()
6558 subprog->sub_insn_off = 0; in bpf_object__relocate_calls()
6575 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__free_relocs()
6576 prog = &obj->programs[i]; in bpf_object__free_relocs()
6577 zfree(&prog->reloc_desc); in bpf_object__free_relocs()
6578 prog->nr_reloc = 0; in bpf_object__free_relocs()
6587 if (a->insn_idx != b->insn_idx) in cmp_relocs()
6588 return a->insn_idx < b->insn_idx ? -1 : 1; in cmp_relocs()
6591 if (a->type != b->type) in cmp_relocs()
6592 return a->type < b->type ? -1 : 1; in cmp_relocs()
6601 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__sort_relos()
6602 struct bpf_program *p = &obj->programs[i]; in bpf_object__sort_relos()
6604 if (!p->nr_reloc) in bpf_object__sort_relos()
6607 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); in bpf_object__sort_relos()
6618 if (obj->btf_ext) { in bpf_object__relocate()
6621 pr_warn("failed to perform CO-RE relocations: %d\n", in bpf_object__relocate()
6628 /* Before relocating calls pre-process relocations and mark in bpf_object__relocate()
6635 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__relocate()
6636 prog = &obj->programs[i]; in bpf_object__relocate()
6637 for (j = 0; j < prog->nr_reloc; j++) { in bpf_object__relocate()
6638 struct reloc_desc *relo = &prog->reloc_desc[j]; in bpf_object__relocate()
6639 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; in bpf_object__relocate()
6642 if (relo->type == RELO_SUBPROG_ADDR) in bpf_object__relocate()
6654 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__relocate()
6655 prog = &obj->programs[i]; in bpf_object__relocate()
6656 /* sub-program's sub-calls are relocated within the context of in bpf_object__relocate()
6661 if (!prog->autoload) in bpf_object__relocate()
6667 prog->name, err); in bpf_object__relocate()
6672 if (prog->exception_cb_idx >= 0) { in bpf_object__relocate()
6673 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx]; in bpf_object__relocate()
6680 if (subprog->sub_insn_off == 0) { in bpf_object__relocate()
6691 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__relocate()
6692 prog = &obj->programs[i]; in bpf_object__relocate()
6695 if (!prog->autoload) in bpf_object__relocate()
6700 prog->name, err); in bpf_object__relocate()
6728 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) in bpf_object__collect_map_relos()
6729 return -EINVAL; in bpf_object__collect_map_relos()
6730 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); in bpf_object__collect_map_relos()
6732 return -EINVAL; in bpf_object__collect_map_relos()
6734 nrels = shdr->sh_size / shdr->sh_entsize; in bpf_object__collect_map_relos()
6739 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_map_relos()
6742 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); in bpf_object__collect_map_relos()
6745 i, (size_t)ELF64_R_SYM(rel->r_info)); in bpf_object__collect_map_relos()
6746 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_map_relos()
6748 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; in bpf_object__collect_map_relos()
6750 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n", in bpf_object__collect_map_relos()
6751 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value, in bpf_object__collect_map_relos()
6752 (size_t)rel->r_offset, sym->st_name, name); in bpf_object__collect_map_relos()
6754 for (j = 0; j < obj->nr_maps; j++) { in bpf_object__collect_map_relos()
6755 map = &obj->maps[j]; in bpf_object__collect_map_relos()
6756 if (map->sec_idx != obj->efile.btf_maps_shndx) in bpf_object__collect_map_relos()
6759 vi = btf_var_secinfos(sec) + map->btf_var_idx; in bpf_object__collect_map_relos()
6760 if (vi->offset <= rel->r_offset && in bpf_object__collect_map_relos()
6761 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size) in bpf_object__collect_map_relos()
6764 if (j == obj->nr_maps) { in bpf_object__collect_map_relos()
6765 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n", in bpf_object__collect_map_relos()
6766 i, name, (size_t)rel->r_offset); in bpf_object__collect_map_relos()
6767 return -EINVAL; in bpf_object__collect_map_relos()
6770 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type); in bpf_object__collect_map_relos()
6771 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY; in bpf_object__collect_map_relos()
6774 if (sym->st_shndx != obj->efile.btf_maps_shndx) { in bpf_object__collect_map_relos()
6775 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", in bpf_object__collect_map_relos()
6777 return -LIBBPF_ERRNO__RELOC; in bpf_object__collect_map_relos()
6779 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && in bpf_object__collect_map_relos()
6780 map->def.key_size != sizeof(int)) { in bpf_object__collect_map_relos()
6781 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", in bpf_object__collect_map_relos()
6782 i, map->name, sizeof(int)); in bpf_object__collect_map_relos()
6783 return -EINVAL; in bpf_object__collect_map_relos()
6789 return -ESRCH; in bpf_object__collect_map_relos()
6796 return -ESRCH; in bpf_object__collect_map_relos()
6798 if (targ_prog->sec_idx != sym->st_shndx || in bpf_object__collect_map_relos()
6799 targ_prog->sec_insn_off * 8 != sym->st_value || in bpf_object__collect_map_relos()
6801 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n", in bpf_object__collect_map_relos()
6803 return -LIBBPF_ERRNO__RELOC; in bpf_object__collect_map_relos()
6806 return -EINVAL; in bpf_object__collect_map_relos()
6809 var = btf__type_by_id(obj->btf, vi->type); in bpf_object__collect_map_relos()
6810 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); in bpf_object__collect_map_relos()
6812 return -EINVAL; in bpf_object__collect_map_relos()
6813 member = btf_members(def) + btf_vlen(def) - 1; in bpf_object__collect_map_relos()
6814 mname = btf__name_by_offset(obj->btf, member->name_off); in bpf_object__collect_map_relos()
6816 return -EINVAL; in bpf_object__collect_map_relos()
6818 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; in bpf_object__collect_map_relos()
6819 if (rel->r_offset - vi->offset < moff) in bpf_object__collect_map_relos()
6820 return -EINVAL; in bpf_object__collect_map_relos()
6822 moff = rel->r_offset - vi->offset - moff; in bpf_object__collect_map_relos()
6827 return -EINVAL; in bpf_object__collect_map_relos()
6829 if (moff >= map->init_slots_sz) { in bpf_object__collect_map_relos()
6831 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz); in bpf_object__collect_map_relos()
6833 return -ENOMEM; in bpf_object__collect_map_relos()
6834 map->init_slots = tmp; in bpf_object__collect_map_relos()
6835 memset(map->init_slots + map->init_slots_sz, 0, in bpf_object__collect_map_relos()
6836 (new_sz - map->init_slots_sz) * host_ptr_sz); in bpf_object__collect_map_relos()
6837 map->init_slots_sz = new_sz; in bpf_object__collect_map_relos()
6839 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog; in bpf_object__collect_map_relos()
6842 i, map->name, moff, type, name); in bpf_object__collect_map_relos()
6852 for (i = 0; i < obj->efile.sec_cnt; i++) { in bpf_object__collect_relos()
6853 struct elf_sec_desc *sec_desc = &obj->efile.secs[i]; in bpf_object__collect_relos()
6858 if (sec_desc->sec_type != SEC_RELO) in bpf_object__collect_relos()
6861 shdr = sec_desc->shdr; in bpf_object__collect_relos()
6862 data = sec_desc->data; in bpf_object__collect_relos()
6863 idx = shdr->sh_info; in bpf_object__collect_relos()
6865 if (shdr->sh_type != SHT_REL) { in bpf_object__collect_relos()
6867 return -LIBBPF_ERRNO__INTERNAL; in bpf_object__collect_relos()
6870 if (idx == obj->efile.st_ops_shndx || idx == obj->efile.st_ops_link_shndx) in bpf_object__collect_relos()
6872 else if (idx == obj->efile.btf_maps_shndx) in bpf_object__collect_relos()
6886 if (BPF_CLASS(insn->code) == BPF_JMP && in insn_is_helper_call()
6887 BPF_OP(insn->code) == BPF_CALL && in insn_is_helper_call()
6888 BPF_SRC(insn->code) == BPF_K && in insn_is_helper_call()
6889 insn->src_reg == 0 && in insn_is_helper_call()
6890 insn->dst_reg == 0) { in insn_is_helper_call()
6891 *func_id = insn->imm; in insn_is_helper_call()
6899 struct bpf_insn *insn = prog->insns; in bpf_object__sanitize_prog()
6903 if (obj->gen_loader) in bpf_object__sanitize_prog()
6906 for (i = 0; i < prog->insns_cnt; i++, insn++) { in bpf_object__sanitize_prog()
6918 insn->imm = BPF_FUNC_probe_read; in bpf_object__sanitize_prog()
6923 insn->imm = BPF_FUNC_probe_read_str; in bpf_object__sanitize_prog()
6935 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
6942 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) in libbpf_prepare_prog_load()
6943 opts->expected_attach_type = 0; in libbpf_prepare_prog_load()
6946 opts->prog_flags |= BPF_F_SLEEPABLE; in libbpf_prepare_prog_load()
6948 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) in libbpf_prepare_prog_load()
6949 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; in libbpf_prepare_prog_load()
6952 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) in libbpf_prepare_prog_load()
6953 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; in libbpf_prepare_prog_load()
6955 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { in libbpf_prepare_prog_load()
6959 attach_name = strchr(prog->sec_name, '/'); in libbpf_prepare_prog_load()
6970 …pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\… in libbpf_prepare_prog_load()
6971 prog->name); in libbpf_prepare_prog_load()
6972 return -EINVAL; in libbpf_prepare_prog_load()
6981 prog->attach_btf_obj_fd = btf_obj_fd; in libbpf_prepare_prog_load()
6982 prog->attach_btf_id = btf_type_id; in libbpf_prepare_prog_load()
6985 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because in libbpf_prepare_prog_load()
6989 opts->attach_btf_obj_fd = btf_obj_fd; in libbpf_prepare_prog_load()
6990 opts->attach_btf_id = btf_type_id; in libbpf_prepare_prog_load()
7008 __u32 log_level = prog->log_level; in bpf_object_load_prog()
7010 if (prog->type == BPF_PROG_TYPE_UNSPEC) { in bpf_object_load_prog()
7016 prog->name, prog->sec_name); in bpf_object_load_prog()
7017 return -EINVAL; in bpf_object_load_prog()
7021 return -EINVAL; in bpf_object_load_prog()
7024 prog_name = prog->name; in bpf_object_load_prog()
7025 load_attr.attach_prog_fd = prog->attach_prog_fd; in bpf_object_load_prog()
7026 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; in bpf_object_load_prog()
7027 load_attr.attach_btf_id = prog->attach_btf_id; in bpf_object_load_prog()
7029 load_attr.prog_ifindex = prog->prog_ifindex; in bpf_object_load_prog()
7035 load_attr.func_info = prog->func_info; in bpf_object_load_prog()
7036 load_attr.func_info_rec_size = prog->func_info_rec_size; in bpf_object_load_prog()
7037 load_attr.func_info_cnt = prog->func_info_cnt; in bpf_object_load_prog()
7038 load_attr.line_info = prog->line_info; in bpf_object_load_prog()
7039 load_attr.line_info_rec_size = prog->line_info_rec_size; in bpf_object_load_prog()
7040 load_attr.line_info_cnt = prog->line_info_cnt; in bpf_object_load_prog()
7043 load_attr.prog_flags = prog->prog_flags; in bpf_object_load_prog()
7044 load_attr.fd_array = obj->fd_array; in bpf_object_load_prog()
7047 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) { in bpf_object_load_prog()
7048 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie); in bpf_object_load_prog()
7051 prog->name, err); in bpf_object_load_prog()
7054 insns = prog->insns; in bpf_object_load_prog()
7055 insns_cnt = prog->insns_cnt; in bpf_object_load_prog()
7059 load_attr.expected_attach_type = prog->expected_attach_type; in bpf_object_load_prog()
7061 if (obj->gen_loader) { in bpf_object_load_prog()
7062 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, in bpf_object_load_prog()
7064 prog - obj->programs); in bpf_object_load_prog()
7065 *prog_fd = -1; in bpf_object_load_prog()
7076 if (prog->log_buf) { in bpf_object_load_prog()
7077 log_buf = prog->log_buf; in bpf_object_load_prog()
7078 log_buf_size = prog->log_size; in bpf_object_load_prog()
7080 } else if (obj->log_buf) { in bpf_object_load_prog()
7081 log_buf = obj->log_buf; in bpf_object_load_prog()
7082 log_buf_size = obj->log_size; in bpf_object_load_prog()
7088 ret = -ENOMEM; in bpf_object_load_prog()
7101 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); in bpf_object_load_prog()
7104 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", in bpf_object_load_prog()
7105 prog->name, log_buf); in bpf_object_load_prog()
7108 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) { in bpf_object_load_prog()
7112 for (i = 0; i < obj->nr_maps; i++) { in bpf_object_load_prog()
7113 map = &prog->obj->maps[i]; in bpf_object_load_prog()
7114 if (map->libbpf_type != LIBBPF_MAP_RODATA) in bpf_object_load_prog()
7120 prog->name, map->real_name, cp); in bpf_object_load_prog()
7140 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2). in bpf_object_load_prog()
7145 ret = -errno; in bpf_object_load_prog()
7147 /* post-process verifier log to improve error descriptions */ in bpf_object_load_prog()
7151 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp); in bpf_object_load_prog()
7155 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", in bpf_object_load_prog()
7156 prog->name, log_buf); in bpf_object_load_prog()
7172 p = cur - 1; in find_prev_line()
7173 while (p - 1 >= buf && *(p - 1) != '\n') in find_prev_line()
7174 p--; in find_prev_line()
7182 /* size of the remaining log content to the right from the to-be-replaced part */ in patch_log()
7183 size_t rem_sz = (buf + log_sz) - (orig + orig_sz); in patch_log()
7188 * shift log contents by (patch_sz - orig_sz) bytes to the right in patch_log()
7189 * starting from after to-be-replaced part of the log. in patch_log()
7192 * shift log contents by (orig_sz - patch_sz) bytes to the left in patch_log()
7193 * starting from after to-be-replaced part of the log in patch_log()
7202 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1; in patch_log()
7204 } else if (patch_sz - orig_sz > buf_sz - log_sz) { in patch_log()
7206 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz); in patch_log()
7220 /* Expected log for failed and not properly guarded CO-RE relocation: in fixup_log_failed_core_relo()
7221 * line1 -> 123: (85) call unknown#195896080 in fixup_log_failed_core_relo()
7222 * line2 -> invalid func unknown#195896080 in fixup_log_failed_core_relo()
7223 * line3 -> <anything else or end of buffer> in fixup_log_failed_core_relo()
7226 * instruction index to find corresponding CO-RE relocation and in fixup_log_failed_core_relo()
7228 * failed CO-RE relocation. in fixup_log_failed_core_relo()
7242 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec); in fixup_log_failed_core_relo()
7248 "%d: <invalid CO-RE relocation>\n" in fixup_log_failed_core_relo()
7249 "failed to resolve CO-RE relocation %s%s\n", in fixup_log_failed_core_relo()
7252 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); in fixup_log_failed_core_relo()
7260 * line1 -> 123: (85) call unknown#2001000345 in fixup_log_missing_map_load()
7261 * line2 -> invalid func unknown#2001000345 in fixup_log_missing_map_load()
7262 * line3 -> <anything else or end of buffer> in fixup_log_missing_map_load()
7265 * "345" in "2001000345" is a map index in obj->maps to fetch map name. in fixup_log_missing_map_load()
7267 struct bpf_object *obj = prog->obj; in fixup_log_missing_map_load()
7275 map_idx -= POISON_LDIMM64_MAP_BASE; in fixup_log_missing_map_load()
7276 if (map_idx < 0 || map_idx >= obj->nr_maps) in fixup_log_missing_map_load()
7278 map = &obj->maps[map_idx]; in fixup_log_missing_map_load()
7283 insn_idx, map->name); in fixup_log_missing_map_load()
7285 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); in fixup_log_missing_map_load()
7293 * line1 -> 123: (85) call unknown#2002000345 in fixup_log_missing_kfunc_call()
7294 * line2 -> invalid func unknown#2002000345 in fixup_log_missing_kfunc_call()
7295 * line3 -> <anything else or end of buffer> in fixup_log_missing_kfunc_call()
7298 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name. in fixup_log_missing_kfunc_call()
7300 struct bpf_object *obj = prog->obj; in fixup_log_missing_kfunc_call()
7301 const struct extern_desc *ext; in fixup_log_missing_kfunc_call() local
7308 ext_idx -= POISON_CALL_KFUNC_BASE; in fixup_log_missing_kfunc_call()
7309 if (ext_idx < 0 || ext_idx >= obj->nr_extern) in fixup_log_missing_kfunc_call()
7311 ext = &obj->externs[ext_idx]; in fixup_log_missing_kfunc_call()
7316 insn_idx, ext->name); in fixup_log_missing_kfunc_call()
7318 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); in fixup_log_missing_kfunc_call()
7333 next_line = buf + log_sz - 1; in fixup_verifier_log()
7345 /* failed CO-RE relocation case */ in fixup_verifier_log()
7373 struct bpf_object *obj = prog->obj; in bpf_program_record_relos()
7376 for (i = 0; i < prog->nr_reloc; i++) { in bpf_program_record_relos()
7377 struct reloc_desc *relo = &prog->reloc_desc[i]; in bpf_program_record_relos()
7378 struct extern_desc *ext = &obj->externs[relo->ext_idx]; in bpf_program_record_relos() local
7381 switch (relo->type) { in bpf_program_record_relos()
7383 if (ext->type != EXT_KSYM) in bpf_program_record_relos()
7385 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ? in bpf_program_record_relos()
7387 bpf_gen__record_extern(obj->gen_loader, ext->name, in bpf_program_record_relos()
7388 ext->is_weak, !ext->ksym.type_id, in bpf_program_record_relos()
7389 true, kind, relo->insn_idx); in bpf_program_record_relos()
7392 bpf_gen__record_extern(obj->gen_loader, ext->name, in bpf_program_record_relos()
7393 ext->is_weak, false, false, BTF_KIND_FUNC, in bpf_program_record_relos()
7394 relo->insn_idx); in bpf_program_record_relos()
7398 .insn_off = relo->insn_idx * 8, in bpf_program_record_relos()
7399 .type_id = relo->core_relo->type_id, in bpf_program_record_relos()
7400 .access_str_off = relo->core_relo->access_str_off, in bpf_program_record_relos()
7401 .kind = relo->core_relo->kind, in bpf_program_record_relos()
7404 bpf_gen__record_relo_core(obj->gen_loader, &cr); in bpf_program_record_relos()
7421 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__load_progs()
7422 prog = &obj->programs[i]; in bpf_object__load_progs()
7428 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__load_progs()
7429 prog = &obj->programs[i]; in bpf_object__load_progs()
7432 if (!prog->autoload) { in bpf_object__load_progs()
7433 pr_debug("prog '%s': skipped loading\n", prog->name); in bpf_object__load_progs()
7436 prog->log_level |= log_level; in bpf_object__load_progs()
7438 if (obj->gen_loader) in bpf_object__load_progs()
7441 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, in bpf_object__load_progs()
7442 obj->license, obj->kern_version, &prog->fd); in bpf_object__load_progs()
7444 pr_warn("prog '%s': failed to load: %d\n", prog->name, err); in bpf_object__load_progs()
7461 prog->sec_def = find_sec_def(prog->sec_name); in bpf_object_init_progs()
7462 if (!prog->sec_def) { in bpf_object_init_progs()
7465 prog->name, prog->sec_name); in bpf_object_init_progs()
7469 prog->type = prog->sec_def->prog_type; in bpf_object_init_progs()
7470 prog->expected_attach_type = prog->sec_def->expected_attach_type; in bpf_object_init_progs()
7475 if (prog->sec_def->prog_setup_fn) { in bpf_object_init_progs()
7476 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie); in bpf_object_init_progs()
7479 prog->name, err); in bpf_object_init_progs()
7491 const char *obj_name, *kconfig, *btf_tmp_path; in bpf_object_open() local
7502 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); in bpf_object_open()
7506 return ERR_PTR(-EINVAL); in bpf_object_open()
7511 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", in bpf_object_open()
7524 return ERR_PTR(-EINVAL); in bpf_object_open()
7526 return ERR_PTR(-EINVAL); in bpf_object_open()
7532 obj->log_buf = log_buf; in bpf_object_open()
7533 obj->log_size = log_size; in bpf_object_open()
7534 obj->log_level = log_level; in bpf_object_open()
7539 err = -ENAMETOOLONG; in bpf_object_open()
7542 obj->btf_custom_path = strdup(btf_tmp_path); in bpf_object_open()
7543 if (!obj->btf_custom_path) { in bpf_object_open()
7544 err = -ENOMEM; in bpf_object_open()
7549 kconfig = OPTS_GET(opts, kconfig, NULL); in bpf_object_open()
7550 if (kconfig) { in bpf_object_open()
7551 obj->kconfig = strdup(kconfig); in bpf_object_open()
7552 if (!obj->kconfig) { in bpf_object_open()
7553 err = -ENOMEM; in bpf_object_open()
7581 return libbpf_err_ptr(-EINVAL); in bpf_object__open_file()
7598 return libbpf_err_ptr(-EINVAL); in bpf_object__open_mem()
7608 return libbpf_err(-EINVAL); in bpf_object_unload()
7610 for (i = 0; i < obj->nr_maps; i++) { in bpf_object_unload()
7611 zclose(obj->maps[i].fd); in bpf_object_unload()
7612 if (obj->maps[i].st_ops) in bpf_object_unload()
7613 zfree(&obj->maps[i].st_ops->kern_vdata); in bpf_object_unload()
7616 for (i = 0; i < obj->nr_programs; i++) in bpf_object_unload()
7617 bpf_program__unload(&obj->programs[i]); in bpf_object_unload()
7630 m->def.map_flags &= ~BPF_F_MMAPABLE; in bpf_object__sanitize_maps()
7645 err = -errno; in libbpf_kallsyms_parse()
7657 err = -EINVAL; in libbpf_kallsyms_parse()
7675 struct extern_desc *ext; in kallsyms_cb() local
7677 ext = find_extern_by_name(obj, sym_name); in kallsyms_cb()
7678 if (!ext || ext->type != EXT_KSYM) in kallsyms_cb()
7681 t = btf__type_by_id(obj->btf, ext->btf_id); in kallsyms_cb()
7685 if (ext->is_set && ext->ksym.addr != sym_addr) { in kallsyms_cb()
7687 sym_name, ext->ksym.addr, sym_addr); in kallsyms_cb()
7688 return -EINVAL; in kallsyms_cb()
7690 if (!ext->is_set) { in kallsyms_cb()
7691 ext->is_set = true; in kallsyms_cb()
7692 ext->ksym.addr = sym_addr; in kallsyms_cb()
7711 btf = obj->btf_vmlinux; in find_ksym_btf_id()
7715 if (id == -ENOENT) { in find_ksym_btf_id()
7720 for (i = 0; i < obj->btf_module_cnt; i++) { in find_ksym_btf_id()
7722 mod_btf = &obj->btf_modules[i]; in find_ksym_btf_id()
7723 btf = mod_btf->btf; in find_ksym_btf_id()
7725 if (id != -ENOENT) in find_ksym_btf_id()
7730 return -ESRCH; in find_ksym_btf_id()
7738 struct extern_desc *ext) in bpf_object__resolve_ksym_var_btf_id() argument
7747 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf); in bpf_object__resolve_ksym_var_btf_id()
7749 if (id == -ESRCH && ext->is_weak) in bpf_object__resolve_ksym_var_btf_id()
7752 ext->name); in bpf_object__resolve_ksym_var_btf_id()
7757 local_type_id = ext->ksym.type_id; in bpf_object__resolve_ksym_var_btf_id()
7761 targ_var_name = btf__name_by_offset(btf, targ_var->name_off); in bpf_object__resolve_ksym_var_btf_id()
7762 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id); in bpf_object__resolve_ksym_var_btf_id()
7764 err = bpf_core_types_are_compat(obj->btf, local_type_id, in bpf_object__resolve_ksym_var_btf_id()
7770 local_type = btf__type_by_id(obj->btf, local_type_id); in bpf_object__resolve_ksym_var_btf_id()
7771 local_name = btf__name_by_offset(obj->btf, local_type->name_off); in bpf_object__resolve_ksym_var_btf_id()
7772 targ_name = btf__name_by_offset(btf, targ_type->name_off); in bpf_object__resolve_ksym_var_btf_id()
7775 ext->name, local_type_id, in bpf_object__resolve_ksym_var_btf_id()
7778 return -EINVAL; in bpf_object__resolve_ksym_var_btf_id()
7781 ext->is_set = true; in bpf_object__resolve_ksym_var_btf_id()
7782 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; in bpf_object__resolve_ksym_var_btf_id()
7783 ext->ksym.kernel_btf_id = id; in bpf_object__resolve_ksym_var_btf_id()
7785 ext->name, id, btf_kind_str(targ_var), targ_var_name); in bpf_object__resolve_ksym_var_btf_id()
7791 struct extern_desc *ext) in bpf_object__resolve_ksym_func_btf_id() argument
7799 local_func_proto_id = ext->ksym.type_id; in bpf_object__resolve_ksym_func_btf_id()
7801 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf, in bpf_object__resolve_ksym_func_btf_id()
7804 if (kfunc_id == -ESRCH && ext->is_weak) in bpf_object__resolve_ksym_func_btf_id()
7807 ext->name); in bpf_object__resolve_ksym_func_btf_id()
7812 kfunc_proto_id = kern_func->type; in bpf_object__resolve_ksym_func_btf_id()
7814 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id, in bpf_object__resolve_ksym_func_btf_id()
7817 if (ext->is_weak) in bpf_object__resolve_ksym_func_btf_id()
7821 ext->name, local_func_proto_id, in bpf_object__resolve_ksym_func_btf_id()
7822 mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id); in bpf_object__resolve_ksym_func_btf_id()
7823 return -EINVAL; in bpf_object__resolve_ksym_func_btf_id()
7827 if (mod_btf && !mod_btf->fd_array_idx) { in bpf_object__resolve_ksym_func_btf_id()
7828 /* insn->off is s16 */ in bpf_object__resolve_ksym_func_btf_id()
7829 if (obj->fd_array_cnt == INT16_MAX) { in bpf_object__resolve_ksym_func_btf_id()
7831 ext->name, mod_btf->fd_array_idx); in bpf_object__resolve_ksym_func_btf_id()
7832 return -E2BIG; in bpf_object__resolve_ksym_func_btf_id()
7835 if (!obj->fd_array_cnt) in bpf_object__resolve_ksym_func_btf_id()
7836 obj->fd_array_cnt = 1; in bpf_object__resolve_ksym_func_btf_id()
7838 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), in bpf_object__resolve_ksym_func_btf_id()
7839 obj->fd_array_cnt + 1); in bpf_object__resolve_ksym_func_btf_id()
7842 mod_btf->fd_array_idx = obj->fd_array_cnt; in bpf_object__resolve_ksym_func_btf_id()
7844 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd; in bpf_object__resolve_ksym_func_btf_id()
7847 ext->is_set = true; in bpf_object__resolve_ksym_func_btf_id()
7848 ext->ksym.kernel_btf_id = kfunc_id; in bpf_object__resolve_ksym_func_btf_id()
7849 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0; in bpf_object__resolve_ksym_func_btf_id()
7852 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call. in bpf_object__resolve_ksym_func_btf_id()
7853 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64. in bpf_object__resolve_ksym_func_btf_id()
7855 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; in bpf_object__resolve_ksym_func_btf_id()
7857 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id); in bpf_object__resolve_ksym_func_btf_id()
7865 struct extern_desc *ext; in bpf_object__resolve_ksyms_btf_id() local
7868 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__resolve_ksyms_btf_id()
7869 ext = &obj->externs[i]; in bpf_object__resolve_ksyms_btf_id()
7870 if (ext->type != EXT_KSYM || !ext->ksym.type_id) in bpf_object__resolve_ksyms_btf_id()
7873 if (obj->gen_loader) { in bpf_object__resolve_ksyms_btf_id()
7874 ext->is_set = true; in bpf_object__resolve_ksyms_btf_id()
7875 ext->ksym.kernel_btf_obj_fd = 0; in bpf_object__resolve_ksyms_btf_id()
7876 ext->ksym.kernel_btf_id = 0; in bpf_object__resolve_ksyms_btf_id()
7879 t = btf__type_by_id(obj->btf, ext->btf_id); in bpf_object__resolve_ksyms_btf_id()
7881 err = bpf_object__resolve_ksym_var_btf_id(obj, ext); in bpf_object__resolve_ksyms_btf_id()
7883 err = bpf_object__resolve_ksym_func_btf_id(obj, ext); in bpf_object__resolve_ksyms_btf_id()
7895 struct extern_desc *ext; in bpf_object__resolve_externs() local
7899 if (obj->nr_extern == 0) in bpf_object__resolve_externs()
7902 if (obj->kconfig_map_idx >= 0) in bpf_object__resolve_externs()
7903 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; in bpf_object__resolve_externs()
7905 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__resolve_externs()
7906 ext = &obj->externs[i]; in bpf_object__resolve_externs()
7908 if (ext->type == EXT_KSYM) { in bpf_object__resolve_externs()
7909 if (ext->ksym.type_id) in bpf_object__resolve_externs()
7914 } else if (ext->type == EXT_KCFG) { in bpf_object__resolve_externs()
7915 void *ext_ptr = kcfg_data + ext->kcfg.data_off; in bpf_object__resolve_externs()
7918 /* Kconfig externs need actual /proc/config.gz */ in bpf_object__resolve_externs()
7919 if (str_has_pfx(ext->name, "CONFIG_")) { in bpf_object__resolve_externs()
7925 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { in bpf_object__resolve_externs()
7928 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name); in bpf_object__resolve_externs()
7929 return -EINVAL; in bpf_object__resolve_externs()
7931 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) { in bpf_object__resolve_externs()
7933 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) { in bpf_object__resolve_externs()
7935 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) { in bpf_object__resolve_externs()
7938 * customly by libbpf (their values don't come from Kconfig). in bpf_object__resolve_externs()
7943 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name); in bpf_object__resolve_externs()
7944 return -EINVAL; in bpf_object__resolve_externs()
7947 err = set_kcfg_value_num(ext, ext_ptr, value); in bpf_object__resolve_externs()
7951 ext->name, (long long)value); in bpf_object__resolve_externs()
7953 pr_warn("extern '%s': unrecognized extern kind\n", ext->name); in bpf_object__resolve_externs()
7954 return -EINVAL; in bpf_object__resolve_externs()
7960 return -EINVAL; in bpf_object__resolve_externs()
7962 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__resolve_externs()
7963 ext = &obj->externs[i]; in bpf_object__resolve_externs()
7964 if (ext->type == EXT_KCFG && !ext->is_set) { in bpf_object__resolve_externs()
7973 return -EINVAL; in bpf_object__resolve_externs()
7978 return -EINVAL; in bpf_object__resolve_externs()
7983 return -EINVAL; in bpf_object__resolve_externs()
7985 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__resolve_externs()
7986 ext = &obj->externs[i]; in bpf_object__resolve_externs()
7988 if (!ext->is_set && !ext->is_weak) { in bpf_object__resolve_externs()
7989 pr_warn("extern '%s' (strong): not resolved\n", ext->name); in bpf_object__resolve_externs()
7990 return -ESRCH; in bpf_object__resolve_externs()
7991 } else if (!ext->is_set) { in bpf_object__resolve_externs()
7993 ext->name); in bpf_object__resolve_externs()
8005 st_ops = map->st_ops; in bpf_map_prepare_vdata()
8006 for (i = 0; i < btf_vlen(st_ops->type); i++) { in bpf_map_prepare_vdata()
8007 struct bpf_program *prog = st_ops->progs[i]; in bpf_map_prepare_vdata()
8015 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; in bpf_map_prepare_vdata()
8024 for (i = 0; i < obj->nr_maps; i++) in bpf_object_prepare_struct_ops()
8025 if (bpf_map__is_struct_ops(&obj->maps[i])) in bpf_object_prepare_struct_ops()
8026 bpf_map_prepare_vdata(&obj->maps[i]); in bpf_object_prepare_struct_ops()
8036 return libbpf_err(-EINVAL); in bpf_object_load()
8038 if (obj->loaded) { in bpf_object_load()
8039 pr_warn("object '%s': load can't be attempted twice\n", obj->name); in bpf_object_load()
8040 return libbpf_err(-EINVAL); in bpf_object_load()
8043 if (obj->gen_loader) in bpf_object_load()
8044 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); in bpf_object_load()
8048 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); in bpf_object_load()
8053 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path); in bpf_object_load()
8058 if (obj->gen_loader) { in bpf_object_load()
8060 if (obj->btf) in bpf_object_load()
8061 btf__set_fd(obj->btf, -1); in bpf_object_load()
8062 for (i = 0; i < obj->nr_maps; i++) in bpf_object_load()
8063 obj->maps[i].fd = -1; in bpf_object_load()
8065 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps); in bpf_object_load()
8069 zfree(&obj->fd_array); in bpf_object_load()
8072 for (i = 0; i < obj->btf_module_cnt; i++) { in bpf_object_load()
8073 close(obj->btf_modules[i].fd); in bpf_object_load()
8074 btf__free(obj->btf_modules[i].btf); in bpf_object_load()
8075 free(obj->btf_modules[i].name); in bpf_object_load()
8077 free(obj->btf_modules); in bpf_object_load()
8080 btf__free(obj->btf_vmlinux); in bpf_object_load()
8081 obj->btf_vmlinux = NULL; in bpf_object_load()
8083 obj->loaded = true; /* doesn't matter if successfully or not */ in bpf_object_load()
8090 /* unpin any maps that were auto-pinned during load */ in bpf_object_load()
8091 for (i = 0; i < obj->nr_maps; i++) in bpf_object_load()
8092 if (obj->maps[i].pinned && !obj->maps[i].reused) in bpf_object_load()
8093 bpf_map__unpin(&obj->maps[i], NULL); in bpf_object_load()
8096 pr_warn("failed to load object '%s'\n", obj->path); in bpf_object_load()
8113 return -ENOMEM; in make_parent_dir()
8117 err = -errno; in make_parent_dir()
8121 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); in make_parent_dir()
8135 return -EINVAL; in check_path()
8139 return -ENOMEM; in check_path()
8145 err = -errno; in check_path()
8151 err = -EINVAL; in check_path()
8162 if (prog->fd < 0) { in bpf_program__pin()
8163 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name); in bpf_program__pin()
8164 return libbpf_err(-EINVAL); in bpf_program__pin()
8175 if (bpf_obj_pin(prog->fd, path)) { in bpf_program__pin()
8176 err = -errno; in bpf_program__pin()
8178 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp); in bpf_program__pin()
8182 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path); in bpf_program__pin()
8190 if (prog->fd < 0) { in bpf_program__unpin()
8191 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name); in bpf_program__unpin()
8192 return libbpf_err(-EINVAL); in bpf_program__unpin()
8201 return libbpf_err(-errno); in bpf_program__unpin()
8203 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path); in bpf_program__unpin()
8214 return libbpf_err(-EINVAL); in bpf_map__pin()
8217 if (map->pin_path) { in bpf_map__pin()
8218 if (path && strcmp(path, map->pin_path)) { in bpf_map__pin()
8220 bpf_map__name(map), map->pin_path, path); in bpf_map__pin()
8221 return libbpf_err(-EINVAL); in bpf_map__pin()
8222 } else if (map->pinned) { in bpf_map__pin()
8223 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", in bpf_map__pin()
8224 bpf_map__name(map), map->pin_path); in bpf_map__pin()
8231 return libbpf_err(-EINVAL); in bpf_map__pin()
8232 } else if (map->pinned) { in bpf_map__pin()
8234 return libbpf_err(-EEXIST); in bpf_map__pin()
8237 map->pin_path = strdup(path); in bpf_map__pin()
8238 if (!map->pin_path) { in bpf_map__pin()
8239 err = -errno; in bpf_map__pin()
8244 err = make_parent_dir(map->pin_path); in bpf_map__pin()
8248 err = check_path(map->pin_path); in bpf_map__pin()
8252 if (bpf_obj_pin(map->fd, map->pin_path)) { in bpf_map__pin()
8253 err = -errno; in bpf_map__pin()
8257 map->pinned = true; in bpf_map__pin()
8258 pr_debug("pinned map '%s'\n", map->pin_path); in bpf_map__pin()
8263 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); in bpf_map__pin()
8274 return libbpf_err(-EINVAL); in bpf_map__unpin()
8277 if (map->pin_path) { in bpf_map__unpin()
8278 if (path && strcmp(path, map->pin_path)) { in bpf_map__unpin()
8280 bpf_map__name(map), map->pin_path, path); in bpf_map__unpin()
8281 return libbpf_err(-EINVAL); in bpf_map__unpin()
8283 path = map->pin_path; in bpf_map__unpin()
8287 return libbpf_err(-EINVAL); in bpf_map__unpin()
8296 return libbpf_err(-errno); in bpf_map__unpin()
8298 map->pinned = false; in bpf_map__unpin()
8311 return libbpf_err(-errno); in bpf_map__set_pin_path()
8314 free(map->pin_path); in bpf_map__set_pin_path()
8315 map->pin_path = new; in bpf_map__set_pin_path()
8324 return map->pin_path; in bpf_map__pin_path()
8329 return map->pinned; in bpf_map__is_pinned()
8348 return libbpf_err(-ENOENT); in bpf_object__pin_maps()
8350 if (!obj->loaded) { in bpf_object__pin_maps()
8352 return libbpf_err(-ENOENT); in bpf_object__pin_maps()
8359 if (!map->autocreate) in bpf_object__pin_maps()
8368 } else if (!map->pin_path) { in bpf_object__pin_maps()
8381 if (!map->pin_path) in bpf_object__pin_maps()
8396 return libbpf_err(-ENOENT); in bpf_object__unpin_maps()
8408 } else if (!map->pin_path) { in bpf_object__unpin_maps()
8427 return libbpf_err(-ENOENT); in bpf_object__pin_programs()
8429 if (!obj->loaded) { in bpf_object__pin_programs()
8431 return libbpf_err(-ENOENT); in bpf_object__pin_programs()
8435 err = pathname_concat(buf, sizeof(buf), path, prog->name); in bpf_object__pin_programs()
8448 if (pathname_concat(buf, sizeof(buf), path, prog->name)) in bpf_object__pin_programs()
8463 return libbpf_err(-ENOENT); in bpf_object__unpin_programs()
8468 err = pathname_concat(buf, sizeof(buf), path, prog->name); in bpf_object__unpin_programs()
8514 if (map->inner_map) { in bpf_map__destroy()
8515 bpf_map__destroy(map->inner_map); in bpf_map__destroy()
8516 zfree(&map->inner_map); in bpf_map__destroy()
8519 zfree(&map->init_slots); in bpf_map__destroy()
8520 map->init_slots_sz = 0; in bpf_map__destroy()
8522 if (map->mmaped) { in bpf_map__destroy()
8525 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); in bpf_map__destroy()
8526 munmap(map->mmaped, mmap_sz); in bpf_map__destroy()
8527 map->mmaped = NULL; in bpf_map__destroy()
8530 if (map->st_ops) { in bpf_map__destroy()
8531 zfree(&map->st_ops->data); in bpf_map__destroy()
8532 zfree(&map->st_ops->progs); in bpf_map__destroy()
8533 zfree(&map->st_ops->kern_func_off); in bpf_map__destroy()
8534 zfree(&map->st_ops); in bpf_map__destroy()
8537 zfree(&map->name); in bpf_map__destroy()
8538 zfree(&map->real_name); in bpf_map__destroy()
8539 zfree(&map->pin_path); in bpf_map__destroy()
8541 if (map->fd >= 0) in bpf_map__destroy()
8542 zclose(map->fd); in bpf_map__destroy()
8552 usdt_manager_free(obj->usdt_man); in bpf_object__close()
8553 obj->usdt_man = NULL; in bpf_object__close()
8555 bpf_gen__free(obj->gen_loader); in bpf_object__close()
8558 btf__free(obj->btf); in bpf_object__close()
8559 btf__free(obj->btf_vmlinux); in bpf_object__close()
8560 btf_ext__free(obj->btf_ext); in bpf_object__close()
8562 for (i = 0; i < obj->nr_maps; i++) in bpf_object__close()
8563 bpf_map__destroy(&obj->maps[i]); in bpf_object__close()
8565 zfree(&obj->btf_custom_path); in bpf_object__close()
8566 zfree(&obj->kconfig); in bpf_object__close()
8568 for (i = 0; i < obj->nr_extern; i++) in bpf_object__close()
8569 zfree(&obj->externs[i].essent_name); in bpf_object__close()
8571 zfree(&obj->externs); in bpf_object__close()
8572 obj->nr_extern = 0; in bpf_object__close()
8574 zfree(&obj->maps); in bpf_object__close()
8575 obj->nr_maps = 0; in bpf_object__close()
8577 if (obj->programs && obj->nr_programs) { in bpf_object__close()
8578 for (i = 0; i < obj->nr_programs; i++) in bpf_object__close()
8579 bpf_program__exit(&obj->programs[i]); in bpf_object__close()
8581 zfree(&obj->programs); in bpf_object__close()
8588 return obj ? obj->name : libbpf_err_ptr(-EINVAL); in bpf_object__name()
8593 return obj ? obj->kern_version : 0; in bpf_object__kversion()
8598 return obj ? obj->btf : NULL; in bpf_object__btf()
8603 return obj->btf ? btf__fd(obj->btf) : -1; in bpf_object__btf_fd()
8608 if (obj->loaded) in bpf_object__set_kversion()
8609 return libbpf_err(-EINVAL); in bpf_object__set_kversion()
8611 obj->kern_version = kern_version; in bpf_object__set_kversion()
8621 return -EFAULT; in bpf_object__gen_loader()
8623 return -EINVAL; in bpf_object__gen_loader()
8626 return -ENOMEM; in bpf_object__gen_loader()
8627 gen->opts = opts; in bpf_object__gen_loader()
8628 obj->gen_loader = gen; in bpf_object__gen_loader()
8636 size_t nr_programs = obj->nr_programs; in __bpf_program__iter()
8644 return forward ? &obj->programs[0] : in __bpf_program__iter()
8645 &obj->programs[nr_programs - 1]; in __bpf_program__iter()
8647 if (p->obj != obj) { in __bpf_program__iter()
8652 idx = (p - obj->programs) + (forward ? 1 : -1); in __bpf_program__iter()
8653 if (idx >= obj->nr_programs || idx < 0) in __bpf_program__iter()
8655 return &obj->programs[idx]; in __bpf_program__iter()
8684 prog->prog_ifindex = ifindex; in bpf_program__set_ifindex()
8689 return prog->name; in bpf_program__name()
8694 return prog->sec_name; in bpf_program__section_name()
8699 return prog->autoload; in bpf_program__autoload()
8704 if (prog->obj->loaded) in bpf_program__set_autoload()
8705 return libbpf_err(-EINVAL); in bpf_program__set_autoload()
8707 prog->autoload = autoload; in bpf_program__set_autoload()
8713 return prog->autoattach; in bpf_program__autoattach()
8718 prog->autoattach = autoattach; in bpf_program__set_autoattach()
8723 return prog->insns; in bpf_program__insns()
8728 return prog->insns_cnt; in bpf_program__insn_cnt()
8736 if (prog->obj->loaded) in bpf_program__set_insns()
8737 return -EBUSY; in bpf_program__set_insns()
8739 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns)); in bpf_program__set_insns()
8742 pr_warn("prog '%s': failed to realloc prog code\n", prog->name); in bpf_program__set_insns()
8743 return -ENOMEM; in bpf_program__set_insns()
8747 prog->insns = insns; in bpf_program__set_insns()
8748 prog->insns_cnt = new_insn_cnt; in bpf_program__set_insns()
8755 return libbpf_err(-EINVAL); in bpf_program__fd()
8757 if (prog->fd < 0) in bpf_program__fd()
8758 return libbpf_err(-ENOENT); in bpf_program__fd()
8760 return prog->fd; in bpf_program__fd()
8768 return prog->type; in bpf_program__type()
8779 if (prog->obj->loaded) in bpf_program__set_type()
8780 return libbpf_err(-EBUSY); in bpf_program__set_type()
8783 if (prog->type == type) in bpf_program__set_type()
8786 prog->type = type; in bpf_program__set_type()
8790 * fallback handler, which by definition is program type-agnostic and in bpf_program__set_type()
8791 * is a catch-all custom handler, optionally set by the application, in bpf_program__set_type()
8794 if (prog->sec_def != &custom_fallback_def) in bpf_program__set_type()
8795 prog->sec_def = NULL; in bpf_program__set_type()
8804 return prog->expected_attach_type; in bpf_program__expected_attach_type()
8810 if (prog->obj->loaded) in bpf_program__set_expected_attach_type()
8811 return libbpf_err(-EBUSY); in bpf_program__set_expected_attach_type()
8813 prog->expected_attach_type = type; in bpf_program__set_expected_attach_type()
8819 return prog->prog_flags; in bpf_program__flags()
8824 if (prog->obj->loaded) in bpf_program__set_flags()
8825 return libbpf_err(-EBUSY); in bpf_program__set_flags()
8827 prog->prog_flags = flags; in bpf_program__set_flags()
8833 return prog->log_level; in bpf_program__log_level()
8838 if (prog->obj->loaded) in bpf_program__set_log_level()
8839 return libbpf_err(-EBUSY); in bpf_program__set_log_level()
8841 prog->log_level = log_level; in bpf_program__set_log_level()
8847 *log_size = prog->log_size; in bpf_program__log_buf()
8848 return prog->log_buf; in bpf_program__log_buf()
8854 return -EINVAL; in bpf_program__set_log_buf()
8855 if (prog->log_size > UINT_MAX) in bpf_program__set_log_buf()
8856 return -EINVAL; in bpf_program__set_log_buf()
8857 if (prog->obj->loaded) in bpf_program__set_log_buf()
8858 return -EBUSY; in bpf_program__set_log_buf()
8860 prog->log_buf = log_buf; in bpf_program__set_log_buf()
8861 prog->log_size = log_size; in bpf_program__set_log_buf()
8926 SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace),
8989 return libbpf_err(-EINVAL); in libbpf_register_prog_handler()
8992 return libbpf_err(-E2BIG); in libbpf_register_prog_handler()
8998 return libbpf_err(-ENOMEM); in libbpf_register_prog_handler()
9004 return libbpf_err(-EBUSY); in libbpf_register_prog_handler()
9009 sec_def->sec = sec ? strdup(sec) : NULL; in libbpf_register_prog_handler()
9010 if (sec && !sec_def->sec) in libbpf_register_prog_handler()
9011 return libbpf_err(-ENOMEM); in libbpf_register_prog_handler()
9013 sec_def->prog_type = prog_type; in libbpf_register_prog_handler()
9014 sec_def->expected_attach_type = exp_attach_type; in libbpf_register_prog_handler()
9015 sec_def->cookie = OPTS_GET(opts, cookie, 0); in libbpf_register_prog_handler()
9017 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL); in libbpf_register_prog_handler()
9018 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL); in libbpf_register_prog_handler()
9019 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL); in libbpf_register_prog_handler()
9021 sec_def->handler_id = ++last_custom_sec_def_handler_id; in libbpf_register_prog_handler()
9028 return sec_def->handler_id; in libbpf_register_prog_handler()
9037 return libbpf_err(-EINVAL); in libbpf_unregister_prog_handler()
9051 return libbpf_err(-ENOENT); in libbpf_unregister_prog_handler()
9055 custom_sec_defs[i - 1] = custom_sec_defs[i]; in libbpf_unregister_prog_handler()
9056 custom_sec_def_cnt--; in libbpf_unregister_prog_handler()
9072 size_t len = strlen(sec_def->sec); in sec_def_matches()
9075 if (sec_def->sec[len - 1] == '/') { in sec_def_matches()
9076 if (str_has_pfx(sec_name, sec_def->sec)) in sec_def_matches()
9082 * well-formed SEC("type/extras") with proper '/' separator in sec_def_matches()
9084 if (sec_def->sec[len - 1] == '+') { in sec_def_matches()
9085 len--; in sec_def_matches()
9087 if (strncmp(sec_name, sec_def->sec, len) != 0) in sec_def_matches()
9095 return strcmp(sec_name, sec_def->sec) == 0; in sec_def_matches()
9140 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) in libbpf_get_type_names()
9143 if (!(sec_def->cookie & SEC_ATTACHABLE)) in libbpf_get_type_names()
9165 return libbpf_err(-EINVAL); in libbpf_prog_type_by_name()
9169 *prog_type = sec_def->prog_type; in libbpf_prog_type_by_name()
9170 *expected_attach_type = sec_def->expected_attach_type; in libbpf_prog_type_by_name()
9181 return libbpf_err(-ESRCH); in libbpf_prog_type_by_name()
9223 for (i = 0; i < obj->nr_maps; i++) { in find_struct_ops_map_by_offset()
9224 map = &obj->maps[i]; in find_struct_ops_map_by_offset()
9227 if (map->sec_idx == sec_idx && in find_struct_ops_map_by_offset()
9228 map->sec_offset <= offset && in find_struct_ops_map_by_offset()
9229 offset - map->sec_offset < map->def.value_size) in find_struct_ops_map_by_offset()
9236 /* Collect the reloc from ELF and populate the st_ops->progs[] */
9253 btf = obj->btf; in bpf_object__collect_st_ops_relos()
9254 nrels = shdr->sh_size / shdr->sh_entsize; in bpf_object__collect_st_ops_relos()
9259 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_st_ops_relos()
9262 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); in bpf_object__collect_st_ops_relos()
9265 (size_t)ELF64_R_SYM(rel->r_info)); in bpf_object__collect_st_ops_relos()
9266 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_st_ops_relos()
9269 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; in bpf_object__collect_st_ops_relos()
9270 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset); in bpf_object__collect_st_ops_relos()
9272 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n", in bpf_object__collect_st_ops_relos()
9273 (size_t)rel->r_offset); in bpf_object__collect_st_ops_relos()
9274 return -EINVAL; in bpf_object__collect_st_ops_relos()
9277 moff = rel->r_offset - map->sec_offset; in bpf_object__collect_st_ops_relos()
9278 shdr_idx = sym->st_shndx; in bpf_object__collect_st_ops_relos()
9279 st_ops = map->st_ops; in bpf_object__collect_st_ops_relos()
9280 …pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %… in bpf_object__collect_st_ops_relos()
9281 map->name, in bpf_object__collect_st_ops_relos()
9282 (long long)(rel->r_info >> 32), in bpf_object__collect_st_ops_relos()
9283 (long long)sym->st_value, in bpf_object__collect_st_ops_relos()
9284 shdr_idx, (size_t)rel->r_offset, in bpf_object__collect_st_ops_relos()
9285 map->sec_offset, sym->st_name, name); in bpf_object__collect_st_ops_relos()
9288 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n", in bpf_object__collect_st_ops_relos()
9289 map->name, (size_t)rel->r_offset, shdr_idx); in bpf_object__collect_st_ops_relos()
9290 return -LIBBPF_ERRNO__RELOC; in bpf_object__collect_st_ops_relos()
9292 if (sym->st_value % BPF_INSN_SZ) { in bpf_object__collect_st_ops_relos()
9294 map->name, (unsigned long long)sym->st_value); in bpf_object__collect_st_ops_relos()
9295 return -LIBBPF_ERRNO__FORMAT; in bpf_object__collect_st_ops_relos()
9297 insn_idx = sym->st_value / BPF_INSN_SZ; in bpf_object__collect_st_ops_relos()
9299 member = find_member_by_offset(st_ops->type, moff * 8); in bpf_object__collect_st_ops_relos()
9302 map->name, moff); in bpf_object__collect_st_ops_relos()
9303 return -EINVAL; in bpf_object__collect_st_ops_relos()
9305 member_idx = member - btf_members(st_ops->type); in bpf_object__collect_st_ops_relos()
9306 name = btf__name_by_offset(btf, member->name_off); in bpf_object__collect_st_ops_relos()
9308 if (!resolve_func_ptr(btf, member->type, NULL)) { in bpf_object__collect_st_ops_relos()
9310 map->name, name); in bpf_object__collect_st_ops_relos()
9311 return -EINVAL; in bpf_object__collect_st_ops_relos()
9317 map->name, shdr_idx, name); in bpf_object__collect_st_ops_relos()
9318 return -EINVAL; in bpf_object__collect_st_ops_relos()
9322 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) { in bpf_object__collect_st_ops_relos()
9324 map->name, prog->name); in bpf_object__collect_st_ops_relos()
9325 return -EINVAL; in bpf_object__collect_st_ops_relos()
9331 if (!prog->attach_btf_id) { in bpf_object__collect_st_ops_relos()
9332 prog->attach_btf_id = st_ops->type_id; in bpf_object__collect_st_ops_relos()
9333 prog->expected_attach_type = member_idx; in bpf_object__collect_st_ops_relos()
9336 /* struct_ops BPF prog can be re-used between multiple in bpf_object__collect_st_ops_relos()
9341 if (prog->attach_btf_id != st_ops->type_id || in bpf_object__collect_st_ops_relos()
9342 prog->expected_attach_type != member_idx) { in bpf_object__collect_st_ops_relos()
9344 map->name, prog->name, prog->sec_name, prog->type, in bpf_object__collect_st_ops_relos()
9345 prog->attach_btf_id, prog->expected_attach_type, name); in bpf_object__collect_st_ops_relos()
9346 return -EINVAL; in bpf_object__collect_st_ops_relos()
9349 st_ops->progs[member_idx] = prog; in bpf_object__collect_st_ops_relos()
9396 return -ENAMETOOLONG; in find_btf_by_prefix_kind()
9446 err = -EINVAL; in libbpf_find_prog_btf_id()
9473 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type); in find_kernel_btf_id()
9479 if (ret != -ENOENT) in find_kernel_btf_id()
9486 for (i = 0; i < obj->btf_module_cnt; i++) { in find_kernel_btf_id()
9487 const struct module_btf *mod = &obj->btf_modules[i]; in find_kernel_btf_id()
9489 ret = find_attach_btf_id(mod->btf, attach_name, attach_type); in find_kernel_btf_id()
9491 *btf_obj_fd = mod->fd; in find_kernel_btf_id()
9495 if (ret == -ENOENT) in find_kernel_btf_id()
9501 return -ESRCH; in find_kernel_btf_id()
9507 enum bpf_attach_type attach_type = prog->expected_attach_type; in libbpf_find_attach_btf_id()
9508 __u32 attach_prog_fd = prog->attach_prog_fd; in libbpf_find_attach_btf_id()
9512 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) { in libbpf_find_attach_btf_id()
9514 pr_warn("prog '%s': attach program FD is not set\n", prog->name); in libbpf_find_attach_btf_id()
9515 return -EINVAL; in libbpf_find_attach_btf_id()
9520 prog->name, attach_prog_fd, attach_name, err); in libbpf_find_attach_btf_id()
9529 if (prog->obj->gen_loader) { in libbpf_find_attach_btf_id()
9530 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); in libbpf_find_attach_btf_id()
9534 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id); in libbpf_find_attach_btf_id()
9538 prog->name, attach_name, err); in libbpf_find_attach_btf_id()
9551 return libbpf_err(-EINVAL); in libbpf_attach_type_by_name()
9562 return libbpf_err(-EINVAL); in libbpf_attach_type_by_name()
9565 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) in libbpf_attach_type_by_name()
9566 return libbpf_err(-EINVAL); in libbpf_attach_type_by_name()
9567 if (!(sec_def->cookie & SEC_ATTACHABLE)) in libbpf_attach_type_by_name()
9568 return libbpf_err(-EINVAL); in libbpf_attach_type_by_name()
9570 *attach_type = sec_def->expected_attach_type; in libbpf_attach_type_by_name()
9576 return map ? map->fd : libbpf_err(-EINVAL); in bpf_map__fd()
9582 * their user-visible name differs from kernel-visible name. Users see in map_uses_real_name()
9587 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0) in map_uses_real_name()
9589 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0) in map_uses_real_name()
9600 return map->real_name; in bpf_map__name()
9602 return map->name; in bpf_map__name()
9607 return map->def.type; in bpf_map__type()
9612 if (map->fd >= 0) in bpf_map__set_type()
9613 return libbpf_err(-EBUSY); in bpf_map__set_type()
9614 map->def.type = type; in bpf_map__set_type()
9620 return map->def.map_flags; in bpf_map__map_flags()
9625 if (map->fd >= 0) in bpf_map__set_map_flags()
9626 return libbpf_err(-EBUSY); in bpf_map__set_map_flags()
9627 map->def.map_flags = flags; in bpf_map__set_map_flags()
9633 return map->map_extra; in bpf_map__map_extra()
9638 if (map->fd >= 0) in bpf_map__set_map_extra()
9639 return libbpf_err(-EBUSY); in bpf_map__set_map_extra()
9640 map->map_extra = map_extra; in bpf_map__set_map_extra()
9646 return map->numa_node; in bpf_map__numa_node()
9651 if (map->fd >= 0) in bpf_map__set_numa_node()
9652 return libbpf_err(-EBUSY); in bpf_map__set_numa_node()
9653 map->numa_node = numa_node; in bpf_map__set_numa_node()
9659 return map->def.key_size; in bpf_map__key_size()
9664 if (map->fd >= 0) in bpf_map__set_key_size()
9665 return libbpf_err(-EBUSY); in bpf_map__set_key_size()
9666 map->def.key_size = size; in bpf_map__set_key_size()
9672 return map->def.value_size; in bpf_map__value_size()
9686 btf = bpf_object__btf(map->obj); in map_btf_datasec_resize()
9688 return -ENOENT; in map_btf_datasec_resize()
9695 return -EINVAL; in map_btf_datasec_resize()
9703 return -EINVAL; in map_btf_datasec_resize()
9707 var = &btf_var_secinfos(datasec_type)[vlen - 1]; in map_btf_datasec_resize()
9708 var_type = btf_type_by_id(btf, var->type); in map_btf_datasec_resize()
9709 array_type = skip_mods_and_typedefs(btf, var_type->type, NULL); in map_btf_datasec_resize()
9713 return -EINVAL; in map_btf_datasec_resize()
9718 element_sz = btf__resolve_size(btf, array->type); in map_btf_datasec_resize()
9719 if (element_sz <= 0 || (size - var->offset) % element_sz != 0) { in map_btf_datasec_resize()
9722 return -EINVAL; in map_btf_datasec_resize()
9726 nr_elements = (size - var->offset) / element_sz; in map_btf_datasec_resize()
9727 new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements); in map_btf_datasec_resize()
9734 datasec_type = btf_type_by_id(btf, map->btf_value_type_id); in map_btf_datasec_resize()
9735 var = &btf_var_secinfos(datasec_type)[vlen - 1]; in map_btf_datasec_resize()
9736 var_type = btf_type_by_id(btf, var->type); in map_btf_datasec_resize()
9739 datasec_type->size = size; in map_btf_datasec_resize()
9740 var->size = size - var->offset; in map_btf_datasec_resize()
9741 var_type->type = new_array_id; in map_btf_datasec_resize()
9748 if (map->fd >= 0) in bpf_map__set_value_size()
9749 return libbpf_err(-EBUSY); in bpf_map__set_value_size()
9751 if (map->mmaped) { in bpf_map__set_value_size()
9755 mmap_old_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); in bpf_map__set_value_size()
9756 mmap_new_sz = bpf_map_mmap_sz(size, map->def.max_entries); in bpf_map__set_value_size()
9759 pr_warn("map '%s': failed to resize memory-mapped region: %d\n", in bpf_map__set_value_size()
9764 if (err && err != -ENOENT) { in bpf_map__set_value_size()
9767 map->btf_value_type_id = 0; in bpf_map__set_value_size()
9768 map->btf_key_type_id = 0; in bpf_map__set_value_size()
9772 map->def.value_size = size; in bpf_map__set_value_size()
9778 return map ? map->btf_key_type_id : 0; in bpf_map__btf_key_type_id()
9783 return map ? map->btf_value_type_id : 0; in bpf_map__btf_value_type_id()
9789 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG || in bpf_map__set_initial_value()
9790 size != map->def.value_size || map->fd >= 0) in bpf_map__set_initial_value()
9791 return libbpf_err(-EINVAL); in bpf_map__set_initial_value()
9793 memcpy(map->mmaped, data, size); in bpf_map__set_initial_value()
9799 if (!map->mmaped) in bpf_map__initial_value()
9801 *psize = map->def.value_size; in bpf_map__initial_value()
9802 return map->mmaped; in bpf_map__initial_value()
9807 return map->libbpf_type != LIBBPF_MAP_UNSPEC; in bpf_map__is_internal()
9812 return map->map_ifindex; in bpf_map__ifindex()
9817 if (map->fd >= 0) in bpf_map__set_ifindex()
9818 return libbpf_err(-EBUSY); in bpf_map__set_ifindex()
9819 map->map_ifindex = ifindex; in bpf_map__set_ifindex()
9825 if (!bpf_map_type__is_map_in_map(map->def.type)) { in bpf_map__set_inner_map_fd()
9827 return libbpf_err(-EINVAL); in bpf_map__set_inner_map_fd()
9829 if (map->inner_map_fd != -1) { in bpf_map__set_inner_map_fd()
9831 return libbpf_err(-EINVAL); in bpf_map__set_inner_map_fd()
9833 if (map->inner_map) { in bpf_map__set_inner_map_fd()
9834 bpf_map__destroy(map->inner_map); in bpf_map__set_inner_map_fd()
9835 zfree(&map->inner_map); in bpf_map__set_inner_map_fd()
9837 map->inner_map_fd = fd; in bpf_map__set_inner_map_fd()
9847 if (!obj || !obj->maps) in __bpf_map__iter()
9850 s = obj->maps; in __bpf_map__iter()
9851 e = obj->maps + obj->nr_maps; in __bpf_map__iter()
9859 idx = (m - obj->maps) + i; in __bpf_map__iter()
9860 if (idx >= obj->nr_maps || idx < 0) in __bpf_map__iter()
9862 return &obj->maps[idx]; in __bpf_map__iter()
9869 return obj->maps; in bpf_object__next_map()
9878 if (!obj->nr_maps) in bpf_object__prev_map()
9880 return obj->maps + obj->nr_maps - 1; in bpf_object__prev_map()
9883 return __bpf_map__iter(next, obj, -1); in bpf_object__prev_map()
9897 if (pos->real_name && strcmp(pos->real_name, name) == 0) in bpf_object__find_map_by_name()
9903 if (strcmp(pos->real_name, name) == 0) in bpf_object__find_map_by_name()
9907 if (strcmp(pos->name, name) == 0) in bpf_object__find_map_by_name()
9922 if (map->fd <= 0) in validate_map_op()
9923 return -ENOENT; in validate_map_op()
9925 if (map->def.key_size != key_sz) { in validate_map_op()
9927 map->name, key_sz, map->def.key_size); in validate_map_op()
9928 return -EINVAL; in validate_map_op()
9934 switch (map->def.type) { in validate_map_op()
9940 size_t elem_sz = roundup(map->def.value_size, 8); in validate_map_op()
9943 … pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n", in validate_map_op()
9944 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz); in validate_map_op()
9945 return -EINVAL; in validate_map_op()
9950 if (map->def.value_size != value_sz) { in validate_map_op()
9952 map->name, value_sz, map->def.value_size); in validate_map_op()
9953 return -EINVAL; in validate_map_op()
9970 return bpf_map_lookup_elem_flags(map->fd, key, value, flags); in bpf_map__lookup_elem()
9983 return bpf_map_update_elem(map->fd, key, value, flags); in bpf_map__update_elem()
9995 return bpf_map_delete_elem_flags(map->fd, key, flags); in bpf_map__delete_elem()
10008 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags); in bpf_map__lookup_and_delete_elem()
10020 return bpf_map_get_next_key(map->fd, cur_key, next_key); in bpf_map__get_next_key()
10029 errno = -PTR_ERR(ptr); in libbpf_get_error()
10036 return -errno; in libbpf_get_error()
10060 link->disconnected = true; in bpf_link__disconnect()
10070 if (!link->disconnected && link->detach) in bpf_link__destroy()
10071 err = link->detach(link); in bpf_link__destroy()
10072 if (link->pin_path) in bpf_link__destroy()
10073 free(link->pin_path); in bpf_link__destroy()
10074 if (link->dealloc) in bpf_link__destroy()
10075 link->dealloc(link); in bpf_link__destroy()
10084 return link->fd; in bpf_link__fd()
10089 return link->pin_path; in bpf_link__pin_path()
10094 return libbpf_err_errno(close(link->fd)); in bpf_link__detach_fd()
10104 fd = -errno; in bpf_link__open()
10112 return libbpf_err_ptr(-ENOMEM); in bpf_link__open()
10114 link->detach = &bpf_link__detach_fd; in bpf_link__open()
10115 link->fd = fd; in bpf_link__open()
10117 link->pin_path = strdup(path); in bpf_link__open()
10118 if (!link->pin_path) { in bpf_link__open()
10120 return libbpf_err_ptr(-ENOMEM); in bpf_link__open()
10128 return bpf_link_detach(link->fd) ? -errno : 0; in bpf_link__detach()
10135 if (link->pin_path) in bpf_link__pin()
10136 return libbpf_err(-EBUSY); in bpf_link__pin()
10144 link->pin_path = strdup(path); in bpf_link__pin()
10145 if (!link->pin_path) in bpf_link__pin()
10146 return libbpf_err(-ENOMEM); in bpf_link__pin()
10148 if (bpf_obj_pin(link->fd, link->pin_path)) { in bpf_link__pin()
10149 err = -errno; in bpf_link__pin()
10150 zfree(&link->pin_path); in bpf_link__pin()
10154 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); in bpf_link__pin()
10162 if (!link->pin_path) in bpf_link__unpin()
10163 return libbpf_err(-EINVAL); in bpf_link__unpin()
10165 err = unlink(link->pin_path); in bpf_link__unpin()
10167 return -errno; in bpf_link__unpin()
10169 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); in bpf_link__unpin()
10170 zfree(&link->pin_path); in bpf_link__unpin()
10191 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0) in bpf_link_perf_detach()
10192 err = -errno; in bpf_link_perf_detach()
10194 if (perf_link->perf_event_fd != link->fd) in bpf_link_perf_detach()
10195 close(perf_link->perf_event_fd); in bpf_link_perf_detach()
10196 close(link->fd); in bpf_link_perf_detach()
10199 if (perf_link->legacy_probe_name) { in bpf_link_perf_detach()
10200 if (perf_link->legacy_is_kprobe) { in bpf_link_perf_detach()
10201 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name, in bpf_link_perf_detach()
10202 perf_link->legacy_is_retprobe); in bpf_link_perf_detach()
10204 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name, in bpf_link_perf_detach()
10205 perf_link->legacy_is_retprobe); in bpf_link_perf_detach()
10216 free(perf_link->legacy_probe_name); in bpf_link_perf_dealloc()
10225 int prog_fd, link_fd = -1, err; in bpf_program__attach_perf_event_opts()
10229 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_perf_event_opts()
10233 prog->name, pfd); in bpf_program__attach_perf_event_opts()
10234 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_perf_event_opts()
10239 prog->name); in bpf_program__attach_perf_event_opts()
10240 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_perf_event_opts()
10245 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_perf_event_opts()
10246 link->link.detach = &bpf_link_perf_detach; in bpf_program__attach_perf_event_opts()
10247 link->link.dealloc = &bpf_link_perf_dealloc; in bpf_program__attach_perf_event_opts()
10248 link->perf_event_fd = pfd; in bpf_program__attach_perf_event_opts()
10251 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) { in bpf_program__attach_perf_event_opts()
10257 err = -errno; in bpf_program__attach_perf_event_opts()
10259 prog->name, pfd, in bpf_program__attach_perf_event_opts()
10263 link->link.fd = link_fd; in bpf_program__attach_perf_event_opts()
10266 pr_warn("prog '%s': user context value is not supported\n", prog->name); in bpf_program__attach_perf_event_opts()
10267 err = -EOPNOTSUPP; in bpf_program__attach_perf_event_opts()
10272 err = -errno; in bpf_program__attach_perf_event_opts()
10274 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); in bpf_program__attach_perf_event_opts()
10275 if (err == -EPROTO) in bpf_program__attach_perf_event_opts()
10277 prog->name, pfd); in bpf_program__attach_perf_event_opts()
10280 link->link.fd = pfd; in bpf_program__attach_perf_event_opts()
10283 err = -errno; in bpf_program__attach_perf_event_opts()
10285 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); in bpf_program__attach_perf_event_opts()
10289 return &link->link; in bpf_program__attach_perf_event_opts()
10303 * this function is expected to parse integer in the range of [0, 2^31-1] from
10315 err = -errno; in parse_uint_from_file()
10322 err = err == EOF ? -EIO : -errno; in parse_uint_from_file()
10372 return -EINVAL; in perf_event_open_probe()
10404 pid < 0 ? -1 : pid /* pid */, in perf_event_open_probe()
10405 pid == -1 ? 0 : -1 /* cpu */, in perf_event_open_probe()
10406 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); in perf_event_open_probe()
10407 return pfd >= 0 ? pfd : -errno; in perf_event_open_probe()
10421 return -EINVAL; in append_to_file()
10425 return -errno; in append_to_file()
10428 err = -errno; in append_to_file()
10439 static int has_debugfs = -1; in use_debugfs()
10501 return append_to_file(tracefs_kprobe_events(), "-:%s/%s", in remove_kprobe_event_legacy()
10545 pid < 0 ? -1 : pid, /* pid */ in perf_event_kprobe_open_legacy()
10546 pid == -1 ? 0 : -1, /* cpu */ in perf_event_kprobe_open_legacy()
10547 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); in perf_event_kprobe_open_legacy()
10549 err = -errno; in perf_event_kprobe_open_legacy()
10635 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_opts()
10650 return libbpf_err_ptr(-ENOTSUP); in bpf_program__attach_kprobe_opts()
10654 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) in bpf_program__attach_kprobe_opts()
10655 return libbpf_err_ptr(-ENOTSUP); in bpf_program__attach_kprobe_opts()
10660 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_opts()
10666 -1 /* pid */, 0 /* ref_ctr_off */); in bpf_program__attach_kprobe_opts()
10675 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_kprobe_opts()
10678 offset, -1 /* pid */); in bpf_program__attach_kprobe_opts()
10681 err = -errno; in bpf_program__attach_kprobe_opts()
10683 prog->name, retprobe ? "kretprobe" : "kprobe", in bpf_program__attach_kprobe_opts()
10693 prog->name, retprobe ? "kretprobe" : "kprobe", in bpf_program__attach_kprobe_opts()
10701 perf_link->legacy_probe_name = legacy_probe; in bpf_program__attach_kprobe_opts()
10702 perf_link->legacy_is_kprobe = true; in bpf_program__attach_kprobe_opts()
10703 perf_link->legacy_is_retprobe = retprobe; in bpf_program__attach_kprobe_opts()
10735 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_ksyscall()
10737 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) { in bpf_program__attach_ksyscall()
10804 struct kprobe_multi_resolve *res = data->res; in avail_kallsyms_cb()
10807 if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) in avail_kallsyms_cb()
10810 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1); in avail_kallsyms_cb()
10814 res->addrs[res->cnt++] = (unsigned long)sym_addr; in avail_kallsyms_cb()
10830 err = -errno; in libbpf_available_kallsyms_parse()
10844 err = -EINVAL; in libbpf_available_kallsyms_parse()
10848 if (!glob_match(sym_name, res->pattern)) in libbpf_available_kallsyms_parse()
10857 err = -errno; in libbpf_available_kallsyms_parse()
10866 err = -ENOENT; in libbpf_available_kallsyms_parse()
10878 if (res->cnt == 0) in libbpf_available_kallsyms_parse()
10879 err = -ENOENT; in libbpf_available_kallsyms_parse()
10892 return access(tracefs_available_filter_functions_addrs(), R_OK) != -1; in has_available_filter_functions_addrs()
10905 err = -errno; in libbpf_available_kprobes_parse()
10918 err = -EINVAL; in libbpf_available_kprobes_parse()
10922 if (!glob_match(sym_name, res->pattern)) in libbpf_available_kprobes_parse()
10925 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, in libbpf_available_kprobes_parse()
10926 sizeof(*res->addrs), res->cnt + 1); in libbpf_available_kprobes_parse()
10930 res->addrs[res->cnt++] = (unsigned long)sym_addr; in libbpf_available_kprobes_parse()
10933 if (res->cnt == 0) in libbpf_available_kprobes_parse()
10934 err = -ENOENT; in libbpf_available_kprobes_parse()
10960 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
10968 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
10970 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
10972 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
10974 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_kprobe_multi_opts()
10997 err = -ENOMEM; in bpf_program__attach_kprobe_multi_opts()
11000 link->detach = &bpf_link__detach_fd; in bpf_program__attach_kprobe_multi_opts()
11005 err = -errno; in bpf_program__attach_kprobe_multi_opts()
11007 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); in bpf_program__attach_kprobe_multi_opts()
11010 link->fd = link_fd; in bpf_program__attach_kprobe_multi_opts()
11030 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */ in attach_kprobe()
11031 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0) in attach_kprobe()
11034 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/"); in attach_kprobe()
11036 func_name = prog->sec_name + sizeof("kretprobe/") - 1; in attach_kprobe()
11038 func_name = prog->sec_name + sizeof("kprobe/") - 1; in attach_kprobe()
11040 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset); in attach_kprobe()
11043 return -EINVAL; in attach_kprobe()
11048 return -EINVAL; in attach_kprobe()
11064 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */ in attach_ksyscall()
11065 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0) in attach_ksyscall()
11068 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/"); in attach_ksyscall()
11070 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1; in attach_ksyscall()
11072 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1; in attach_ksyscall()
11075 return *link ? 0 : -errno; in attach_ksyscall()
11087 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */ in attach_kprobe_multi()
11088 if (strcmp(prog->sec_name, "kprobe.multi") == 0 || in attach_kprobe_multi()
11089 strcmp(prog->sec_name, "kretprobe.multi") == 0) in attach_kprobe_multi()
11092 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/"); in attach_kprobe_multi()
11094 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1; in attach_kprobe_multi()
11096 spec = prog->sec_name + sizeof("kprobe.multi/") - 1; in attach_kprobe_multi()
11098 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern); in attach_kprobe_multi()
11101 return -EINVAL; in attach_kprobe_multi()
11113 int n, ret = -EINVAL; in attach_uprobe_multi()
11117 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", in attach_uprobe_multi()
11121 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ in attach_uprobe_multi()
11126 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts); in attach_uprobe_multi()
11130 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, in attach_uprobe_multi()
11131 prog->sec_name); in attach_uprobe_multi()
11165 return append_to_file(tracefs_uprobe_events(), "-:%s/%s", in remove_uprobe_event_legacy()
11206 pid < 0 ? -1 : pid, /* pid */ in perf_event_uprobe_open_legacy()
11207 pid == -1 ? 0 : -1, /* cpu */ in perf_event_uprobe_open_legacy()
11208 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); in perf_event_uprobe_open_legacy()
11210 err = -errno; in perf_event_uprobe_open_legacy()
11258 ret = -LIBBPF_ERRNO__FORMAT; in elf_find_func_offset_from_archive()
11265 elf_errmsg(-1)); in elf_find_func_offset_from_archive()
11266 ret = -LIBBPF_ERRNO__LIBELF; in elf_find_func_offset_from_archive()
11293 return "/lib/x86_64-linux-gnu"; in arch_specific_lib_paths()
11295 return "/lib/i386-linux-gnu"; in arch_specific_lib_paths()
11297 return "/lib/s390x-linux-gnu"; in arch_specific_lib_paths()
11299 return "/lib/s390-linux-gnu"; in arch_specific_lib_paths()
11301 return "/lib/arm-linux-gnueabi"; in arch_specific_lib_paths()
11303 return "/lib/arm-linux-gnueabihf"; in arch_specific_lib_paths()
11305 return "/lib/aarch64-linux-gnu"; in arch_specific_lib_paths()
11307 return "/lib/mips64el-linux-gnuabi64"; in arch_specific_lib_paths()
11309 return "/lib/mipsel-linux-gnu"; in arch_specific_lib_paths()
11311 return "/lib/powerpc64le-linux-gnu"; in arch_specific_lib_paths()
11313 return "/lib/sparc64-linux-gnu"; in arch_specific_lib_paths()
11315 return "/lib/riscv64-linux-gnu"; in arch_specific_lib_paths()
11350 seg_len = next_path ? next_path - s : strlen(s); in resolve_full_path()
11361 return -ENOENT; in resolve_full_path()
11383 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_multi()
11399 * - syms and offsets are mutually exclusive in bpf_program__attach_uprobe_multi()
11400 * - ref_ctr_offsets and cookies are optional in bpf_program__attach_uprobe_multi()
11406 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_multi()
11408 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_multi()
11412 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_multi()
11415 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_multi()
11423 prog->name, path, err); in bpf_program__attach_uprobe_multi()
11455 err = -ENOMEM; in bpf_program__attach_uprobe_multi()
11458 link->detach = &bpf_link__detach_fd; in bpf_program__attach_uprobe_multi()
11463 err = -errno; in bpf_program__attach_uprobe_multi()
11464 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n", in bpf_program__attach_uprobe_multi()
11465 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); in bpf_program__attach_uprobe_multi()
11468 link->fd = link_fd; in bpf_program__attach_uprobe_multi()
11495 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_opts()
11503 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_opts()
11510 min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1))); in bpf_program__attach_uprobe_opts()
11517 prog->name, binary_path, err); in bpf_program__attach_uprobe_opts()
11546 return libbpf_err_ptr(-ENOTSUP); in bpf_program__attach_uprobe_opts()
11550 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) in bpf_program__attach_uprobe_opts()
11551 return libbpf_err_ptr(-ENOTSUP); in bpf_program__attach_uprobe_opts()
11556 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_opts()
11566 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_uprobe_opts()
11573 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_uprobe_opts()
11579 err = -errno; in bpf_program__attach_uprobe_opts()
11581 prog->name, retprobe ? "uretprobe" : "uprobe", in bpf_program__attach_uprobe_opts()
11592 prog->name, retprobe ? "uretprobe" : "uprobe", in bpf_program__attach_uprobe_opts()
11600 perf_link->legacy_probe_name = legacy_probe; in bpf_program__attach_uprobe_opts()
11601 perf_link->legacy_is_kprobe = false; in bpf_program__attach_uprobe_opts()
11602 perf_link->legacy_is_retprobe = retprobe; in bpf_program__attach_uprobe_opts()
11614 /* Format of u[ret]probe section definition supporting auto-attach:
11621 * specified (and auto-attach is not possible) or the above format is specified for
11622 * auto-attach.
11628 int n, c, ret = -EINVAL; in attach_uprobe()
11633 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", in attach_uprobe()
11637 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ in attach_uprobe()
11642 prog->name, prog->sec_name); in attach_uprobe()
11660 prog->name); in attach_uprobe()
11664 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts); in attach_uprobe()
11668 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, in attach_uprobe()
11669 prog->sec_name); in attach_uprobe()
11695 struct bpf_object *obj = prog->obj; in bpf_program__attach_usdt()
11701 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_usdt()
11705 prog->name); in bpf_program__attach_usdt()
11706 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_usdt()
11710 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_usdt()
11716 prog->name, binary_path, err); in bpf_program__attach_usdt()
11725 if (IS_ERR(obj->usdt_man)) in bpf_program__attach_usdt()
11726 return libbpf_ptr(obj->usdt_man); in bpf_program__attach_usdt()
11727 if (!obj->usdt_man) { in bpf_program__attach_usdt()
11728 obj->usdt_man = usdt_manager_new(obj); in bpf_program__attach_usdt()
11729 if (IS_ERR(obj->usdt_man)) in bpf_program__attach_usdt()
11730 return libbpf_ptr(obj->usdt_man); in bpf_program__attach_usdt()
11734 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path, in bpf_program__attach_usdt()
11750 /* no auto-attach for just SEC("usdt") */ in attach_usdt()
11759 err = -EINVAL; in attach_usdt()
11761 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path, in attach_usdt()
11780 return -errno; in determine_tracepoint_id()
11784 return -E2BIG; in determine_tracepoint_id()
11810 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, in perf_event_open_tracepoint()
11811 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); in perf_event_open_tracepoint()
11813 err = -errno; in perf_event_open_tracepoint()
11833 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_tracepoint_opts()
11840 prog->name, tp_category, tp_name, in bpf_program__attach_tracepoint_opts()
11849 prog->name, tp_category, tp_name, in bpf_program__attach_tracepoint_opts()
11869 /* no auto-attach for SEC("tp") or SEC("tracepoint") */ in attach_tp()
11870 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0) in attach_tp()
11873 sec_name = strdup(prog->sec_name); in attach_tp()
11875 return -ENOMEM; in attach_tp()
11878 if (str_has_pfx(prog->sec_name, "tp/")) in attach_tp()
11879 tp_cat = sec_name + sizeof("tp/") - 1; in attach_tp()
11881 tp_cat = sec_name + sizeof("tracepoint/") - 1; in attach_tp()
11885 return -EINVAL; in attach_tp()
11904 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_raw_tracepoint()
11905 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_raw_tracepoint()
11910 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_raw_tracepoint()
11911 link->detach = &bpf_link__detach_fd; in bpf_program__attach_raw_tracepoint()
11915 pfd = -errno; in bpf_program__attach_raw_tracepoint()
11918 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); in bpf_program__attach_raw_tracepoint()
11921 link->fd = pfd; in bpf_program__attach_raw_tracepoint()
11941 if (!str_has_pfx(prog->sec_name, prefixes[i])) in attach_raw_tp()
11945 /* no auto-attach case of, e.g., SEC("raw_tp") */ in attach_raw_tp()
11946 if (prog->sec_name[pfx_len] == '\0') in attach_raw_tp()
11949 if (prog->sec_name[pfx_len] != '/') in attach_raw_tp()
11952 tp_name = prog->sec_name + pfx_len + 1; in attach_raw_tp()
11958 prog->name, prog->sec_name); in attach_raw_tp()
11959 return -EINVAL; in attach_raw_tp()
11976 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_btf_id()
11980 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_btf_id()
11981 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_btf_id()
11986 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_btf_id()
11987 link->detach = &bpf_link__detach_fd; in bpf_program__attach_btf_id()
11993 pfd = -errno; in bpf_program__attach_btf_id()
11996 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); in bpf_program__attach_btf_id()
11999 link->fd = pfd; in bpf_program__attach_btf_id()
12043 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program_attach_fd()
12044 return libbpf_err_ptr(-EINVAL); in bpf_program_attach_fd()
12049 return libbpf_err_ptr(-ENOMEM); in bpf_program_attach_fd()
12050 link->detach = &bpf_link__detach_fd; in bpf_program_attach_fd()
12055 link_fd = -errno; in bpf_program_attach_fd()
12058 prog->name, target_name, in bpf_program_attach_fd()
12062 link->fd = link_fd; in bpf_program_attach_fd()
12093 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_tcx()
12098 /* validate we don't have unexpected combinations of non-zero fields */ in bpf_program__attach_tcx()
12101 prog->name); in bpf_program__attach_tcx()
12102 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_tcx()
12106 prog->name); in bpf_program__attach_tcx()
12107 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_tcx()
12127 prog->name); in bpf_program__attach_freplace()
12128 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_freplace()
12131 if (prog->type != BPF_PROG_TYPE_EXT) { in bpf_program__attach_freplace()
12133 prog->name); in bpf_program__attach_freplace()
12134 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_freplace()
12167 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_iter()
12174 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_iter()
12175 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_iter()
12180 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_iter()
12181 link->detach = &bpf_link__detach_fd; in bpf_program__attach_iter()
12186 link_fd = -errno; in bpf_program__attach_iter()
12189 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); in bpf_program__attach_iter()
12192 link->fd = link_fd; in bpf_program__attach_iter()
12210 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_netfilter()
12214 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_netfilter()
12215 return libbpf_err_ptr(-EINVAL); in bpf_program__attach_netfilter()
12220 return libbpf_err_ptr(-ENOMEM); in bpf_program__attach_netfilter()
12222 link->detach = &bpf_link__detach_fd; in bpf_program__attach_netfilter()
12233 link_fd = -errno; in bpf_program__attach_netfilter()
12236 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); in bpf_program__attach_netfilter()
12239 link->fd = link_fd; in bpf_program__attach_netfilter()
12249 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) in bpf_program__attach()
12250 return libbpf_err_ptr(-EOPNOTSUPP); in bpf_program__attach()
12252 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link); in bpf_program__attach()
12256 /* When calling bpf_program__attach() explicitly, auto-attach support in bpf_program__attach()
12262 return libbpf_err_ptr(-EOPNOTSUPP); in bpf_program__attach()
12279 if (st_link->map_fd < 0) in bpf_link__detach_struct_ops()
12281 return bpf_map_delete_elem(link->fd, &zero); in bpf_link__detach_struct_ops()
12283 return close(link->fd); in bpf_link__detach_struct_ops()
12292 if (!bpf_map__is_struct_ops(map) || map->fd == -1) in bpf_map__attach_struct_ops()
12293 return libbpf_err_ptr(-EINVAL); in bpf_map__attach_struct_ops()
12297 return libbpf_err_ptr(-EINVAL); in bpf_map__attach_struct_ops()
12300 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); in bpf_map__attach_struct_ops()
12306 if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) { in bpf_map__attach_struct_ops()
12311 link->link.detach = bpf_link__detach_struct_ops; in bpf_map__attach_struct_ops()
12313 if (!(map->def.map_flags & BPF_F_LINK)) { in bpf_map__attach_struct_ops()
12315 link->link.fd = map->fd; in bpf_map__attach_struct_ops()
12316 link->map_fd = -1; in bpf_map__attach_struct_ops()
12317 return &link->link; in bpf_map__attach_struct_ops()
12320 fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL); in bpf_map__attach_struct_ops()
12326 link->link.fd = fd; in bpf_map__attach_struct_ops()
12327 link->map_fd = map->fd; in bpf_map__attach_struct_ops()
12329 return &link->link; in bpf_map__attach_struct_ops()
12341 if (!bpf_map__is_struct_ops(map) || map->fd < 0) in bpf_link__update_map()
12342 return -EINVAL; in bpf_link__update_map()
12346 if (st_ops_link->map_fd < 0) in bpf_link__update_map()
12347 return -EINVAL; in bpf_link__update_map()
12349 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); in bpf_link__update_map()
12355 if (err && err != -EBUSY) in bpf_link__update_map()
12358 err = bpf_link_update(link->fd, map->fd, NULL); in bpf_link__update_map()
12362 st_ops_link->map_fd = map->fd; in bpf_link__update_map()
12377 __u64 data_tail = header->data_tail; in perf_event_read_simple()
12384 ehdr = base + (data_tail & (mmap_size - 1)); in perf_event_read_simple()
12385 ehdr_size = ehdr->size; in perf_event_read_simple()
12389 size_t len_first = base + mmap_size - copy_start; in perf_event_read_simple()
12390 size_t len_secnd = ehdr_size - len_first; in perf_event_read_simple()
12424 /* sample_cb and lost_cb are higher-level common-case callbacks */
12463 if (cpu_buf->base && in perf_buffer__free_cpu_buf()
12464 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) in perf_buffer__free_cpu_buf()
12465 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); in perf_buffer__free_cpu_buf()
12466 if (cpu_buf->fd >= 0) { in perf_buffer__free_cpu_buf()
12467 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); in perf_buffer__free_cpu_buf()
12468 close(cpu_buf->fd); in perf_buffer__free_cpu_buf()
12470 free(cpu_buf->buf); in perf_buffer__free_cpu_buf()
12480 if (pb->cpu_bufs) { in perf_buffer__free()
12481 for (i = 0; i < pb->cpu_cnt; i++) { in perf_buffer__free()
12482 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; in perf_buffer__free()
12487 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); in perf_buffer__free()
12490 free(pb->cpu_bufs); in perf_buffer__free()
12492 if (pb->epoll_fd >= 0) in perf_buffer__free()
12493 close(pb->epoll_fd); in perf_buffer__free()
12494 free(pb->events); in perf_buffer__free()
12508 return ERR_PTR(-ENOMEM); in perf_buffer__open_cpu_buf()
12510 cpu_buf->pb = pb; in perf_buffer__open_cpu_buf()
12511 cpu_buf->cpu = cpu; in perf_buffer__open_cpu_buf()
12512 cpu_buf->map_key = map_key; in perf_buffer__open_cpu_buf()
12514 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, in perf_buffer__open_cpu_buf()
12515 -1, PERF_FLAG_FD_CLOEXEC); in perf_buffer__open_cpu_buf()
12516 if (cpu_buf->fd < 0) { in perf_buffer__open_cpu_buf()
12517 err = -errno; in perf_buffer__open_cpu_buf()
12523 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, in perf_buffer__open_cpu_buf()
12525 cpu_buf->fd, 0); in perf_buffer__open_cpu_buf()
12526 if (cpu_buf->base == MAP_FAILED) { in perf_buffer__open_cpu_buf()
12527 cpu_buf->base = NULL; in perf_buffer__open_cpu_buf()
12528 err = -errno; in perf_buffer__open_cpu_buf()
12534 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { in perf_buffer__open_cpu_buf()
12535 err = -errno; in perf_buffer__open_cpu_buf()
12563 return libbpf_err_ptr(-EINVAL); in perf_buffer__new()
12593 return libbpf_err_ptr(-EINVAL); in perf_buffer__new_raw()
12596 return libbpf_err_ptr(-EINVAL); in perf_buffer__new_raw()
12619 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) { in __perf_buffer__new()
12622 return ERR_PTR(-EINVAL); in __perf_buffer__new()
12625 /* best-effort sanity checks */ in __perf_buffer__new()
12630 err = -errno; in __perf_buffer__new()
12632 * -EBADFD, -EFAULT, or -E2BIG on real error in __perf_buffer__new()
12634 if (err != -EINVAL) { in __perf_buffer__new()
12645 return ERR_PTR(-EINVAL); in __perf_buffer__new()
12651 return ERR_PTR(-ENOMEM); in __perf_buffer__new()
12653 pb->event_cb = p->event_cb; in __perf_buffer__new()
12654 pb->sample_cb = p->sample_cb; in __perf_buffer__new()
12655 pb->lost_cb = p->lost_cb; in __perf_buffer__new()
12656 pb->ctx = p->ctx; in __perf_buffer__new()
12658 pb->page_size = getpagesize(); in __perf_buffer__new()
12659 pb->mmap_size = pb->page_size * page_cnt; in __perf_buffer__new()
12660 pb->map_fd = map_fd; in __perf_buffer__new()
12662 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); in __perf_buffer__new()
12663 if (pb->epoll_fd < 0) { in __perf_buffer__new()
12664 err = -errno; in __perf_buffer__new()
12670 if (p->cpu_cnt > 0) { in __perf_buffer__new()
12671 pb->cpu_cnt = p->cpu_cnt; in __perf_buffer__new()
12673 pb->cpu_cnt = libbpf_num_possible_cpus(); in __perf_buffer__new()
12674 if (pb->cpu_cnt < 0) { in __perf_buffer__new()
12675 err = pb->cpu_cnt; in __perf_buffer__new()
12678 if (map.max_entries && map.max_entries < pb->cpu_cnt) in __perf_buffer__new()
12679 pb->cpu_cnt = map.max_entries; in __perf_buffer__new()
12682 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); in __perf_buffer__new()
12683 if (!pb->events) { in __perf_buffer__new()
12684 err = -ENOMEM; in __perf_buffer__new()
12688 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); in __perf_buffer__new()
12689 if (!pb->cpu_bufs) { in __perf_buffer__new()
12690 err = -ENOMEM; in __perf_buffer__new()
12701 for (i = 0, j = 0; i < pb->cpu_cnt; i++) { in __perf_buffer__new()
12705 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; in __perf_buffer__new()
12706 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; in __perf_buffer__new()
12711 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) in __perf_buffer__new()
12714 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); in __perf_buffer__new()
12720 pb->cpu_bufs[j] = cpu_buf; in __perf_buffer__new()
12722 err = bpf_map_update_elem(pb->map_fd, &map_key, in __perf_buffer__new()
12723 &cpu_buf->fd, 0); in __perf_buffer__new()
12725 err = -errno; in __perf_buffer__new()
12726 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", in __perf_buffer__new()
12727 cpu, map_key, cpu_buf->fd, in __perf_buffer__new()
12732 pb->events[j].events = EPOLLIN; in __perf_buffer__new()
12733 pb->events[j].data.ptr = cpu_buf; in __perf_buffer__new()
12734 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, in __perf_buffer__new()
12735 &pb->events[j]) < 0) { in __perf_buffer__new()
12736 err = -errno; in __perf_buffer__new()
12738 cpu, cpu_buf->fd, in __perf_buffer__new()
12744 pb->cpu_cnt = j; in __perf_buffer__new()
12773 struct perf_buffer *pb = cpu_buf->pb; in perf_buffer__process_record()
12777 if (pb->event_cb) in perf_buffer__process_record()
12778 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); in perf_buffer__process_record()
12780 switch (e->type) { in perf_buffer__process_record()
12784 if (pb->sample_cb) in perf_buffer__process_record()
12785 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); in perf_buffer__process_record()
12791 if (pb->lost_cb) in perf_buffer__process_record()
12792 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); in perf_buffer__process_record()
12796 pr_warn("unknown perf sample type %d\n", e->type); in perf_buffer__process_record()
12807 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size, in perf_buffer__process_records()
12808 pb->page_size, &cpu_buf->buf, in perf_buffer__process_records()
12809 &cpu_buf->buf_size, in perf_buffer__process_records()
12818 return pb->epoll_fd; in perf_buffer__epoll_fd()
12825 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); in perf_buffer__poll()
12827 return -errno; in perf_buffer__poll()
12830 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; in perf_buffer__poll()
12846 return pb->cpu_cnt; in perf_buffer__buffer_cnt()
12858 if (buf_idx >= pb->cpu_cnt) in perf_buffer__buffer_fd()
12859 return libbpf_err(-EINVAL); in perf_buffer__buffer_fd()
12861 cpu_buf = pb->cpu_bufs[buf_idx]; in perf_buffer__buffer_fd()
12863 return libbpf_err(-ENOENT); in perf_buffer__buffer_fd()
12865 return cpu_buf->fd; in perf_buffer__buffer_fd()
12872 if (buf_idx >= pb->cpu_cnt) in perf_buffer__buffer()
12873 return libbpf_err(-EINVAL); in perf_buffer__buffer()
12875 cpu_buf = pb->cpu_bufs[buf_idx]; in perf_buffer__buffer()
12877 return libbpf_err(-ENOENT); in perf_buffer__buffer()
12879 *buf = cpu_buf->base; in perf_buffer__buffer()
12880 *buf_size = pb->mmap_size; in perf_buffer__buffer()
12889 * - 0 on success;
12890 * - <0 on failure.
12896 if (buf_idx >= pb->cpu_cnt) in perf_buffer__consume_buffer()
12897 return libbpf_err(-EINVAL); in perf_buffer__consume_buffer()
12899 cpu_buf = pb->cpu_bufs[buf_idx]; in perf_buffer__consume_buffer()
12901 return libbpf_err(-ENOENT); in perf_buffer__consume_buffer()
12910 for (i = 0; i < pb->cpu_cnt; i++) { in perf_buffer__consume()
12911 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; in perf_buffer__consume()
12932 return libbpf_err(-EINVAL); in bpf_program__set_attach_target()
12934 if (prog->obj->loaded) in bpf_program__set_attach_target()
12935 return libbpf_err(-EINVAL); in bpf_program__set_attach_target()
12941 prog->attach_prog_fd = attach_prog_fd; in bpf_program__set_attach_target()
12952 return libbpf_err(-EINVAL); in bpf_program__set_attach_target()
12955 err = bpf_object__load_vmlinux_btf(prog->obj, true); in bpf_program__set_attach_target()
12958 err = find_kernel_btf_id(prog->obj, attach_func_name, in bpf_program__set_attach_target()
12959 prog->expected_attach_type, in bpf_program__set_attach_target()
12965 prog->attach_btf_id = btf_id; in bpf_program__set_attach_target()
12966 prog->attach_btf_obj_fd = btf_obj_fd; in bpf_program__set_attach_target()
12967 prog->attach_prog_fd = attach_prog_fd; in bpf_program__set_attach_target()
12973 int err = 0, n, len, start, end = -1; in parse_cpu_mask_str()
12979 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ in parse_cpu_mask_str()
12985 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); in parse_cpu_mask_str()
12988 err = -EINVAL; in parse_cpu_mask_str()
12996 err = -EINVAL; in parse_cpu_mask_str()
13001 err = -ENOMEM; in parse_cpu_mask_str()
13005 memset(tmp + *mask_sz, 0, start - *mask_sz); in parse_cpu_mask_str()
13006 memset(tmp + start, 1, end - start + 1); in parse_cpu_mask_str()
13012 return -EINVAL; in parse_cpu_mask_str()
13028 err = -errno; in parse_cpu_mask_file()
13035 err = len ? -errno : -EINVAL; in parse_cpu_mask_file()
13041 return -E2BIG; in parse_cpu_mask_file()
13088 return -ESRCH; in populate_skeleton_maps()
13091 /* externs shouldn't be pre-setup from user code */ in populate_skeleton_maps()
13092 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) in populate_skeleton_maps()
13093 *mmaped = (*map)->mmaped; in populate_skeleton_maps()
13111 return -ESRCH; in populate_skeleton_progs()
13121 .object_name = s->name, in bpf_object__open_skeleton()
13126 /* Attempt to preserve opts->object_name, unless overriden by user in bpf_object__open_skeleton()
13134 if (!opts->object_name) in bpf_object__open_skeleton()
13135 skel_opts.object_name = s->name; in bpf_object__open_skeleton()
13138 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts); in bpf_object__open_skeleton()
13142 s->name, err); in bpf_object__open_skeleton()
13146 *s->obj = obj; in bpf_object__open_skeleton()
13147 err = populate_skeleton_maps(obj, s->maps, s->map_cnt); in bpf_object__open_skeleton()
13149 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err); in bpf_object__open_skeleton()
13153 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt); in bpf_object__open_skeleton()
13155 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err); in bpf_object__open_skeleton()
13173 if (!s->obj) in bpf_object__open_subskeleton()
13174 return libbpf_err(-EINVAL); in bpf_object__open_subskeleton()
13176 btf = bpf_object__btf(s->obj); in bpf_object__open_subskeleton()
13179 bpf_object__name(s->obj)); in bpf_object__open_subskeleton()
13180 return libbpf_err(-errno); in bpf_object__open_subskeleton()
13183 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt); in bpf_object__open_subskeleton()
13189 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt); in bpf_object__open_subskeleton()
13195 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) { in bpf_object__open_subskeleton()
13196 var_skel = &s->vars[var_idx]; in bpf_object__open_subskeleton()
13197 map = *var_skel->map; in bpf_object__open_subskeleton()
13205 return libbpf_err(-EINVAL); in bpf_object__open_subskeleton()
13211 var_type = btf__type_by_id(btf, var->type); in bpf_object__open_subskeleton()
13212 var_name = btf__name_by_offset(btf, var_type->name_off); in bpf_object__open_subskeleton()
13213 if (strcmp(var_name, var_skel->name) == 0) { in bpf_object__open_subskeleton()
13214 *var_skel->addr = map->mmaped + var->offset; in bpf_object__open_subskeleton()
13226 free(s->maps); in bpf_object__destroy_subskeleton()
13227 free(s->progs); in bpf_object__destroy_subskeleton()
13228 free(s->vars); in bpf_object__destroy_subskeleton()
13236 err = bpf_object__load(*s->obj); in bpf_object__load_skeleton()
13238 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err); in bpf_object__load_skeleton()
13242 for (i = 0; i < s->map_cnt; i++) { in bpf_object__load_skeleton()
13243 struct bpf_map *map = *s->maps[i].map; in bpf_object__load_skeleton()
13244 size_t mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); in bpf_object__load_skeleton()
13246 void **mmaped = s->maps[i].mmaped; in bpf_object__load_skeleton()
13251 if (!(map->def.map_flags & BPF_F_MMAPABLE)) { in bpf_object__load_skeleton()
13256 if (map->def.map_flags & BPF_F_RDONLY_PROG) in bpf_object__load_skeleton()
13261 /* Remap anonymous mmap()-ed "map initialization image" as in bpf_object__load_skeleton()
13262 * a BPF map-backed mmap()-ed memory, but preserving the same in bpf_object__load_skeleton()
13271 *mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map_fd, 0); in bpf_object__load_skeleton()
13273 err = -errno; in bpf_object__load_skeleton()
13275 pr_warn("failed to re-mmap() map '%s': %d\n", in bpf_object__load_skeleton()
13288 for (i = 0; i < s->prog_cnt; i++) { in bpf_object__attach_skeleton()
13289 struct bpf_program *prog = *s->progs[i].prog; in bpf_object__attach_skeleton()
13290 struct bpf_link **link = s->progs[i].link; in bpf_object__attach_skeleton()
13292 if (!prog->autoload || !prog->autoattach) in bpf_object__attach_skeleton()
13295 /* auto-attaching not supported for this program */ in bpf_object__attach_skeleton()
13296 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) in bpf_object__attach_skeleton()
13299 /* if user already set the link manually, don't attempt auto-attach */ in bpf_object__attach_skeleton()
13303 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link); in bpf_object__attach_skeleton()
13305 pr_warn("prog '%s': failed to auto-attach: %d\n", in bpf_object__attach_skeleton()
13310 /* It's possible that for some SEC() definitions auto-attach in bpf_object__attach_skeleton()
13315 * auto-attached. But if not, it shouldn't trigger skeleton's in bpf_object__attach_skeleton()
13329 for (i = 0; i < s->prog_cnt; i++) { in bpf_object__detach_skeleton()
13330 struct bpf_link **link = s->progs[i].link; in bpf_object__detach_skeleton()
13342 if (s->progs) in bpf_object__destroy_skeleton()
13344 if (s->obj) in bpf_object__destroy_skeleton()
13345 bpf_object__close(*s->obj); in bpf_object__destroy_skeleton()
13346 free(s->maps); in bpf_object__destroy_skeleton()
13347 free(s->progs); in bpf_object__destroy_skeleton()