Lines Matching refs:obj
76 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
77 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
474 struct bpf_object *obj; member
545 struct bpf_object *obj; member
740 static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
741 static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
742 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
743 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
744 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn);
745 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
746 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
747 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx);
798 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, in bpf_object__init_prog() argument
809 prog->obj = obj; in bpf_object__init_prog()
836 prog->log_level = obj->log_level; in bpf_object__init_prog()
859 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, in bpf_object__add_programs() argument
862 Elf_Data *symbols = obj->efile.symbols; in bpf_object__add_programs()
870 progs = obj->programs; in bpf_object__add_programs()
871 nr_progs = obj->nr_programs; in bpf_object__add_programs()
875 sym = elf_sym_by_idx(obj, i); in bpf_object__add_programs()
885 name = elf_sym_str(obj, sym->st_name); in bpf_object__add_programs()
898 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { in bpf_object__add_programs()
917 obj->programs = progs; in bpf_object__add_programs()
921 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name, in bpf_object__add_programs()
939 obj->nr_programs = nr_progs; in bpf_object__add_programs()
974 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
983 find_struct_ops_kern_types(struct bpf_object *obj, const char *tname_raw, in find_struct_ops_kern_types() argument
999 kern_type_id = find_ksym_btf_id(obj, tname, BTF_KIND_STRUCT, in find_struct_ops_kern_types()
1053 static bool is_valid_st_ops_program(struct bpf_object *obj, in is_valid_st_ops_program() argument
1058 for (i = 0; i < obj->nr_programs; i++) { in is_valid_st_ops_program()
1059 if (&obj->programs[i] == prog) in is_valid_st_ops_program()
1071 static int bpf_object_adjust_struct_ops_autoload(struct bpf_object *obj) in bpf_object_adjust_struct_ops_autoload() argument
1077 for (i = 0; i < obj->nr_programs; ++i) { in bpf_object_adjust_struct_ops_autoload()
1081 prog = &obj->programs[i]; in bpf_object_adjust_struct_ops_autoload()
1085 for (j = 0; j < obj->nr_maps; ++j) { in bpf_object_adjust_struct_ops_autoload()
1088 map = &obj->maps[j]; in bpf_object_adjust_struct_ops_autoload()
1092 type = btf__type_by_id(obj->btf, map->st_ops->type_id); in bpf_object_adjust_struct_ops_autoload()
1117 struct bpf_object *obj = map->obj; in bpf_map__init_kern_struct_ops() local
1118 const struct btf *btf = obj->btf; in bpf_map__init_kern_struct_ops()
1129 err = find_struct_ops_kern_types(obj, tname, &mod_btf, in bpf_map__init_kern_struct_ops()
1136 kern_btf = mod_btf ? mod_btf->btf : obj->btf_vmlinux; in bpf_map__init_kern_struct_ops()
1236 if (!is_valid_st_ops_program(obj, prog)) { in bpf_map__init_kern_struct_ops()
1311 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) in bpf_object__init_kern_struct_ops_maps() argument
1317 for (i = 0; i < obj->nr_maps; i++) { in bpf_object__init_kern_struct_ops_maps()
1318 map = &obj->maps[i]; in bpf_object__init_kern_struct_ops_maps()
1334 static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name, in init_struct_ops_maps() argument
1349 btf = obj->btf; in init_struct_ops_maps()
1361 type = btf__type_by_id(obj->btf, vsi->type); in init_struct_ops_maps()
1362 var_name = btf__name_by_offset(obj->btf, type->name_off); in init_struct_ops_maps()
1364 type_id = btf__resolve_type(obj->btf, vsi->type); in init_struct_ops_maps()
1371 type = btf__type_by_id(obj->btf, type_id); in init_struct_ops_maps()
1372 tname = btf__name_by_offset(obj->btf, type->name_off); in init_struct_ops_maps()
1382 map = bpf_object__add_map(obj); in init_struct_ops_maps()
1438 static int bpf_object_init_struct_ops(struct bpf_object *obj) in bpf_object_init_struct_ops() argument
1443 for (sec_idx = 0; sec_idx < obj->efile.sec_cnt; ++sec_idx) { in bpf_object_init_struct_ops()
1444 struct elf_sec_desc *desc = &obj->efile.secs[sec_idx]; in bpf_object_init_struct_ops()
1449 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); in bpf_object_init_struct_ops()
1453 err = init_struct_ops_maps(obj, sec_name, sec_idx, desc->data); in bpf_object_init_struct_ops()
1466 struct bpf_object *obj; in bpf_object__new() local
1469 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); in bpf_object__new()
1470 if (!obj) { in bpf_object__new()
1475 strcpy(obj->path, path); in bpf_object__new()
1477 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); in bpf_object__new()
1480 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); in bpf_object__new()
1481 end = strchr(obj->name, '.'); in bpf_object__new()
1486 obj->efile.fd = -1; in bpf_object__new()
1493 obj->efile.obj_buf = obj_buf; in bpf_object__new()
1494 obj->efile.obj_buf_sz = obj_buf_sz; in bpf_object__new()
1495 obj->efile.btf_maps_shndx = -1; in bpf_object__new()
1496 obj->kconfig_map_idx = -1; in bpf_object__new()
1497 obj->arena_map_idx = -1; in bpf_object__new()
1499 obj->kern_version = get_kernel_version(); in bpf_object__new()
1500 obj->loaded = false; in bpf_object__new()
1502 return obj; in bpf_object__new()
1505 static void bpf_object__elf_finish(struct bpf_object *obj) in bpf_object__elf_finish() argument
1507 if (!obj->efile.elf) in bpf_object__elf_finish()
1510 elf_end(obj->efile.elf); in bpf_object__elf_finish()
1511 obj->efile.elf = NULL; in bpf_object__elf_finish()
1512 obj->efile.symbols = NULL; in bpf_object__elf_finish()
1513 obj->efile.arena_data = NULL; in bpf_object__elf_finish()
1515 zfree(&obj->efile.secs); in bpf_object__elf_finish()
1516 obj->efile.sec_cnt = 0; in bpf_object__elf_finish()
1517 zclose(obj->efile.fd); in bpf_object__elf_finish()
1518 obj->efile.obj_buf = NULL; in bpf_object__elf_finish()
1519 obj->efile.obj_buf_sz = 0; in bpf_object__elf_finish()
1522 static int bpf_object__elf_init(struct bpf_object *obj) in bpf_object__elf_init() argument
1528 if (obj->efile.elf) { in bpf_object__elf_init()
1533 if (obj->efile.obj_buf_sz > 0) { in bpf_object__elf_init()
1535 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz); in bpf_object__elf_init()
1537 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC); in bpf_object__elf_init()
1538 if (obj->efile.fd < 0) { in bpf_object__elf_init()
1543 pr_warn("elf: failed to open %s: %s\n", obj->path, cp); in bpf_object__elf_init()
1547 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL); in bpf_object__elf_init()
1551 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1)); in bpf_object__elf_init()
1556 obj->efile.elf = elf; in bpf_object__elf_init()
1560 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path); in bpf_object__elf_init()
1566 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path); in bpf_object__elf_init()
1570 obj->efile.ehdr = ehdr = elf64_getehdr(elf); in bpf_object__elf_init()
1571 if (!obj->efile.ehdr) { in bpf_object__elf_init()
1572 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1)); in bpf_object__elf_init()
1577 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) { in bpf_object__elf_init()
1579 obj->path, elf_errmsg(-1)); in bpf_object__elf_init()
1585 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) { in bpf_object__elf_init()
1587 obj->path, elf_errmsg(-1)); in bpf_object__elf_init()
1594 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path); in bpf_object__elf_init()
1601 bpf_object__elf_finish(obj); in bpf_object__elf_init()
1605 static int bpf_object__check_endianness(struct bpf_object *obj) in bpf_object__check_endianness() argument
1608 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB) in bpf_object__check_endianness()
1611 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB) in bpf_object__check_endianness()
1616 pr_warn("elf: endianness mismatch in %s.\n", obj->path); in bpf_object__check_endianness()
1621 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) in bpf_object__init_license() argument
1624 pr_warn("invalid license section in %s\n", obj->path); in bpf_object__init_license()
1630 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license))); in bpf_object__init_license()
1631 pr_debug("license of %s is %s\n", obj->path, obj->license); in bpf_object__init_license()
1636 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) in bpf_object__init_kversion() argument
1641 pr_warn("invalid kver section in %s\n", obj->path); in bpf_object__init_kversion()
1645 obj->kern_version = kver; in bpf_object__init_kversion()
1646 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); in bpf_object__init_kversion()
1658 static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size) in find_elf_sec_sz() argument
1666 scn = elf_sec_by_name(obj, name); in find_elf_sec_sz()
1667 data = elf_sec_data(obj, scn); in find_elf_sec_sz()
1676 static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name) in find_elf_var_sym() argument
1678 Elf_Data *symbols = obj->efile.symbols; in find_elf_var_sym()
1683 Elf64_Sym *sym = elf_sym_by_idx(obj, si); in find_elf_var_sym()
1692 sname = elf_sym_str(obj, sym->st_name); in find_elf_var_sym()
1727 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) in bpf_object__add_map() argument
1732 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap, in bpf_object__add_map()
1733 sizeof(*obj->maps), obj->nr_maps + 1); in bpf_object__add_map()
1737 map = &obj->maps[obj->nr_maps++]; in bpf_object__add_map()
1738 map->obj = obj; in bpf_object__add_map()
1804 static char *internal_map_name(struct bpf_object *obj, const char *real_name) in internal_map_name() argument
1849 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name)); in internal_map_name()
1851 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, in internal_map_name()
1863 map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map);
1870 static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map) in map_is_mmapable() argument
1879 t = btf__type_by_id(obj->btf, map->btf_value_type_id); in map_is_mmapable()
1885 vt = btf__type_by_id(obj->btf, vsi->type); in map_is_mmapable()
1897 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, in bpf_object__init_internal_map() argument
1905 map = bpf_object__add_map(obj); in bpf_object__init_internal_map()
1913 map->name = internal_map_name(obj, real_name); in bpf_object__init_internal_map()
1929 (void) map_fill_btf_type_info(obj, map); in bpf_object__init_internal_map()
1931 if (map_is_mmapable(obj, map)) in bpf_object__init_internal_map()
1953 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); in bpf_object__init_internal_map()
1957 static int bpf_object__init_global_data_maps(struct bpf_object *obj) in bpf_object__init_global_data_maps() argument
1966 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) { in bpf_object__init_global_data_maps()
1967 sec_desc = &obj->efile.secs[sec_idx]; in bpf_object__init_global_data_maps()
1975 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); in bpf_object__init_global_data_maps()
1976 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, in bpf_object__init_global_data_maps()
1982 obj->has_rodata = true; in bpf_object__init_global_data_maps()
1983 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); in bpf_object__init_global_data_maps()
1984 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, in bpf_object__init_global_data_maps()
1990 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); in bpf_object__init_global_data_maps()
1991 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, in bpf_object__init_global_data_maps()
2007 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, in find_extern_by_name() argument
2012 for (i = 0; i < obj->nr_extern; i++) { in find_extern_by_name()
2013 if (strcmp(obj->externs[i].name, name) == 0) in find_extern_by_name()
2014 return &obj->externs[i]; in find_extern_by_name()
2019 static struct extern_desc *find_extern_by_name_with_len(const struct bpf_object *obj, in find_extern_by_name_with_len() argument
2025 for (i = 0; i < obj->nr_extern; i++) { in find_extern_by_name_with_len()
2026 ext_name = obj->externs[i].name; in find_extern_by_name_with_len()
2028 return &obj->externs[i]; in find_extern_by_name_with_len()
2183 static int bpf_object__process_kconfig_line(struct bpf_object *obj, in bpf_object__process_kconfig_line() argument
2213 ext = find_extern_by_name(obj, buf); in bpf_object__process_kconfig_line()
2247 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) in bpf_object__read_kconfig_file() argument
2272 err = bpf_object__process_kconfig_line(obj, buf, data); in bpf_object__read_kconfig_file()
2285 static int bpf_object__read_kconfig_mem(struct bpf_object *obj, in bpf_object__read_kconfig_mem() argument
2300 err = bpf_object__process_kconfig_line(obj, buf, data); in bpf_object__read_kconfig_mem()
2312 static int bpf_object__init_kconfig_map(struct bpf_object *obj) in bpf_object__init_kconfig_map() argument
2318 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__init_kconfig_map()
2319 ext = &obj->externs[i]; in bpf_object__init_kconfig_map()
2328 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, in bpf_object__init_kconfig_map()
2329 ".kconfig", obj->efile.symbols_shndx, in bpf_object__init_kconfig_map()
2334 obj->kconfig_map_idx = obj->nr_maps - 1; in bpf_object__init_kconfig_map()
2830 static int bpf_object__init_user_btf_map(struct bpf_object *obj, in bpf_object__init_user_btf_map() argument
2845 var = btf__type_by_id(obj->btf, vi->type); in bpf_object__init_user_btf_map()
2847 map_name = btf__name_by_offset(obj->btf, var->name_off); in bpf_object__init_user_btf_map()
2868 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); in bpf_object__init_user_btf_map()
2879 map = bpf_object__add_map(obj); in bpf_object__init_user_btf_map()
2895 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def); in bpf_object__init_user_btf_map()
2925 err = map_fill_btf_type_info(obj, map); in bpf_object__init_user_btf_map()
2932 static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map, in init_arena_map_data() argument
2946 obj->arena_data = malloc(data_sz); in init_arena_map_data()
2947 if (!obj->arena_data) in init_arena_map_data()
2949 memcpy(obj->arena_data, data, data_sz); in init_arena_map_data()
2950 obj->arena_data_sz = data_sz; in init_arena_map_data()
2953 map->mmaped = obj->arena_data; in init_arena_map_data()
2958 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, in bpf_object__init_user_btf_maps() argument
2968 if (obj->efile.btf_maps_shndx < 0) in bpf_object__init_user_btf_maps()
2971 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); in bpf_object__init_user_btf_maps()
2972 data = elf_sec_data(obj, scn); in bpf_object__init_user_btf_maps()
2975 MAPS_ELF_SEC, obj->path); in bpf_object__init_user_btf_maps()
2979 nr_types = btf__type_cnt(obj->btf); in bpf_object__init_user_btf_maps()
2981 t = btf__type_by_id(obj->btf, i); in bpf_object__init_user_btf_maps()
2984 name = btf__name_by_offset(obj->btf, t->name_off); in bpf_object__init_user_btf_maps()
2987 obj->efile.btf_maps_sec_btf_id = i; in bpf_object__init_user_btf_maps()
2999 err = bpf_object__init_user_btf_map(obj, sec, i, in bpf_object__init_user_btf_maps()
3000 obj->efile.btf_maps_shndx, in bpf_object__init_user_btf_maps()
3007 for (i = 0; i < obj->nr_maps; i++) { in bpf_object__init_user_btf_maps()
3008 struct bpf_map *map = &obj->maps[i]; in bpf_object__init_user_btf_maps()
3013 if (obj->arena_map_idx >= 0) { in bpf_object__init_user_btf_maps()
3015 map->name, obj->maps[obj->arena_map_idx].name); in bpf_object__init_user_btf_maps()
3018 obj->arena_map_idx = i; in bpf_object__init_user_btf_maps()
3020 if (obj->efile.arena_data) { in bpf_object__init_user_btf_maps()
3021 err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx, in bpf_object__init_user_btf_maps()
3022 obj->efile.arena_data->d_buf, in bpf_object__init_user_btf_maps()
3023 obj->efile.arena_data->d_size); in bpf_object__init_user_btf_maps()
3028 if (obj->efile.arena_data && obj->arena_map_idx < 0) { in bpf_object__init_user_btf_maps()
3037 static int bpf_object__init_maps(struct bpf_object *obj, in bpf_object__init_maps() argument
3047 err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path); in bpf_object__init_maps()
3048 err = err ?: bpf_object__init_global_data_maps(obj); in bpf_object__init_maps()
3049 err = err ?: bpf_object__init_kconfig_map(obj); in bpf_object__init_maps()
3050 err = err ?: bpf_object_init_struct_ops(obj); in bpf_object__init_maps()
3055 static bool section_have_execinstr(struct bpf_object *obj, int idx) in section_have_execinstr() argument
3059 sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx)); in section_have_execinstr()
3071 static bool btf_needs_sanitization(struct bpf_object *obj) in btf_needs_sanitization() argument
3073 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); in btf_needs_sanitization()
3074 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); in btf_needs_sanitization()
3075 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); in btf_needs_sanitization()
3076 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); in btf_needs_sanitization()
3077 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); in btf_needs_sanitization()
3078 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); in btf_needs_sanitization()
3079 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); in btf_needs_sanitization()
3080 bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC); in btf_needs_sanitization()
3086 static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) in bpf_object__sanitize_btf() argument
3088 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); in bpf_object__sanitize_btf()
3089 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); in bpf_object__sanitize_btf()
3090 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); in bpf_object__sanitize_btf()
3091 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); in bpf_object__sanitize_btf()
3092 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); in bpf_object__sanitize_btf()
3093 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); in bpf_object__sanitize_btf()
3094 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); in bpf_object__sanitize_btf()
3095 bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC); in bpf_object__sanitize_btf()
3195 static bool libbpf_needs_btf(const struct bpf_object *obj) in libbpf_needs_btf() argument
3197 return obj->efile.btf_maps_shndx >= 0 || in libbpf_needs_btf()
3198 obj->efile.has_st_ops || in libbpf_needs_btf()
3199 obj->nr_extern > 0; in libbpf_needs_btf()
3202 static bool kernel_needs_btf(const struct bpf_object *obj) in kernel_needs_btf() argument
3204 return obj->efile.has_st_ops; in kernel_needs_btf()
3207 static int bpf_object__init_btf(struct bpf_object *obj, in bpf_object__init_btf() argument
3214 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); in bpf_object__init_btf()
3215 err = libbpf_get_error(obj->btf); in bpf_object__init_btf()
3217 obj->btf = NULL; in bpf_object__init_btf()
3222 btf__set_pointer_size(obj->btf, 8); in bpf_object__init_btf()
3228 if (!obj->btf) { in bpf_object__init_btf()
3233 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); in bpf_object__init_btf()
3234 err = libbpf_get_error(obj->btf_ext); in bpf_object__init_btf()
3238 obj->btf_ext = NULL; in bpf_object__init_btf()
3243 ext_segs[0] = &obj->btf_ext->func_info; in bpf_object__init_btf()
3244 ext_segs[1] = &obj->btf_ext->line_info; in bpf_object__init_btf()
3245 ext_segs[2] = &obj->btf_ext->core_relo_info; in bpf_object__init_btf()
3268 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); in bpf_object__init_btf()
3271 scn = elf_sec_by_name(obj, sec_name); in bpf_object__init_btf()
3280 if (err && libbpf_needs_btf(obj)) { in bpf_object__init_btf()
3295 static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, in btf_fixup_datasec() argument
3327 err = find_elf_sec_sz(obj, sec_name, &size); in btf_fixup_datasec()
3361 sym = find_elf_var_sym(obj, var_name); in btf_fixup_datasec()
3388 static int bpf_object_fixup_btf(struct bpf_object *obj) in bpf_object_fixup_btf() argument
3392 if (!obj->btf) in bpf_object_fixup_btf()
3395 n = btf__type_cnt(obj->btf); in bpf_object_fixup_btf()
3397 struct btf_type *t = btf_type_by_id(obj->btf, i); in bpf_object_fixup_btf()
3405 err = btf_fixup_datasec(obj, obj->btf, t); in bpf_object_fixup_btf()
3434 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) in obj_needs_vmlinux_btf() argument
3443 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path) in obj_needs_vmlinux_btf()
3447 for (i = 0; i < obj->nr_extern; i++) { in obj_needs_vmlinux_btf()
3450 ext = &obj->externs[i]; in obj_needs_vmlinux_btf()
3455 bpf_object__for_each_program(prog, obj) { in obj_needs_vmlinux_btf()
3462 bpf_object__for_each_map(map, obj) { in obj_needs_vmlinux_btf()
3470 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force) in bpf_object__load_vmlinux_btf() argument
3475 if (obj->btf_vmlinux || obj->gen_loader) in bpf_object__load_vmlinux_btf()
3478 if (!force && !obj_needs_vmlinux_btf(obj)) in bpf_object__load_vmlinux_btf()
3481 obj->btf_vmlinux = btf__load_vmlinux_btf(); in bpf_object__load_vmlinux_btf()
3482 err = libbpf_get_error(obj->btf_vmlinux); in bpf_object__load_vmlinux_btf()
3485 obj->btf_vmlinux = NULL; in bpf_object__load_vmlinux_btf()
3491 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) in bpf_object__sanitize_and_load_btf() argument
3493 struct btf *kern_btf = obj->btf; in bpf_object__sanitize_and_load_btf()
3497 if (!obj->btf) in bpf_object__sanitize_and_load_btf()
3500 if (!kernel_supports(obj, FEAT_BTF)) { in bpf_object__sanitize_and_load_btf()
3501 if (kernel_needs_btf(obj)) { in bpf_object__sanitize_and_load_btf()
3517 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__sanitize_and_load_btf()
3518 struct bpf_program *prog = &obj->programs[i]; in bpf_object__sanitize_and_load_btf()
3523 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) in bpf_object__sanitize_and_load_btf()
3526 n = btf__type_cnt(obj->btf); in bpf_object__sanitize_and_load_btf()
3528 t = btf_type_by_id(obj->btf, j); in bpf_object__sanitize_and_load_btf()
3532 name = btf__str_by_offset(obj->btf, t->name_off); in bpf_object__sanitize_and_load_btf()
3541 sanitize = btf_needs_sanitization(obj); in bpf_object__sanitize_and_load_btf()
3547 raw_data = btf__raw_data(obj->btf, &sz); in bpf_object__sanitize_and_load_btf()
3554 btf__set_pointer_size(obj->btf, 8); in bpf_object__sanitize_and_load_btf()
3555 err = bpf_object__sanitize_btf(obj, kern_btf); in bpf_object__sanitize_and_load_btf()
3560 if (obj->gen_loader) { in bpf_object__sanitize_and_load_btf()
3566 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size); in bpf_object__sanitize_and_load_btf()
3573 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size, in bpf_object__sanitize_and_load_btf()
3574 obj->log_level ? 1 : 0, obj->token_fd); in bpf_object__sanitize_and_load_btf()
3579 btf__set_fd(obj->btf, btf__fd(kern_btf)); in bpf_object__sanitize_and_load_btf()
3586 btf_mandatory = kernel_needs_btf(obj); in bpf_object__sanitize_and_load_btf()
3596 static const char *elf_sym_str(const struct bpf_object *obj, size_t off) in elf_sym_str() argument
3600 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off); in elf_sym_str()
3603 off, obj->path, elf_errmsg(-1)); in elf_sym_str()
3610 static const char *elf_sec_str(const struct bpf_object *obj, size_t off) in elf_sec_str() argument
3614 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off); in elf_sec_str()
3617 off, obj->path, elf_errmsg(-1)); in elf_sec_str()
3624 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx) in elf_sec_by_idx() argument
3628 scn = elf_getscn(obj->efile.elf, idx); in elf_sec_by_idx()
3631 idx, obj->path, elf_errmsg(-1)); in elf_sec_by_idx()
3637 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name) in elf_sec_by_name() argument
3640 Elf *elf = obj->efile.elf; in elf_sec_by_name()
3644 sec_name = elf_sec_name(obj, scn); in elf_sec_by_name()
3656 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn) in elf_sec_hdr() argument
3666 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); in elf_sec_hdr()
3673 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn) in elf_sec_name() argument
3681 sh = elf_sec_hdr(obj, scn); in elf_sec_name()
3685 name = elf_sec_str(obj, sh->sh_name); in elf_sec_name()
3688 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); in elf_sec_name()
3695 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn) in elf_sec_data() argument
3705 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>", in elf_sec_data()
3706 obj->path, elf_errmsg(-1)); in elf_sec_data()
3713 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx) in elf_sym_by_idx() argument
3715 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym)) in elf_sym_by_idx()
3718 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx; in elf_sym_by_idx()
3781 static int bpf_object__elf_collect(struct bpf_object *obj) in bpf_object__elf_collect() argument
3784 Elf *elf = obj->efile.elf; in bpf_object__elf_collect()
3798 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) { in bpf_object__elf_collect()
3800 obj->path, elf_errmsg(-1)); in bpf_object__elf_collect()
3803 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs)); in bpf_object__elf_collect()
3804 if (!obj->efile.secs) in bpf_object__elf_collect()
3812 sh = elf_sec_hdr(obj, scn); in bpf_object__elf_collect()
3817 if (obj->efile.symbols) { in bpf_object__elf_collect()
3818 pr_warn("elf: multiple symbol tables in %s\n", obj->path); in bpf_object__elf_collect()
3822 data = elf_sec_data(obj, scn); in bpf_object__elf_collect()
3828 obj->efile.symbols = data; in bpf_object__elf_collect()
3829 obj->efile.symbols_shndx = idx; in bpf_object__elf_collect()
3830 obj->efile.strtabidx = sh->sh_link; in bpf_object__elf_collect()
3834 if (!obj->efile.symbols) { in bpf_object__elf_collect()
3836 obj->path); in bpf_object__elf_collect()
3843 sec_desc = &obj->efile.secs[idx]; in bpf_object__elf_collect()
3845 sh = elf_sec_hdr(obj, scn); in bpf_object__elf_collect()
3849 name = elf_sec_str(obj, sh->sh_name); in bpf_object__elf_collect()
3856 data = elf_sec_data(obj, scn); in bpf_object__elf_collect()
3866 err = bpf_object__init_license(obj, data->d_buf, data->d_size); in bpf_object__elf_collect()
3870 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); in bpf_object__elf_collect()
3877 obj->efile.btf_maps_shndx = idx; in bpf_object__elf_collect()
3891 obj->efile.text_shndx = idx; in bpf_object__elf_collect()
3892 err = bpf_object__add_programs(obj, data, name, idx); in bpf_object__elf_collect()
3912 obj->efile.has_st_ops = true; in bpf_object__elf_collect()
3914 obj->efile.arena_data = data; in bpf_object__elf_collect()
3915 obj->efile.arena_data_shndx = idx; in bpf_object__elf_collect()
3924 targ_sec_idx >= obj->efile.sec_cnt) in bpf_object__elf_collect()
3928 if (!section_have_execinstr(obj, targ_sec_idx) && in bpf_object__elf_collect()
3936 elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>"); in bpf_object__elf_collect()
3954 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { in bpf_object__elf_collect()
3955 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path); in bpf_object__elf_collect()
3962 if (obj->nr_programs) in bpf_object__elf_collect()
3963 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs); in bpf_object__elf_collect()
3965 return bpf_object__init_btf(obj, btf_data, btf_ext_data); in bpf_object__elf_collect()
4174 static int bpf_object__collect_externs(struct bpf_object *obj) in bpf_object__collect_externs() argument
4185 if (!obj->efile.symbols) in bpf_object__collect_externs()
4188 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); in bpf_object__collect_externs()
4189 sh = elf_sec_hdr(obj, scn); in bpf_object__collect_externs()
4193 dummy_var_btf_id = add_dummy_ksym_var(obj->btf); in bpf_object__collect_externs()
4201 Elf64_Sym *sym = elf_sym_by_idx(obj, i); in bpf_object__collect_externs()
4207 ext_name = elf_sym_str(obj, sym->st_name); in bpf_object__collect_externs()
4211 ext = obj->externs; in bpf_object__collect_externs()
4212 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); in bpf_object__collect_externs()
4215 obj->externs = ext; in bpf_object__collect_externs()
4216 ext = &ext[obj->nr_extern]; in bpf_object__collect_externs()
4218 obj->nr_extern++; in bpf_object__collect_externs()
4220 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); in bpf_object__collect_externs()
4226 t = btf__type_by_id(obj->btf, ext->btf_id); in bpf_object__collect_externs()
4227 ext->name = strdup(btf__name_by_offset(obj->btf, t->name_off)); in bpf_object__collect_externs()
4241 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); in bpf_object__collect_externs()
4247 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); in bpf_object__collect_externs()
4248 sec_name = btf__name_by_offset(obj->btf, sec->name_off); in bpf_object__collect_externs()
4258 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); in bpf_object__collect_externs()
4264 ext->kcfg.align = btf__align_of(obj->btf, t->type); in bpf_object__collect_externs()
4270 ext->kcfg.type = find_kcfg_type(obj->btf, t->type, in bpf_object__collect_externs()
4279 skip_mods_and_typedefs(obj->btf, t->type, in bpf_object__collect_externs()
4286 pr_debug("collected %d externs total\n", obj->nr_extern); in bpf_object__collect_externs()
4288 if (!obj->nr_extern) in bpf_object__collect_externs()
4292 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); in bpf_object__collect_externs()
4302 int int_btf_id = find_int_btf_id(obj->btf); in bpf_object__collect_externs()
4310 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id); in bpf_object__collect_externs()
4311 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__collect_externs()
4312 ext = &obj->externs[i]; in bpf_object__collect_externs()
4325 vt = (void *)btf__type_by_id(obj->btf, vs->type); in bpf_object__collect_externs()
4326 ext_name = btf__name_by_offset(obj->btf, vt->name_off); in bpf_object__collect_externs()
4327 ext = find_extern_by_name(obj, ext_name); in bpf_object__collect_externs()
4338 func_proto = btf__type_by_id(obj->btf, in bpf_object__collect_externs()
4365 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__collect_externs()
4366 ext = &obj->externs[i]; in bpf_object__collect_externs()
4380 t = btf__type_by_id(obj->btf, vs->type); in bpf_object__collect_externs()
4381 ext_name = btf__name_by_offset(obj->btf, t->name_off); in bpf_object__collect_externs()
4382 ext = find_extern_by_name(obj, ext_name); in bpf_object__collect_externs()
4395 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog) in prog_is_subprog() argument
4397 return prog->sec_idx == obj->efile.text_shndx; in prog_is_subprog()
4401 bpf_object__find_program_by_name(const struct bpf_object *obj, in bpf_object__find_program_by_name() argument
4406 bpf_object__for_each_program(prog, obj) { in bpf_object__find_program_by_name()
4407 if (prog_is_subprog(obj, prog)) in bpf_object__find_program_by_name()
4415 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, in bpf_object__shndx_is_data() argument
4418 switch (obj->efile.secs[shndx].sec_type) { in bpf_object__shndx_is_data()
4428 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, in bpf_object__shndx_is_maps() argument
4431 return shndx == obj->efile.btf_maps_shndx; in bpf_object__shndx_is_maps()
4435 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) in bpf_object__section_to_libbpf_map_type() argument
4437 if (shndx == obj->efile.symbols_shndx) in bpf_object__section_to_libbpf_map_type()
4440 switch (obj->efile.secs[shndx].sec_type) { in bpf_object__section_to_libbpf_map_type()
4458 size_t map_idx, nr_maps = prog->obj->nr_maps; in bpf_program__record_reloc()
4459 struct bpf_object *obj = prog->obj; in bpf_program__record_reloc() local
4473 int i, n = obj->nr_extern; in bpf_program__record_reloc()
4477 ext = &obj->externs[i]; in bpf_program__record_reloc()
4504 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { in bpf_program__record_reloc()
4505 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); in bpf_program__record_reloc()
4528 if (sym_is_subprog(sym, obj->efile.text_shndx)) { in bpf_program__record_reloc()
4544 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); in bpf_program__record_reloc()
4545 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); in bpf_program__record_reloc()
4548 if (shdr_idx == obj->efile.arena_data_shndx) { in bpf_program__record_reloc()
4549 if (obj->arena_map_idx < 0) { in bpf_program__record_reloc()
4556 reloc_desc->map_idx = obj->arena_map_idx; in bpf_program__record_reloc()
4559 map = &obj->maps[obj->arena_map_idx]; in bpf_program__record_reloc()
4561 prog->name, obj->arena_map_idx, map->name, map->sec_idx, in bpf_program__record_reloc()
4568 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { in bpf_program__record_reloc()
4574 map = &obj->maps[map_idx]; in bpf_program__record_reloc()
4597 if (!bpf_object__shndx_is_data(obj, shdr_idx)) { in bpf_program__record_reloc()
4603 map = &obj->maps[map_idx]; in bpf_program__record_reloc()
4630 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj, in find_prog_by_sec_insn() argument
4633 int l = 0, r = obj->nr_programs - 1, m; in find_prog_by_sec_insn()
4636 if (!obj->nr_programs) in find_prog_by_sec_insn()
4641 prog = &obj->programs[m]; in find_prog_by_sec_insn()
4652 prog = &obj->programs[l]; in find_prog_by_sec_insn()
4659 bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) in bpf_object__collect_prog_relos() argument
4673 if (sec_idx >= obj->efile.sec_cnt) in bpf_object__collect_prog_relos()
4676 scn = elf_sec_by_idx(obj, sec_idx); in bpf_object__collect_prog_relos()
4677 scn_data = elf_sec_data(obj, scn); in bpf_object__collect_prog_relos()
4681 relo_sec_name = elf_sec_str(obj, shdr->sh_name); in bpf_object__collect_prog_relos()
4682 sec_name = elf_sec_name(obj, scn); in bpf_object__collect_prog_relos()
4698 sym = elf_sym_by_idx(obj, sym_idx); in bpf_object__collect_prog_relos()
4705 if (sym->st_shndx >= obj->efile.sec_cnt) { in bpf_object__collect_prog_relos()
4725 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)); in bpf_object__collect_prog_relos()
4727 sym_name = elf_sym_str(obj, sym->st_name); in bpf_object__collect_prog_relos()
4733 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); in bpf_object__collect_prog_relos()
4758 static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map) in map_fill_btf_type_info() argument
4762 if (!obj->btf) in map_fill_btf_type_info()
4769 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map)) in map_fill_btf_type_info()
4779 id = btf__find_by_name(obj->btf, map->real_name); in map_fill_btf_type_info()
4831 if (map->obj->loaded) in bpf_map__set_autocreate()
4925 if (map->obj->loaded) in bpf_map__set_max_entries()
4937 static int bpf_object_prepare_token(struct bpf_object *obj) in bpf_object_prepare_token() argument
4945 if (obj->token_path && obj->token_path[0] == '\0') { in bpf_object_prepare_token()
4946 pr_debug("object '%s': token is prevented, skipping...\n", obj->name); in bpf_object_prepare_token()
4950 mandatory = obj->token_path != NULL; in bpf_object_prepare_token()
4953 bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH; in bpf_object_prepare_token()
4958 obj->name, err, bpffs_path, in bpf_object_prepare_token()
4968 obj->name, bpffs_path); in bpf_object_prepare_token()
4972 obj->name, token_fd, bpffs_path, in bpf_object_prepare_token()
4977 obj->feat_cache = calloc(1, sizeof(*obj->feat_cache)); in bpf_object_prepare_token()
4978 if (!obj->feat_cache) { in bpf_object_prepare_token()
4983 obj->token_fd = token_fd; in bpf_object_prepare_token()
4984 obj->feat_cache->token_fd = token_fd; in bpf_object_prepare_token()
4990 bpf_object__probe_loading(struct bpf_object *obj) in bpf_object__probe_loading() argument
4999 .token_fd = obj->token_fd, in bpf_object__probe_loading()
5000 .prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0, in bpf_object__probe_loading()
5003 if (obj->gen_loader) in bpf_object__probe_loading()
5028 bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) in kernel_supports() argument
5030 if (obj->gen_loader) in kernel_supports()
5036 if (obj->token_fd) in kernel_supports()
5037 return feat_supported(obj->feat_cache, feat_id); in kernel_supports()
5107 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) in bpf_object__populate_internal_map() argument
5114 if (obj->gen_loader) { in bpf_object__populate_internal_map()
5115 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, in bpf_object__populate_internal_map()
5118 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); in bpf_object__populate_internal_map()
5181 return map->obj->loaded || map->reused; in map_is_created()
5184 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner) in bpf_object__create_map() argument
5191 if (kernel_supports(obj, FEAT_PROG_NAME)) in bpf_object__create_map()
5197 create_attr.token_fd = obj->token_fd; in bpf_object__create_map()
5198 if (obj->token_fd) in bpf_object__create_map()
5209 if (obj->btf && btf__fd(obj->btf) >= 0) { in bpf_object__create_map()
5210 create_attr.btf_fd = btf__fd(obj->btf); in bpf_object__create_map()
5220 err = bpf_object__create_map(obj, map->inner_map, true); in bpf_object__create_map()
5260 if (obj->gen_loader) { in bpf_object__create_map()
5261 bpf_gen__map_create(obj->gen_loader, def->type, map_name, in bpf_object__create_map()
5263 &create_attr, is_inner ? -1 : map - obj->maps); in bpf_object__create_map()
5294 if (obj->gen_loader) in bpf_object__create_map()
5315 static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) in init_map_in_map_slots() argument
5328 if (obj->gen_loader) { in init_map_in_map_slots()
5329 bpf_gen__populate_outer_map(obj->gen_loader, in init_map_in_map_slots()
5330 map - obj->maps, i, in init_map_in_map_slots()
5331 targ_map - obj->maps); in init_map_in_map_slots()
5351 static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) in init_prog_array_slots() argument
5357 if (obj->gen_loader) in init_prog_array_slots()
5384 static int bpf_object_init_prog_arrays(struct bpf_object *obj) in bpf_object_init_prog_arrays() argument
5389 for (i = 0; i < obj->nr_maps; i++) { in bpf_object_init_prog_arrays()
5390 map = &obj->maps[i]; in bpf_object_init_prog_arrays()
5395 err = init_prog_array_slots(obj, map); in bpf_object_init_prog_arrays()
5421 bpf_object__create_maps(struct bpf_object *obj) in bpf_object__create_maps() argument
5429 for (i = 0; i < obj->nr_maps; i++) { in bpf_object__create_maps()
5430 map = &obj->maps[i]; in bpf_object__create_maps()
5446 if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA)) in bpf_object__create_maps()
5479 err = bpf_object__create_map(obj, map, false); in bpf_object__create_maps()
5487 err = bpf_object__populate_internal_map(obj, map); in bpf_object__create_maps()
5502 if (obj->arena_data) { in bpf_object__create_maps()
5503 memcpy(map->mmaped, obj->arena_data, obj->arena_data_sz); in bpf_object__create_maps()
5504 zfree(&obj->arena_data); in bpf_object__create_maps()
5508 err = init_map_in_map_slots(obj, map); in bpf_object__create_maps()
5535 zclose(obj->maps[j].fd); in bpf_object__create_maps()
5624 static int load_module_btfs(struct bpf_object *obj) in load_module_btfs() argument
5633 if (obj->btf_modules_loaded) in load_module_btfs()
5636 if (obj->gen_loader) in load_module_btfs()
5640 obj->btf_modules_loaded = true; in load_module_btfs()
5643 if (!kernel_supports(obj, FEAT_MODULE_BTF)) in load_module_btfs()
5687 btf = btf_get_from_fd(fd, obj->btf_vmlinux); in load_module_btfs()
5695 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, in load_module_btfs()
5696 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); in load_module_btfs()
5700 mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; in load_module_btfs()
5721 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id) in bpf_core_find_cands() argument
5747 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; in bpf_core_find_cands()
5757 if (obj->btf_vmlinux_override) in bpf_core_find_cands()
5761 err = load_module_btfs(obj); in bpf_core_find_cands()
5765 for (i = 0; i < obj->btf_module_cnt; i++) { in bpf_core_find_cands()
5767 obj->btf_modules[i].btf, in bpf_core_find_cands()
5768 obj->btf_modules[i].name, in bpf_core_find_cands()
5769 btf__type_cnt(obj->btf_vmlinux), in bpf_core_find_cands()
5881 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); in bpf_core_resolve_relo()
5900 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) in bpf_object__relocate_core() argument
5913 if (obj->btf_ext->core_relo_info.len == 0) in bpf_object__relocate_core()
5917 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL); in bpf_object__relocate_core()
5918 err = libbpf_get_error(obj->btf_vmlinux_override); in bpf_object__relocate_core()
5931 seg = &obj->btf_ext->core_relo_info; in bpf_object__relocate_core()
5937 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); in bpf_object__relocate_core()
5949 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); in bpf_object__relocate_core()
5985 if (prog->obj->gen_loader) in bpf_object__relocate_core()
5988 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res); in bpf_object__relocate_core()
6006 btf__free(obj->btf_vmlinux_override); in bpf_object__relocate_core()
6007 obj->btf_vmlinux_override = NULL; in bpf_object__relocate_core()
6078 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) in bpf_object__relocate_data() argument
6090 map = &obj->maps[relo->map_idx]; in bpf_object__relocate_data()
6091 if (obj->gen_loader) { in bpf_object__relocate_data()
6103 map = &obj->maps[relo->map_idx]; in bpf_object__relocate_data()
6105 if (obj->gen_loader) { in bpf_object__relocate_data()
6117 ext = &obj->externs[relo->ext_idx]; in bpf_object__relocate_data()
6119 if (obj->gen_loader) { in bpf_object__relocate_data()
6121 insn[0].imm = obj->kconfig_map_idx; in bpf_object__relocate_data()
6124 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; in bpf_object__relocate_data()
6139 ext = &obj->externs[relo->ext_idx]; in bpf_object__relocate_data()
6173 static int adjust_prog_btf_ext_info(const struct bpf_object *obj, in adjust_prog_btf_ext_info() argument
6242 reloc_prog_func_and_line_info(const struct bpf_object *obj, in reloc_prog_func_and_line_info() argument
6251 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC)) in reloc_prog_func_and_line_info()
6260 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, in reloc_prog_func_and_line_info()
6288 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, in reloc_prog_func_and_line_info()
6361 bpf_object__append_subprog_code(struct bpf_object *obj, struct bpf_program *main_prog, in bpf_object__append_subprog_code() argument
6393 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, in bpf_object__reloc_code() argument
6402 err = reloc_prog_func_and_line_info(obj, main_prog, prog); in bpf_object__reloc_code()
6456 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx); in bpf_object__reloc_code()
6474 err = bpf_object__append_subprog_code(obj, main_prog, subprog); in bpf_object__reloc_code()
6477 err = bpf_object__reloc_code(obj, main_prog, subprog); in bpf_object__reloc_code()
6583 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) in bpf_object__relocate_calls() argument
6591 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__relocate_calls()
6592 subprog = &obj->programs[i]; in bpf_object__relocate_calls()
6593 if (!prog_is_subprog(obj, subprog)) in bpf_object__relocate_calls()
6599 err = bpf_object__reloc_code(obj, prog, prog); in bpf_object__relocate_calls()
6607 bpf_object__free_relocs(struct bpf_object *obj) in bpf_object__free_relocs() argument
6613 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__free_relocs()
6614 prog = &obj->programs[i]; in bpf_object__free_relocs()
6635 static void bpf_object__sort_relos(struct bpf_object *obj) in bpf_object__sort_relos() argument
6639 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__sort_relos()
6640 struct bpf_program *p = &obj->programs[i]; in bpf_object__sort_relos()
6649 static int bpf_prog_assign_exc_cb(struct bpf_object *obj, struct bpf_program *prog) in bpf_prog_assign_exc_cb() argument
6655 if (!obj->btf || !kernel_supports(obj, FEAT_BTF_DECL_TAG)) in bpf_prog_assign_exc_cb()
6658 n = btf__type_cnt(obj->btf); in bpf_prog_assign_exc_cb()
6663 t = btf_type_by_id(obj->btf, i); in bpf_prog_assign_exc_cb()
6667 name = btf__str_by_offset(obj->btf, t->name_off); in bpf_prog_assign_exc_cb()
6671 t = btf_type_by_id(obj->btf, t->type); in bpf_prog_assign_exc_cb()
6677 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0) in bpf_prog_assign_exc_cb()
6695 for (j = 0; j < obj->nr_programs; j++) { in bpf_prog_assign_exc_cb()
6696 struct bpf_program *subprog = &obj->programs[j]; in bpf_prog_assign_exc_cb()
6698 if (!prog_is_subprog(obj, subprog)) in bpf_prog_assign_exc_cb()
6911 static int bpf_program_fixup_func_info(struct bpf_object *obj, struct bpf_program *prog) in bpf_program_fixup_func_info() argument
6916 struct btf *btf = obj->btf; in bpf_program_fixup_func_info()
6924 if (!obj->btf_ext || !prog->func_info) in bpf_program_fixup_func_info()
6928 if (kernel_supports(obj, FEAT_ARG_CTX_TAG)) in bpf_program_fixup_func_info()
7050 static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) in bpf_object__relocate() argument
7056 if (obj->btf_ext) { in bpf_object__relocate()
7057 err = bpf_object__relocate_core(obj, targ_btf_path); in bpf_object__relocate()
7063 bpf_object__sort_relos(obj); in bpf_object__relocate()
7073 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__relocate()
7074 prog = &obj->programs[i]; in bpf_object__relocate()
7092 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__relocate()
7093 prog = &obj->programs[i]; in bpf_object__relocate()
7097 if (prog_is_subprog(obj, prog)) in bpf_object__relocate()
7102 err = bpf_object__relocate_calls(obj, prog); in bpf_object__relocate()
7109 err = bpf_prog_assign_exc_cb(obj, prog); in bpf_object__relocate()
7114 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx]; in bpf_object__relocate()
7122 err = bpf_object__append_subprog_code(obj, prog, subprog); in bpf_object__relocate()
7125 err = bpf_object__reloc_code(obj, prog, subprog); in bpf_object__relocate()
7131 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__relocate()
7132 prog = &obj->programs[i]; in bpf_object__relocate()
7133 if (prog_is_subprog(obj, prog)) in bpf_object__relocate()
7139 err = bpf_object__relocate_data(obj, prog); in bpf_object__relocate()
7147 err = bpf_program_fixup_func_info(obj, prog); in bpf_object__relocate()
7158 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
7161 static int bpf_object__collect_map_relos(struct bpf_object *obj, in bpf_object__collect_map_relos() argument
7178 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) in bpf_object__collect_map_relos()
7180 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); in bpf_object__collect_map_relos()
7192 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); in bpf_object__collect_map_relos()
7198 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; in bpf_object__collect_map_relos()
7204 for (j = 0; j < obj->nr_maps; j++) { in bpf_object__collect_map_relos()
7205 map = &obj->maps[j]; in bpf_object__collect_map_relos()
7206 if (map->sec_idx != obj->efile.btf_maps_shndx) in bpf_object__collect_map_relos()
7214 if (j == obj->nr_maps) { in bpf_object__collect_map_relos()
7224 if (sym->st_shndx != obj->efile.btf_maps_shndx) { in bpf_object__collect_map_relos()
7235 targ_map = bpf_object__find_map_by_name(obj, name); in bpf_object__collect_map_relos()
7242 targ_prog = bpf_object__find_program_by_name(obj, name); in bpf_object__collect_map_relos()
7250 prog_is_subprog(obj, targ_prog)) { in bpf_object__collect_map_relos()
7259 var = btf__type_by_id(obj->btf, vi->type); in bpf_object__collect_map_relos()
7260 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); in bpf_object__collect_map_relos()
7264 mname = btf__name_by_offset(obj->btf, member->name_off); in bpf_object__collect_map_relos()
7298 static int bpf_object__collect_relos(struct bpf_object *obj) in bpf_object__collect_relos() argument
7302 for (i = 0; i < obj->efile.sec_cnt; i++) { in bpf_object__collect_relos()
7303 struct elf_sec_desc *sec_desc = &obj->efile.secs[i]; in bpf_object__collect_relos()
7315 if (shdr->sh_type != SHT_REL || idx < 0 || idx >= obj->efile.sec_cnt) { in bpf_object__collect_relos()
7320 if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) in bpf_object__collect_relos()
7321 err = bpf_object__collect_st_ops_relos(obj, shdr, data); in bpf_object__collect_relos()
7322 else if (idx == obj->efile.btf_maps_shndx) in bpf_object__collect_relos()
7323 err = bpf_object__collect_map_relos(obj, shdr, data); in bpf_object__collect_relos()
7325 err = bpf_object__collect_prog_relos(obj, shdr, data); in bpf_object__collect_relos()
7330 bpf_object__sort_relos(obj); in bpf_object__collect_relos()
7347 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog) in bpf_object__sanitize_prog() argument
7353 if (obj->gen_loader) in bpf_object__sanitize_prog()
7367 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) in bpf_object__sanitize_prog()
7372 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) in bpf_object__sanitize_prog()
7392 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) in libbpf_prepare_prog_load()
7402 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) { in libbpf_prepare_prog_load()
7453 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, in bpf_object_load_prog() argument
7492 if (kernel_supports(obj, FEAT_PROG_NAME)) in bpf_object_load_prog()
7502 if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) { in bpf_object_load_prog()
7503 load_attr.prog_btf_fd = btf__fd(obj->btf); in bpf_object_load_prog()
7513 load_attr.fd_array = obj->fd_array; in bpf_object_load_prog()
7515 load_attr.token_fd = obj->token_fd; in bpf_object_load_prog()
7516 if (obj->token_fd) in bpf_object_load_prog()
7531 if (obj->gen_loader) { in bpf_object_load_prog()
7532 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, in bpf_object_load_prog()
7534 prog - obj->programs); in bpf_object_load_prog()
7550 } else if (obj->log_buf) { in bpf_object_load_prog()
7551 log_buf = obj->log_buf; in bpf_object_load_prog()
7552 log_buf_size = obj->log_size; in bpf_object_load_prog()
7578 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) { in bpf_object_load_prog()
7582 for (i = 0; i < obj->nr_maps; i++) { in bpf_object_load_prog()
7583 map = &prog->obj->maps[i]; in bpf_object_load_prog()
7712 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec); in fixup_log_failed_core_relo()
7737 struct bpf_object *obj = prog->obj; in fixup_log_missing_map_load() local
7746 if (map_idx < 0 || map_idx >= obj->nr_maps) in fixup_log_missing_map_load()
7748 map = &obj->maps[map_idx]; in fixup_log_missing_map_load()
7770 struct bpf_object *obj = prog->obj; in fixup_log_missing_kfunc_call() local
7779 if (ext_idx < 0 || ext_idx >= obj->nr_extern) in fixup_log_missing_kfunc_call()
7781 ext = &obj->externs[ext_idx]; in fixup_log_missing_kfunc_call()
7843 struct bpf_object *obj = prog->obj; in bpf_program_record_relos() local
7848 struct extern_desc *ext = &obj->externs[relo->ext_idx]; in bpf_program_record_relos()
7855 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ? in bpf_program_record_relos()
7857 bpf_gen__record_extern(obj->gen_loader, ext->name, in bpf_program_record_relos()
7862 bpf_gen__record_extern(obj->gen_loader, ext->name, in bpf_program_record_relos()
7874 bpf_gen__record_relo_core(obj->gen_loader, &cr); in bpf_program_record_relos()
7885 bpf_object__load_progs(struct bpf_object *obj, int log_level) in bpf_object__load_progs() argument
7891 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__load_progs()
7892 prog = &obj->programs[i]; in bpf_object__load_progs()
7893 err = bpf_object__sanitize_prog(obj, prog); in bpf_object__load_progs()
7898 for (i = 0; i < obj->nr_programs; i++) { in bpf_object__load_progs()
7899 prog = &obj->programs[i]; in bpf_object__load_progs()
7900 if (prog_is_subprog(obj, prog)) in bpf_object__load_progs()
7908 if (obj->gen_loader) in bpf_object__load_progs()
7911 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, in bpf_object__load_progs()
7912 obj->license, obj->kern_version, &prog->fd); in bpf_object__load_progs()
7919 bpf_object__free_relocs(obj); in bpf_object__load_progs()
7925 static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts) in bpf_object_init_progs() argument
7930 bpf_object__for_each_program(prog, obj) { in bpf_object_init_progs()
7963 struct bpf_object *obj; in bpf_object_open() local
8007 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); in bpf_object_open()
8008 if (IS_ERR(obj)) in bpf_object_open()
8009 return obj; in bpf_object_open()
8011 obj->log_buf = log_buf; in bpf_object_open()
8012 obj->log_size = log_size; in bpf_object_open()
8013 obj->log_level = log_level; in bpf_object_open()
8016 obj->token_path = strdup(token_path); in bpf_object_open()
8017 if (!obj->token_path) { in bpf_object_open()
8029 obj->btf_custom_path = strdup(btf_tmp_path); in bpf_object_open()
8030 if (!obj->btf_custom_path) { in bpf_object_open()
8038 obj->kconfig = strdup(kconfig); in bpf_object_open()
8039 if (!obj->kconfig) { in bpf_object_open()
8045 err = bpf_object__elf_init(obj); in bpf_object_open()
8046 err = err ? : bpf_object__check_endianness(obj); in bpf_object_open()
8047 err = err ? : bpf_object__elf_collect(obj); in bpf_object_open()
8048 err = err ? : bpf_object__collect_externs(obj); in bpf_object_open()
8049 err = err ? : bpf_object_fixup_btf(obj); in bpf_object_open()
8050 err = err ? : bpf_object__init_maps(obj, opts); in bpf_object_open()
8051 err = err ? : bpf_object_init_progs(obj, opts); in bpf_object_open()
8052 err = err ? : bpf_object__collect_relos(obj); in bpf_object_open()
8056 bpf_object__elf_finish(obj); in bpf_object_open()
8058 return obj; in bpf_object_open()
8060 bpf_object__close(obj); in bpf_object_open()
8093 static int bpf_object_unload(struct bpf_object *obj) in bpf_object_unload() argument
8097 if (!obj) in bpf_object_unload()
8100 for (i = 0; i < obj->nr_maps; i++) { in bpf_object_unload()
8101 zclose(obj->maps[i].fd); in bpf_object_unload()
8102 if (obj->maps[i].st_ops) in bpf_object_unload()
8103 zfree(&obj->maps[i].st_ops->kern_vdata); in bpf_object_unload()
8106 for (i = 0; i < obj->nr_programs; i++) in bpf_object_unload()
8107 bpf_program__unload(&obj->programs[i]); in bpf_object_unload()
8112 static int bpf_object__sanitize_maps(struct bpf_object *obj) in bpf_object__sanitize_maps() argument
8116 bpf_object__for_each_map(m, obj) { in bpf_object__sanitize_maps()
8119 if (!kernel_supports(obj, FEAT_ARRAY_MMAP)) in bpf_object__sanitize_maps()
8166 struct bpf_object *obj = ctx; in kallsyms_cb() local
8173 ext = find_extern_by_name_with_len(obj, sym_name, res - sym_name); in kallsyms_cb()
8175 ext = find_extern_by_name(obj, sym_name); in kallsyms_cb()
8179 t = btf__type_by_id(obj->btf, ext->btf_id); in kallsyms_cb()
8196 static int bpf_object__read_kallsyms_file(struct bpf_object *obj) in bpf_object__read_kallsyms_file() argument
8198 return libbpf_kallsyms_parse(kallsyms_cb, obj); in bpf_object__read_kallsyms_file()
8201 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, in find_ksym_btf_id() argument
8209 btf = obj->btf_vmlinux; in find_ksym_btf_id()
8214 err = load_module_btfs(obj); in find_ksym_btf_id()
8218 for (i = 0; i < obj->btf_module_cnt; i++) { in find_ksym_btf_id()
8220 mod_btf = &obj->btf_modules[i]; in find_ksym_btf_id()
8235 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, in bpf_object__resolve_ksym_var_btf_id() argument
8245 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf); in bpf_object__resolve_ksym_var_btf_id()
8262 err = bpf_core_types_are_compat(obj->btf, local_type_id, in bpf_object__resolve_ksym_var_btf_id()
8268 local_type = btf__type_by_id(obj->btf, local_type_id); in bpf_object__resolve_ksym_var_btf_id()
8269 local_name = btf__name_by_offset(obj->btf, local_type->name_off); in bpf_object__resolve_ksym_var_btf_id()
8288 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, in bpf_object__resolve_ksym_func_btf_id() argument
8299 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf, in bpf_object__resolve_ksym_func_btf_id()
8312 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id, in bpf_object__resolve_ksym_func_btf_id()
8327 if (obj->fd_array_cnt == INT16_MAX) { in bpf_object__resolve_ksym_func_btf_id()
8333 if (!obj->fd_array_cnt) in bpf_object__resolve_ksym_func_btf_id()
8334 obj->fd_array_cnt = 1; in bpf_object__resolve_ksym_func_btf_id()
8336 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), in bpf_object__resolve_ksym_func_btf_id()
8337 obj->fd_array_cnt + 1); in bpf_object__resolve_ksym_func_btf_id()
8340 mod_btf->fd_array_idx = obj->fd_array_cnt; in bpf_object__resolve_ksym_func_btf_id()
8342 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd; in bpf_object__resolve_ksym_func_btf_id()
8360 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj) in bpf_object__resolve_ksyms_btf_id() argument
8366 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__resolve_ksyms_btf_id()
8367 ext = &obj->externs[i]; in bpf_object__resolve_ksyms_btf_id()
8371 if (obj->gen_loader) { in bpf_object__resolve_ksyms_btf_id()
8377 t = btf__type_by_id(obj->btf, ext->btf_id); in bpf_object__resolve_ksyms_btf_id()
8379 err = bpf_object__resolve_ksym_var_btf_id(obj, ext); in bpf_object__resolve_ksyms_btf_id()
8381 err = bpf_object__resolve_ksym_func_btf_id(obj, ext); in bpf_object__resolve_ksyms_btf_id()
8388 static int bpf_object__resolve_externs(struct bpf_object *obj, in bpf_object__resolve_externs() argument
8397 if (obj->nr_extern == 0) in bpf_object__resolve_externs()
8400 if (obj->kconfig_map_idx >= 0) in bpf_object__resolve_externs()
8401 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; in bpf_object__resolve_externs()
8403 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__resolve_externs()
8404 ext = &obj->externs[i]; in bpf_object__resolve_externs()
8430 value = kernel_supports(obj, FEAT_BPF_COOKIE); in bpf_object__resolve_externs()
8432 value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER); in bpf_object__resolve_externs()
8456 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data); in bpf_object__resolve_externs()
8460 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__resolve_externs()
8461 ext = &obj->externs[i]; in bpf_object__resolve_externs()
8469 err = bpf_object__read_kconfig_file(obj, kcfg_data); in bpf_object__resolve_externs()
8474 err = bpf_object__read_kallsyms_file(obj); in bpf_object__resolve_externs()
8479 err = bpf_object__resolve_ksyms_btf_id(obj); in bpf_object__resolve_externs()
8483 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__resolve_externs()
8484 ext = &obj->externs[i]; in bpf_object__resolve_externs()
8505 type = btf__type_by_id(map->obj->btf, st_ops->type_id); in bpf_map_prepare_vdata()
8520 static int bpf_object_prepare_struct_ops(struct bpf_object *obj) in bpf_object_prepare_struct_ops() argument
8525 for (i = 0; i < obj->nr_maps; i++) { in bpf_object_prepare_struct_ops()
8526 map = &obj->maps[i]; in bpf_object_prepare_struct_ops()
8540 static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path) in bpf_object_load() argument
8544 if (!obj) in bpf_object_load()
8547 if (obj->loaded) { in bpf_object_load()
8548 pr_warn("object '%s': load can't be attempted twice\n", obj->name); in bpf_object_load()
8552 if (obj->gen_loader) in bpf_object_load()
8553 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); in bpf_object_load()
8555 err = bpf_object_prepare_token(obj); in bpf_object_load()
8556 err = err ? : bpf_object__probe_loading(obj); in bpf_object_load()
8557 err = err ? : bpf_object__load_vmlinux_btf(obj, false); in bpf_object_load()
8558 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); in bpf_object_load()
8559 err = err ? : bpf_object__sanitize_maps(obj); in bpf_object_load()
8560 err = err ? : bpf_object__init_kern_struct_ops_maps(obj); in bpf_object_load()
8561 err = err ? : bpf_object_adjust_struct_ops_autoload(obj); in bpf_object_load()
8562 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path); in bpf_object_load()
8563 err = err ? : bpf_object__sanitize_and_load_btf(obj); in bpf_object_load()
8564 err = err ? : bpf_object__create_maps(obj); in bpf_object_load()
8565 err = err ? : bpf_object__load_progs(obj, extra_log_level); in bpf_object_load()
8566 err = err ? : bpf_object_init_prog_arrays(obj); in bpf_object_load()
8567 err = err ? : bpf_object_prepare_struct_ops(obj); in bpf_object_load()
8569 if (obj->gen_loader) { in bpf_object_load()
8571 if (obj->btf) in bpf_object_load()
8572 btf__set_fd(obj->btf, -1); in bpf_object_load()
8574 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps); in bpf_object_load()
8578 zfree(&obj->fd_array); in bpf_object_load()
8581 for (i = 0; i < obj->btf_module_cnt; i++) { in bpf_object_load()
8582 close(obj->btf_modules[i].fd); in bpf_object_load()
8583 btf__free(obj->btf_modules[i].btf); in bpf_object_load()
8584 free(obj->btf_modules[i].name); in bpf_object_load()
8586 free(obj->btf_modules); in bpf_object_load()
8589 btf__free(obj->btf_vmlinux); in bpf_object_load()
8590 obj->btf_vmlinux = NULL; in bpf_object_load()
8592 obj->loaded = true; /* doesn't matter if successfully or not */ in bpf_object_load()
8600 for (i = 0; i < obj->nr_maps; i++) in bpf_object_load()
8601 if (obj->maps[i].pinned && !obj->maps[i].reused) in bpf_object_load()
8602 bpf_map__unpin(&obj->maps[i], NULL); in bpf_object_load()
8604 bpf_object_unload(obj); in bpf_object_load()
8605 pr_warn("failed to load object '%s'\n", obj->path); in bpf_object_load()
8609 int bpf_object__load(struct bpf_object *obj) in bpf_object__load() argument
8611 return bpf_object_load(obj, 0, NULL); in bpf_object__load()
8856 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) in bpf_object__pin_maps() argument
8861 if (!obj) in bpf_object__pin_maps()
8864 if (!obj->loaded) { in bpf_object__pin_maps()
8869 bpf_object__for_each_map(map, obj) { in bpf_object__pin_maps()
8894 while ((map = bpf_object__prev_map(obj, map))) { in bpf_object__pin_maps()
8904 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) in bpf_object__unpin_maps() argument
8909 if (!obj) in bpf_object__unpin_maps()
8912 bpf_object__for_each_map(map, obj) { in bpf_object__unpin_maps()
8934 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) in bpf_object__pin_programs() argument
8940 if (!obj) in bpf_object__pin_programs()
8943 if (!obj->loaded) { in bpf_object__pin_programs()
8948 bpf_object__for_each_program(prog, obj) { in bpf_object__pin_programs()
8961 while ((prog = bpf_object__prev_program(obj, prog))) { in bpf_object__pin_programs()
8971 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) in bpf_object__unpin_programs() argument
8976 if (!obj) in bpf_object__unpin_programs()
8979 bpf_object__for_each_program(prog, obj) { in bpf_object__unpin_programs()
8994 int bpf_object__pin(struct bpf_object *obj, const char *path) in bpf_object__pin() argument
8998 err = bpf_object__pin_maps(obj, path); in bpf_object__pin()
9002 err = bpf_object__pin_programs(obj, path); in bpf_object__pin()
9004 bpf_object__unpin_maps(obj, path); in bpf_object__pin()
9011 int bpf_object__unpin(struct bpf_object *obj, const char *path) in bpf_object__unpin() argument
9015 err = bpf_object__unpin_programs(obj, path); in bpf_object__unpin()
9019 err = bpf_object__unpin_maps(obj, path); in bpf_object__unpin()
9036 if (map->mmaped && map->mmaped != map->obj->arena_data) in bpf_map__destroy()
9055 void bpf_object__close(struct bpf_object *obj) in bpf_object__close() argument
9059 if (IS_ERR_OR_NULL(obj)) in bpf_object__close()
9062 usdt_manager_free(obj->usdt_man); in bpf_object__close()
9063 obj->usdt_man = NULL; in bpf_object__close()
9065 bpf_gen__free(obj->gen_loader); in bpf_object__close()
9066 bpf_object__elf_finish(obj); in bpf_object__close()
9067 bpf_object_unload(obj); in bpf_object__close()
9068 btf__free(obj->btf); in bpf_object__close()
9069 btf__free(obj->btf_vmlinux); in bpf_object__close()
9070 btf_ext__free(obj->btf_ext); in bpf_object__close()
9072 for (i = 0; i < obj->nr_maps; i++) in bpf_object__close()
9073 bpf_map__destroy(&obj->maps[i]); in bpf_object__close()
9075 zfree(&obj->btf_custom_path); in bpf_object__close()
9076 zfree(&obj->kconfig); in bpf_object__close()
9078 for (i = 0; i < obj->nr_extern; i++) { in bpf_object__close()
9079 zfree(&obj->externs[i].name); in bpf_object__close()
9080 zfree(&obj->externs[i].essent_name); in bpf_object__close()
9083 zfree(&obj->externs); in bpf_object__close()
9084 obj->nr_extern = 0; in bpf_object__close()
9086 zfree(&obj->maps); in bpf_object__close()
9087 obj->nr_maps = 0; in bpf_object__close()
9089 if (obj->programs && obj->nr_programs) { in bpf_object__close()
9090 for (i = 0; i < obj->nr_programs; i++) in bpf_object__close()
9091 bpf_program__exit(&obj->programs[i]); in bpf_object__close()
9093 zfree(&obj->programs); in bpf_object__close()
9095 zfree(&obj->feat_cache); in bpf_object__close()
9096 zfree(&obj->token_path); in bpf_object__close()
9097 if (obj->token_fd > 0) in bpf_object__close()
9098 close(obj->token_fd); in bpf_object__close()
9100 zfree(&obj->arena_data); in bpf_object__close()
9102 free(obj); in bpf_object__close()
9105 const char *bpf_object__name(const struct bpf_object *obj) in bpf_object__name() argument
9107 return obj ? obj->name : libbpf_err_ptr(-EINVAL); in bpf_object__name()
9110 unsigned int bpf_object__kversion(const struct bpf_object *obj) in bpf_object__kversion() argument
9112 return obj ? obj->kern_version : 0; in bpf_object__kversion()
9115 int bpf_object__token_fd(const struct bpf_object *obj) in bpf_object__token_fd() argument
9117 return obj->token_fd ?: -1; in bpf_object__token_fd()
9120 struct btf *bpf_object__btf(const struct bpf_object *obj) in bpf_object__btf() argument
9122 return obj ? obj->btf : NULL; in bpf_object__btf()
9125 int bpf_object__btf_fd(const struct bpf_object *obj) in bpf_object__btf_fd() argument
9127 return obj->btf ? btf__fd(obj->btf) : -1; in bpf_object__btf_fd()
9130 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version) in bpf_object__set_kversion() argument
9132 if (obj->loaded) in bpf_object__set_kversion()
9135 obj->kern_version = kern_version; in bpf_object__set_kversion()
9140 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts) in bpf_object__gen_loader() argument
9152 obj->gen_loader = gen; in bpf_object__gen_loader()
9157 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, in __bpf_program__iter() argument
9160 size_t nr_programs = obj->nr_programs; in __bpf_program__iter()
9168 return forward ? &obj->programs[0] : in __bpf_program__iter()
9169 &obj->programs[nr_programs - 1]; in __bpf_program__iter()
9171 if (p->obj != obj) { in __bpf_program__iter()
9176 idx = (p - obj->programs) + (forward ? 1 : -1); in __bpf_program__iter()
9177 if (idx >= obj->nr_programs || idx < 0) in __bpf_program__iter()
9179 return &obj->programs[idx]; in __bpf_program__iter()
9183 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev) in bpf_object__next_program() argument
9188 prog = __bpf_program__iter(prog, obj, true); in bpf_object__next_program()
9189 } while (prog && prog_is_subprog(obj, prog)); in bpf_object__next_program()
9195 bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next) in bpf_object__prev_program() argument
9200 prog = __bpf_program__iter(prog, obj, false); in bpf_object__prev_program()
9201 } while (prog && prog_is_subprog(obj, prog)); in bpf_object__prev_program()
9228 if (prog->obj->loaded) in bpf_program__set_autoload()
9260 if (prog->obj->loaded) in bpf_program__set_insns()
9303 if (prog->obj->loaded) in bpf_program__set_type()
9334 if (prog->obj->loaded) in bpf_program__set_expected_attach_type()
9348 if (prog->obj->loaded) in bpf_program__set_flags()
9362 if (prog->obj->loaded) in bpf_program__set_log_level()
9381 if (prog->obj->loaded) in bpf_program__set_log_buf()
9750 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, in find_struct_ops_map_by_offset() argument
9757 for (i = 0; i < obj->nr_maps; i++) { in find_struct_ops_map_by_offset()
9758 map = &obj->maps[i]; in find_struct_ops_map_by_offset()
9773 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, in bpf_object__collect_st_ops_relos() argument
9790 btf = obj->btf; in bpf_object__collect_st_ops_relos()
9799 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); in bpf_object__collect_st_ops_relos()
9806 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; in bpf_object__collect_st_ops_relos()
9807 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset); in bpf_object__collect_st_ops_relos()
9852 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx); in bpf_object__collect_st_ops_relos()
9992 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name, in find_kernel_btf_id() argument
10007 ret = find_attach_btf_id(obj->btf_vmlinux, in find_kernel_btf_id()
10019 ret = load_module_btfs(obj); in find_kernel_btf_id()
10023 for (i = 0; i < obj->btf_module_cnt; i++) { in find_kernel_btf_id()
10024 const struct module_btf *mod = &obj->btf_modules[i]; in find_kernel_btf_id()
10071 if (prog->obj->gen_loader) { in libbpf_find_attach_btf_id()
10072 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); in libbpf_find_attach_btf_id()
10076 err = find_kernel_btf_id(prog->obj, attach_name, in libbpf_find_attach_btf_id()
10234 btf = bpf_object__btf(map->obj); in map_btf_datasec_resize()
10296 if (map->obj->loaded || map->reused) in bpf_map__set_value_size()
10342 if (map->obj->loaded || map->reused) in bpf_map__set_initial_value()
10349 actual_sz = map->obj->arena_data_sz; in bpf_map__set_initial_value()
10371 *psize = map->obj->arena_data_sz; in bpf_map__initial_value()
10415 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) in __bpf_map__iter() argument
10420 if (!obj || !obj->maps) in __bpf_map__iter()
10423 s = obj->maps; in __bpf_map__iter()
10424 e = obj->maps + obj->nr_maps; in __bpf_map__iter()
10432 idx = (m - obj->maps) + i; in __bpf_map__iter()
10433 if (idx >= obj->nr_maps || idx < 0) in __bpf_map__iter()
10435 return &obj->maps[idx]; in __bpf_map__iter()
10439 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev) in bpf_object__next_map() argument
10441 if (prev == NULL && obj != NULL) in bpf_object__next_map()
10442 return obj->maps; in bpf_object__next_map()
10444 return __bpf_map__iter(prev, obj, 1); in bpf_object__next_map()
10448 bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next) in bpf_object__prev_map() argument
10450 if (next == NULL && obj != NULL) { in bpf_object__prev_map()
10451 if (!obj->nr_maps) in bpf_object__prev_map()
10453 return obj->maps + obj->nr_maps - 1; in bpf_object__prev_map()
10456 return __bpf_map__iter(next, obj, -1); in bpf_object__prev_map()
10460 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) in bpf_object__find_map_by_name() argument
10464 bpf_object__for_each_map(pos, obj) { in bpf_object__find_map_by_name()
10487 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) in bpf_object__find_map_fd_by_name() argument
10489 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); in bpf_object__find_map_fd_by_name()
10836 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) { in bpf_program__attach_perf_event_opts()
11239 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) in bpf_program__attach_kprobe_opts()
11322 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) { in bpf_program__attach_ksyscall()
12166 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) in bpf_program__attach_uprobe_opts()
12312 struct bpf_object *obj = prog->obj; in bpf_program__attach_usdt() local
12342 if (IS_ERR(obj->usdt_man)) in bpf_program__attach_usdt()
12343 return libbpf_ptr(obj->usdt_man); in bpf_program__attach_usdt()
12344 if (!obj->usdt_man) { in bpf_program__attach_usdt()
12345 obj->usdt_man = usdt_manager_new(obj); in bpf_program__attach_usdt()
12346 if (IS_ERR(obj->usdt_man)) in bpf_program__attach_usdt()
12347 return libbpf_ptr(obj->usdt_man); in bpf_program__attach_usdt()
12351 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path, in bpf_program__attach_usdt()
13622 if (prog->obj->loaded) in bpf_program__set_attach_target()
13643 err = bpf_object__load_vmlinux_btf(prog->obj, true); in bpf_program__set_attach_target()
13646 err = find_kernel_btf_id(prog->obj, attach_func_name, in bpf_program__set_attach_target()
13762 static int populate_skeleton_maps(const struct bpf_object *obj, in populate_skeleton_maps() argument
13774 *map = bpf_object__find_map_by_name(obj, name); in populate_skeleton_maps()
13787 static int populate_skeleton_progs(const struct bpf_object *obj, in populate_skeleton_progs() argument
13798 *prog = bpf_object__find_program_by_name(obj, name); in populate_skeleton_progs()
13810 struct bpf_object *obj; in bpf_object__open_skeleton() local
13813 obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts); in bpf_object__open_skeleton()
13814 if (IS_ERR(obj)) { in bpf_object__open_skeleton()
13815 err = PTR_ERR(obj); in bpf_object__open_skeleton()
13820 *s->obj = obj; in bpf_object__open_skeleton()
13821 err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz); in bpf_object__open_skeleton()
13827 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz); in bpf_object__open_skeleton()
13847 if (!s->obj) in bpf_object__open_subskeleton()
13850 btf = bpf_object__btf(s->obj); in bpf_object__open_subskeleton()
13853 bpf_object__name(s->obj)); in bpf_object__open_subskeleton()
13857 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz); in bpf_object__open_subskeleton()
13863 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz); in bpf_object__open_subskeleton()
13910 err = bpf_object__load(*s->obj); in bpf_object__load_skeleton()
14041 if (s->obj) in bpf_object__destroy_skeleton()
14042 bpf_object__close(*s->obj); in bpf_object__destroy_skeleton()