Lines Matching +full:alloc +full:- +full:ranges
1 // SPDX-License-Identifier: GPL-2.0
3 * bpf-loader.c
20 #include "bpf-loader.h"
21 #include "bpf-prologue.h"
22 #include "probe-event.h"
23 #include "probe-finder.h" // for MAX_PROBES
24 #include "parse-events.h"
27 #include "llvm-utils.h"
28 #include "c++/clang-c.h"
64 return ERR_PTR(-EINVAL); in bpf__prepare_load_buffer()
91 return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE); in bpf__prepare_load()
127 cleanup_perf_probe_events(&priv->pev, 1); in clear_prog_priv()
128 zfree(&priv->insns_buf); in clear_prog_priv()
129 zfree(&priv->type_mapping); in clear_prog_priv()
130 zfree(&priv->sys_name); in clear_prog_priv()
131 zfree(&priv->evt_name); in clear_prog_priv()
138 pev->uprobes = true; in prog_config__exec()
139 pev->target = strdup(value); in prog_config__exec()
140 if (!pev->target) in prog_config__exec()
141 return -ENOMEM; in prog_config__exec()
148 pev->uprobes = false; in prog_config__module()
149 pev->target = strdup(value); in prog_config__module()
150 if (!pev->target) in prog_config__module()
151 return -ENOMEM; in prog_config__module()
162 return -EINVAL; in prog_config__bool()
238 return -BPF_LOADER_ERRNO__PROGCONF_TERM; in do_prog_config()
251 return ERR_PTR(-ENOMEM); in parse_prog_config_kvpair()
276 main_str = config_str + (line - text); in parse_prog_config_kvpair()
300 return -BPF_LOADER_ERRNO__CONFIG; in parse_prog_config()
313 return -BPF_LOADER_ERRNO__CONFIG; in parse_prog_config()
327 /* Initialize per-program probing setting */ in config_bpf_program()
333 pr_debug("bpf: failed to alloc priv\n"); in config_bpf_program()
334 return -ENOMEM; in config_bpf_program()
336 pev = &priv->pev; in config_bpf_program()
347 priv->is_tp = true; in config_bpf_program()
348 priv->sys_name = strndup(main_str, s - main_str); in config_bpf_program()
349 priv->evt_name = strdup(s + 1); in config_bpf_program()
353 if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) { in config_bpf_program()
356 err = -BPF_LOADER_ERRNO__GROUP; in config_bpf_program()
358 } else if (!pev->group) in config_bpf_program()
359 pev->group = strdup(PERF_BPF_PROBE_GROUP); in config_bpf_program()
361 if (!pev->group) { in config_bpf_program()
363 err = -ENOMEM; in config_bpf_program()
367 if (!pev->event) { in config_bpf_program()
370 err = -BPF_LOADER_ERRNO__EVENTNAME; in config_bpf_program()
424 if (IS_ERR(priv) || !priv || priv->is_tp) in preproc_gen_prologue()
427 pev = &priv->pev; in preproc_gen_prologue()
429 if (n < 0 || n >= priv->nr_types) in preproc_gen_prologue()
433 for (i = 0; i < pev->ntevs; i++) { in preproc_gen_prologue()
434 if (priv->type_mapping[i] == n) in preproc_gen_prologue()
438 if (i >= pev->ntevs) { in preproc_gen_prologue()
440 return -BPF_LOADER_ERRNO__PROLOGUE; in preproc_gen_prologue()
443 tev = &pev->tevs[i]; in preproc_gen_prologue()
445 buf = priv->insns_buf; in preproc_gen_prologue()
446 err = bpf__gen_prologue(tev->args, tev->nargs, in preproc_gen_prologue()
448 BPF_MAXINSNS - orig_insns_cnt); in preproc_gen_prologue()
461 res->new_insn_ptr = buf; in preproc_gen_prologue()
462 res->new_insn_cnt = prologue_cnt + orig_insns_cnt; in preproc_gen_prologue()
463 res->pfd = NULL; in preproc_gen_prologue()
468 return -BPF_LOADER_ERRNO__PROLOGUE; in preproc_gen_prologue()
483 ret = tev2->nargs - tev1->nargs; in compare_tev_args()
487 for (i = 0; i < tev1->nargs; i++) { in compare_tev_args()
491 arg1 = &tev1->args[i]; in compare_tev_args()
492 arg2 = &tev2->args[i]; in compare_tev_args()
494 ret = strcmp(arg1->value, arg2->value); in compare_tev_args()
498 ref1 = arg1->ref; in compare_tev_args()
499 ref2 = arg2->ref; in compare_tev_args()
502 ret = ref2->offset - ref1->offset; in compare_tev_args()
506 ref1 = ref1->next; in compare_tev_args()
507 ref2 = ref2->next; in compare_tev_args()
511 return ref2 ? 1 : -1; in compare_tev_args()
528 size_t array_sz = sizeof(*ptevs) * pev->ntevs; in map_prologue()
532 pr_debug("Not enough memory: alloc ptevs failed\n"); in map_prologue()
533 return -ENOMEM; in map_prologue()
536 pr_debug("In map_prologue, ntevs=%d\n", pev->ntevs); in map_prologue()
537 for (i = 0; i < pev->ntevs; i++) in map_prologue()
538 ptevs[i] = &pev->tevs[i]; in map_prologue()
540 qsort(ptevs, pev->ntevs, sizeof(*ptevs), in map_prologue()
543 for (i = 0; i < pev->ntevs; i++) { in map_prologue()
546 n = ptevs[i] - pev->tevs; in map_prologue()
553 if (compare_tev_args(ptevs + i, ptevs + i - 1) == 0) in map_prologue()
575 return -BPF_LOADER_ERRNO__INTERNAL; in hook_load_preprocessor()
578 if (priv->is_tp) { in hook_load_preprocessor()
579 priv->need_prologue = false; in hook_load_preprocessor()
583 pev = &priv->pev; in hook_load_preprocessor()
584 for (i = 0; i < pev->ntevs; i++) { in hook_load_preprocessor()
585 struct probe_trace_event *tev = &pev->tevs[i]; in hook_load_preprocessor()
587 if (tev->nargs > 0) { in hook_load_preprocessor()
598 priv->need_prologue = false; in hook_load_preprocessor()
602 priv->need_prologue = true; in hook_load_preprocessor()
603 priv->insns_buf = malloc(sizeof(struct bpf_insn) * BPF_MAXINSNS); in hook_load_preprocessor()
604 if (!priv->insns_buf) { in hook_load_preprocessor()
605 pr_debug("Not enough memory: alloc insns_buf failed\n"); in hook_load_preprocessor()
606 return -ENOMEM; in hook_load_preprocessor()
609 priv->type_mapping = malloc(sizeof(int) * pev->ntevs); in hook_load_preprocessor()
610 if (!priv->type_mapping) { in hook_load_preprocessor()
611 pr_debug("Not enough memory: alloc type_mapping failed\n"); in hook_load_preprocessor()
612 return -ENOMEM; in hook_load_preprocessor()
614 memset(priv->type_mapping, -1, in hook_load_preprocessor()
615 sizeof(int) * pev->ntevs); in hook_load_preprocessor()
617 err = map_prologue(pev, priv->type_mapping, &priv->nr_types); in hook_load_preprocessor()
621 err = bpf_program__set_prep(prog, priv->nr_types, in hook_load_preprocessor()
650 if (priv->is_tp) { in bpf__probe()
656 pev = &priv->pev; in bpf__probe()
674 * hook_load_preprocessorr() hooks pre-processor in bpf__probe()
696 if (IS_ERR(priv) || !priv || priv->is_tp) in bpf__unprobe()
699 for (i = 0; i < priv->pev.ntevs; i++) { in bpf__unprobe()
700 struct probe_trace_event *tev = &priv->pev.tevs[i]; in bpf__unprobe()
705 "%s:%s", tev->group, tev->event); in bpf__unprobe()
706 name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0'; in bpf__unprobe()
711 ret = -ENOMEM; in bpf__unprobe()
756 return -BPF_LOADER_ERRNO__INTERNAL; in bpf__foreach_event()
759 if (priv->is_tp) { in bpf__foreach_event()
761 err = (*func)(priv->sys_name, priv->evt_name, fd, obj, arg); in bpf__foreach_event()
769 pev = &priv->pev; in bpf__foreach_event()
770 for (i = 0; i < pev->ntevs; i++) { in bpf__foreach_event()
771 tev = &pev->tevs[i]; in bpf__foreach_event()
773 if (priv->need_prologue) { in bpf__foreach_event()
774 int type = priv->type_mapping[i]; in bpf__foreach_event()
786 err = (*func)(tev->group, tev->event, fd, obj, arg); in bpf__foreach_event()
826 if (!list_empty(&op->list)) in bpf_map_op__delete()
827 list_del_init(&op->list); in bpf_map_op__delete()
828 if (op->key_type == BPF_MAP_KEY_RANGES) in bpf_map_op__delete()
829 parse_events__clear_array(&op->k.array); in bpf_map_op__delete()
838 list_for_each_entry_safe(pos, n, &priv->ops_list, list) { in bpf_map_priv__purge()
839 list_del_init(&pos->list); in bpf_map_priv__purge()
857 op->key_type = BPF_MAP_KEY_ALL; in bpf_map_op_setkey()
861 if (term->array.nr_ranges) { in bpf_map_op_setkey()
862 size_t memsz = term->array.nr_ranges * in bpf_map_op_setkey()
863 sizeof(op->k.array.ranges[0]); in bpf_map_op_setkey()
865 op->k.array.ranges = memdup(term->array.ranges, memsz); in bpf_map_op_setkey()
866 if (!op->k.array.ranges) { in bpf_map_op_setkey()
867 pr_debug("Not enough memory to alloc indices for map\n"); in bpf_map_op_setkey()
868 return -ENOMEM; in bpf_map_op_setkey()
870 op->key_type = BPF_MAP_KEY_RANGES; in bpf_map_op_setkey()
871 op->k.array.nr_ranges = term->array.nr_ranges; in bpf_map_op_setkey()
884 pr_debug("Failed to alloc bpf_map_op\n"); in bpf_map_op__new()
885 return ERR_PTR(-ENOMEM); in bpf_map_op__new()
887 INIT_LIST_HEAD(&op->list); in bpf_map_op__new()
904 pr_debug("Failed to alloc bpf_map_op\n"); in bpf_map_op__clone()
908 INIT_LIST_HEAD(&newop->list); in bpf_map_op__clone()
909 if (op->key_type == BPF_MAP_KEY_RANGES) { in bpf_map_op__clone()
910 size_t memsz = op->k.array.nr_ranges * in bpf_map_op__clone()
911 sizeof(op->k.array.ranges[0]); in bpf_map_op__clone()
913 newop->k.array.ranges = memdup(op->k.array.ranges, memsz); in bpf_map_op__clone()
914 if (!newop->k.array.ranges) { in bpf_map_op__clone()
915 pr_debug("Failed to alloc indices for map\n"); in bpf_map_op__clone()
932 pr_debug("Not enough memory to alloc map private\n"); in bpf_map_priv__clone()
935 INIT_LIST_HEAD(&newpriv->ops_list); in bpf_map_priv__clone()
937 list_for_each_entry(pos, &priv->ops_list, list) { in bpf_map_priv__clone()
943 list_add_tail(&newop->list, &newpriv->ops_list); in bpf_map_priv__clone()
963 pr_debug("Not enough memory to alloc map private\n"); in bpf_map__add_op()
964 return -ENOMEM; in bpf_map__add_op()
966 INIT_LIST_HEAD(&priv->ops_list); in bpf_map__add_op()
970 return -BPF_LOADER_ERRNO__INTERNAL; in bpf_map__add_op()
974 list_add_tail(&op->list, &priv->ops_list); in bpf_map__add_op()
1007 return -BPF_LOADER_ERRNO__INTERNAL; in __bpf_map__config_value()
1010 if (def->type != BPF_MAP_TYPE_ARRAY) { in __bpf_map__config_value()
1013 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE; in __bpf_map__config_value()
1015 if (def->key_size < sizeof(unsigned int)) { in __bpf_map__config_value()
1017 return -BPF_LOADER_ERRNO__OBJCONF_MAP_KEYSIZE; in __bpf_map__config_value()
1019 switch (def->value_size) { in __bpf_map__config_value()
1027 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE; in __bpf_map__config_value()
1033 op->op_type = BPF_MAP_OP_SET_VALUE; in __bpf_map__config_value()
1034 op->v.value = term->val.num; in __bpf_map__config_value()
1043 if (!term->err_val) { in bpf_map__config_value()
1045 return -BPF_LOADER_ERRNO__OBJCONF_CONF; in bpf_map__config_value()
1048 if (term->type_val != PARSE_EVENTS__TERM_TYPE_NUM) { in bpf_map__config_value()
1050 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE; in bpf_map__config_value()
1066 evsel = perf_evlist__find_evsel_by_str(evlist, term->val.str); in __bpf_map__config_event()
1069 map_name, term->val.str); in __bpf_map__config_event()
1070 return -BPF_LOADER_ERRNO__OBJCONF_MAP_NOEVT; in __bpf_map__config_event()
1084 if (def->type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { in __bpf_map__config_event()
1087 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE; in __bpf_map__config_event()
1093 op->op_type = BPF_MAP_OP_SET_EVSEL; in __bpf_map__config_event()
1094 op->v.evsel = evsel; in __bpf_map__config_event()
1103 if (!term->err_val) { in bpf_map__config_event()
1105 return -BPF_LOADER_ERRNO__OBJCONF_CONF; in bpf_map__config_event()
1108 if (term->type_val != PARSE_EVENTS__TERM_TYPE_STR) { in bpf_map__config_event()
1110 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE; in bpf_map__config_event()
1132 struct parse_events_array *array = &term->array; in config_map_indices_range_check()
1136 if (!array->nr_ranges) in config_map_indices_range_check()
1138 if (!array->ranges) { in config_map_indices_range_check()
1139 pr_debug("ERROR: map %s: array->nr_ranges is %d but range array is NULL\n", in config_map_indices_range_check()
1140 map_name, (int)array->nr_ranges); in config_map_indices_range_check()
1141 return -BPF_LOADER_ERRNO__INTERNAL; in config_map_indices_range_check()
1148 return -BPF_LOADER_ERRNO__INTERNAL; in config_map_indices_range_check()
1151 for (i = 0; i < array->nr_ranges; i++) { in config_map_indices_range_check()
1152 unsigned int start = array->ranges[i].start; in config_map_indices_range_check()
1153 size_t length = array->ranges[i].length; in config_map_indices_range_check()
1154 unsigned int idx = start + length - 1; in config_map_indices_range_check()
1156 if (idx >= def->max_entries) { in config_map_indices_range_check()
1158 return -BPF_LOADER_ERRNO__OBJCONF_MAP_IDX2BIG; in config_map_indices_range_check()
1171 char *map_name = strdup(term->config + sizeof("map:") - 1); in bpf__obj_config_map()
1173 int err = -BPF_LOADER_ERRNO__OBJCONF_OPT; in bpf__obj_config_map()
1178 return -ENOMEM; in bpf__obj_config_map()
1188 pr_debug("ERROR: Invalid map option: %s\n", term->config); in bpf__obj_config_map()
1195 err = -BPF_LOADER_ERRNO__OBJCONF_MAP_NOTEXIST; in bpf__obj_config_map()
1203 *key_scan_pos -= strlen(map_opt); in bpf__obj_config_map()
1209 if (strcmp(map_opt, func->config_opt) == 0) { in bpf__obj_config_map()
1210 err = func->config_func(map, term, evlist); in bpf__obj_config_map()
1216 err = -BPF_LOADER_ERRNO__OBJCONF_MAP_OPT; in bpf__obj_config_map()
1233 if (!obj || !term || !term->config) in bpf__config_obj()
1234 return -EINVAL; in bpf__config_obj()
1236 if (strstarts(term->config, "map:")) { in bpf__config_obj()
1237 key_scan_pos = sizeof("map:") - 1; in bpf__config_obj()
1241 err = -BPF_LOADER_ERRNO__OBJCONF_OPT; in bpf__config_obj()
1263 for (i = 0; i < pdef->max_entries; i++) { in foreach_key_array_all()
1283 for (i = 0; i < op->k.array.nr_ranges; i++) { in foreach_key_array_ranges()
1284 unsigned int start = op->k.array.ranges[i].start; in foreach_key_array_ranges()
1285 size_t length = op->k.array.ranges[i].length; in foreach_key_array_ranges()
1314 return -BPF_LOADER_ERRNO__INTERNAL; in bpf_map_config_foreach_key()
1316 if (!priv || list_empty(&priv->ops_list)) { in bpf_map_config_foreach_key()
1324 return -BPF_LOADER_ERRNO__INTERNAL; in bpf_map_config_foreach_key()
1332 list_for_each_entry(op, &priv->ops_list, list) { in bpf_map_config_foreach_key()
1333 switch (def->type) { in bpf_map_config_foreach_key()
1336 switch (op->key_type) { in bpf_map_config_foreach_key()
1349 return -BPF_LOADER_ERRNO__INTERNAL; in bpf_map_config_foreach_key()
1356 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE; in bpf_map_config_foreach_key()
1391 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE; in apply_config_value_for_key()
1394 err = -errno; in apply_config_value_for_key()
1402 struct xyarray *xy = evsel->core.fd; in apply_config_evsel_for_key()
1411 return -BPF_LOADER_ERRNO__INTERNAL; in apply_config_evsel_for_key()
1414 if (xy->row_size / xy->entry_size != 1) { in apply_config_evsel_for_key()
1417 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM; in apply_config_evsel_for_key()
1420 attr = &evsel->core.attr; in apply_config_evsel_for_key()
1421 if (attr->inherit) { in apply_config_evsel_for_key()
1423 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH; in apply_config_evsel_for_key()
1428 if (attr->type == PERF_TYPE_RAW) in apply_config_evsel_for_key()
1430 if (attr->type == PERF_TYPE_HARDWARE) in apply_config_evsel_for_key()
1434 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE; in apply_config_evsel_for_key()
1437 events = xy->entries / (xy->row_size / xy->entry_size); in apply_config_evsel_for_key()
1442 return -BPF_LOADER_ERRNO__OBJCONF_MAP_MAPSIZE; in apply_config_evsel_for_key()
1447 err = -errno; in apply_config_evsel_for_key()
1459 switch (op->op_type) { in apply_obj_config_map_for_key()
1462 pdef->value_size, in apply_obj_config_map_for_key()
1463 op->v.value); in apply_obj_config_map_for_key()
1467 op->v.evsel); in apply_obj_config_map_for_key()
1471 err = -BPF_LOADER_ERRNO__INTERNAL; in apply_obj_config_map_for_key()
1535 return ERR_PTR(-BPF_LOADER_ERRNO__INTERNAL); in bpf__setup_output_event()
1553 if (asprintf(&event_definition, "bpf-output/no-inherit=1,name=%s/", name) < 0) in bpf__setup_output_event()
1554 return ERR_PTR(-ENOMEM); in bpf__setup_output_event()
1560 pr_debug("ERROR: failed to create the \"%s\" bpf-output event\n", name); in bpf__setup_output_event()
1561 return ERR_PTR(-err); in bpf__setup_output_event()
1571 return ERR_PTR(-BPF_LOADER_ERRNO__INTERNAL); in bpf__setup_output_event()
1578 return ERR_PTR(-ENOMEM); in bpf__setup_output_event()
1591 op->op_type = BPF_MAP_OP_SET_EVSEL; in bpf__setup_output_event()
1592 op->v.evsel = evsel; in bpf__setup_output_event()
1605 #define ERRNO_OFFSET(e) ((e) - __BPF_LOADER_ERRNO__START)
1607 #define NR_ERRNO (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
1642 return -1; in bpf_loader_strerror()
1644 err = err > 0 ? err : -err; in bpf_loader_strerror()
1652 buf[size - 1] = '\0'; in bpf_loader_strerror()
1662 buf[size - 1] = '\0'; in bpf_loader_strerror()
1663 return -1; in bpf_loader_strerror()
1671 err = -err;\
1687 buf[size - 1] = '\0';
1698 buf[size - 1] = '\0'; in bpf__strerror_prepare_load()
1702 size -= n; in bpf__strerror_prepare_load()
1705 buf[size - 1] = '\0'; in bpf__strerror_prepare_load()
1714 scnprintf(buf, size, "%s (add -v to see detail)", emsg); in bpf__strerror_probe()
1717 bpf__strerror_entry(EEXIST, "Probe point exist. Try 'perf probe -d \"*\"' and set 'force=yes'"); in bpf__strerror_probe()
1770 "Cannot set event to BPF map in multi-thread tracing"); in bpf__strerror_apply_obj_config()
1772 "%s (Hint: use -i to turn off inherit)", emsg); in bpf__strerror_apply_obj_config()