9834 	struct perf_buffer *pb;  member
9858 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,  in perf_buffer__free_cpu_buf()  argument
9864 	    munmap(cpu_buf->base, pb->mmap_size + pb->page_size))  in perf_buffer__free_cpu_buf()
9874 void perf_buffer__free(struct perf_buffer *pb)  in perf_buffer__free()  argument
9878 	if (IS_ERR_OR_NULL(pb))  in perf_buffer__free()
9880 	if (pb->cpu_bufs) {  in perf_buffer__free()
9881 		for (i = 0; i < pb->cpu_cnt; i++) {  in perf_buffer__free()
9882 			struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];  in perf_buffer__free()
9887 			bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);  in perf_buffer__free()
9888 			perf_buffer__free_cpu_buf(pb, cpu_buf);  in perf_buffer__free()
9890 		free(pb->cpu_bufs);  in perf_buffer__free()
9892 	if (pb->epoll_fd >= 0)  in perf_buffer__free()
9893 		close(pb->epoll_fd);  in perf_buffer__free()
9894 	free(pb->events);  in perf_buffer__free()
9895 	free(pb);  in perf_buffer__free()
9899 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,  in perf_buffer__open_cpu_buf()  argument
9910 	cpu_buf->pb = pb;  in perf_buffer__open_cpu_buf()
9923 	cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,  in perf_buffer__open_cpu_buf()
9944 	perf_buffer__free_cpu_buf(pb, cpu_buf);  in perf_buffer__open_cpu_buf()
9993 	struct perf_buffer *pb;  in __perf_buffer__new()  local
10028 	pb = calloc(1, sizeof(*pb));  in __perf_buffer__new()
10029 	if (!pb)  in __perf_buffer__new()
10032 	pb->event_cb = p->event_cb;  in __perf_buffer__new()
10033 	pb->sample_cb = p->sample_cb;  in __perf_buffer__new()
10034 	pb->lost_cb = p->lost_cb;  in __perf_buffer__new()
10035 	pb->ctx = p->ctx;  in __perf_buffer__new()
10037 	pb->page_size = getpagesize();  in __perf_buffer__new()
10038 	pb->mmap_size = pb->page_size * page_cnt;  in __perf_buffer__new()
10039 	pb->map_fd = map_fd;  in __perf_buffer__new()
10041 	pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);  in __perf_buffer__new()
10042 	if (pb->epoll_fd < 0) {  in __perf_buffer__new()
10050 		pb->cpu_cnt = p->cpu_cnt;  in __perf_buffer__new()
10052 		pb->cpu_cnt = libbpf_num_possible_cpus();  in __perf_buffer__new()
10053 		if (pb->cpu_cnt < 0) {  in __perf_buffer__new()
10054 			err = pb->cpu_cnt;  in __perf_buffer__new()
10057 		if (map.max_entries && map.max_entries < pb->cpu_cnt)  in __perf_buffer__new()
10058 			pb->cpu_cnt = map.max_entries;  in __perf_buffer__new()
10061 	pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));  in __perf_buffer__new()
10062 	if (!pb->events) {  in __perf_buffer__new()
10067 	pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));  in __perf_buffer__new()
10068 	if (!pb->cpu_bufs) {  in __perf_buffer__new()
10080 	for (i = 0, j = 0; i < pb->cpu_cnt; i++) {  in __perf_buffer__new()
10093 		cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);  in __perf_buffer__new()
10099 		pb->cpu_bufs[j] = cpu_buf;  in __perf_buffer__new()
10101 		err = bpf_map_update_elem(pb->map_fd, &map_key,  in __perf_buffer__new()
10111 		pb->events[j].events = EPOLLIN;  in __perf_buffer__new()
10112 		pb->events[j].data.ptr = cpu_buf;  in __perf_buffer__new()
10113 		if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,  in __perf_buffer__new()
10114 			      &pb->events[j]) < 0) {  in __perf_buffer__new()
10123 	pb->cpu_cnt = j;  in __perf_buffer__new()
10126 	return pb;  in __perf_buffer__new()
10130 	if (pb)  in __perf_buffer__new()
10131 		perf_buffer__free(pb);  in __perf_buffer__new()
10152 	struct perf_buffer *pb = cpu_buf->pb;  in perf_buffer__process_record()  local
10156 	if (pb->event_cb)  in perf_buffer__process_record()
10157 		return pb->event_cb(pb->ctx, cpu_buf->cpu, e);  in perf_buffer__process_record()
10163 		if (pb->sample_cb)  in perf_buffer__process_record()
10164 			pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);  in perf_buffer__process_record()
10170 		if (pb->lost_cb)  in perf_buffer__process_record()
10171 			pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);  in perf_buffer__process_record()
10181 static int perf_buffer__process_records(struct perf_buffer *pb,  in perf_buffer__process_records()  argument
10186 	ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,  in perf_buffer__process_records()
10187 					 pb->page_size, &cpu_buf->buf,  in perf_buffer__process_records()
10195 int perf_buffer__epoll_fd(const struct perf_buffer *pb)  in perf_buffer__epoll_fd()  argument
10197 	return pb->epoll_fd;  in perf_buffer__epoll_fd()
10200 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)  in perf_buffer__poll()  argument
10204 	cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);  in perf_buffer__poll()
10209 		struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;  in perf_buffer__poll()
10211 		err = perf_buffer__process_records(pb, cpu_buf);  in perf_buffer__poll()
10223 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)  in perf_buffer__buffer_cnt()  argument
10225 	return pb->cpu_cnt;  in perf_buffer__buffer_cnt()
10233 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)  in perf_buffer__buffer_fd()  argument
10237 	if (buf_idx >= pb->cpu_cnt)  in perf_buffer__buffer_fd()
10240 	cpu_buf = pb->cpu_bufs[buf_idx];  in perf_buffer__buffer_fd()
10255 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)  in perf_buffer__consume_buffer()  argument
10259 	if (buf_idx >= pb->cpu_cnt)  in perf_buffer__consume_buffer()
10262 	cpu_buf = pb->cpu_bufs[buf_idx];  in perf_buffer__consume_buffer()
10266 	return perf_buffer__process_records(pb, cpu_buf);  in perf_buffer__consume_buffer()
10269 int perf_buffer__consume(struct perf_buffer *pb)  in perf_buffer__consume()  argument
10273 	for (i = 0; i < pb->cpu_cnt; i++) {  in perf_buffer__consume()
10274 		struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];  in perf_buffer__consume()
10279 		err = perf_buffer__process_records(pb, cpu_buf);  in perf_buffer__consume()