5312 	struct perf_buffer *pb;  member
5336 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,  in perf_buffer__free_cpu_buf()  argument
5342 	    munmap(cpu_buf->base, pb->mmap_size + pb->page_size))  in perf_buffer__free_cpu_buf()
5352 void perf_buffer__free(struct perf_buffer *pb)  in perf_buffer__free()  argument
5356 	if (!pb)  in perf_buffer__free()
5358 	if (pb->cpu_bufs) {  in perf_buffer__free()
5359 		for (i = 0; i < pb->cpu_cnt && pb->cpu_bufs[i]; i++) {  in perf_buffer__free()
5360 			struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];  in perf_buffer__free()
5362 			bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);  in perf_buffer__free()
5363 			perf_buffer__free_cpu_buf(pb, cpu_buf);  in perf_buffer__free()
5365 		free(pb->cpu_bufs);  in perf_buffer__free()
5367 	if (pb->epoll_fd >= 0)  in perf_buffer__free()
5368 		close(pb->epoll_fd);  in perf_buffer__free()
5369 	free(pb->events);  in perf_buffer__free()
5370 	free(pb);  in perf_buffer__free()
5374 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,  in perf_buffer__open_cpu_buf()  argument
5385 	cpu_buf->pb = pb;  in perf_buffer__open_cpu_buf()
5398 	cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,  in perf_buffer__open_cpu_buf()
5419 	perf_buffer__free_cpu_buf(pb, cpu_buf);  in perf_buffer__open_cpu_buf()
5467 	struct perf_buffer *pb;  in __perf_buffer__new()  local
5492 	pb = calloc(1, sizeof(*pb));  in __perf_buffer__new()
5493 	if (!pb)  in __perf_buffer__new()
5496 	pb->event_cb = p->event_cb;  in __perf_buffer__new()
5497 	pb->sample_cb = p->sample_cb;  in __perf_buffer__new()
5498 	pb->lost_cb = p->lost_cb;  in __perf_buffer__new()
5499 	pb->ctx = p->ctx;  in __perf_buffer__new()
5501 	pb->page_size = getpagesize();  in __perf_buffer__new()
5502 	pb->mmap_size = pb->page_size * page_cnt;  in __perf_buffer__new()
5503 	pb->map_fd = map_fd;  in __perf_buffer__new()
5505 	pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);  in __perf_buffer__new()
5506 	if (pb->epoll_fd < 0) {  in __perf_buffer__new()
5514 		pb->cpu_cnt = p->cpu_cnt;  in __perf_buffer__new()
5516 		pb->cpu_cnt = libbpf_num_possible_cpus();  in __perf_buffer__new()
5517 		if (pb->cpu_cnt < 0) {  in __perf_buffer__new()
5518 			err = pb->cpu_cnt;  in __perf_buffer__new()
5521 		if (map.max_entries < pb->cpu_cnt)  in __perf_buffer__new()
5522 			pb->cpu_cnt = map.max_entries;  in __perf_buffer__new()
5525 	pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));  in __perf_buffer__new()
5526 	if (!pb->events) {  in __perf_buffer__new()
5531 	pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));  in __perf_buffer__new()
5532 	if (!pb->cpu_bufs) {  in __perf_buffer__new()
5538 	for (i = 0; i < pb->cpu_cnt; i++) {  in __perf_buffer__new()
5545 		cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);  in __perf_buffer__new()
5551 		pb->cpu_bufs[i] = cpu_buf;  in __perf_buffer__new()
5553 		err = bpf_map_update_elem(pb->map_fd, &map_key,  in __perf_buffer__new()
5563 		pb->events[i].events = EPOLLIN;  in __perf_buffer__new()
5564 		pb->events[i].data.ptr = cpu_buf;  in __perf_buffer__new()
5565 		if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,  in __perf_buffer__new()
5566 			      &pb->events[i]) < 0) {  in __perf_buffer__new()
5575 	return pb;  in __perf_buffer__new()
5578 	if (pb)  in __perf_buffer__new()
5579 		perf_buffer__free(pb);  in __perf_buffer__new()
5600 	struct perf_buffer *pb = cpu_buf->pb;  in perf_buffer__process_record()  local
5604 	if (pb->event_cb)  in perf_buffer__process_record()
5605 		return pb->event_cb(pb->ctx, cpu_buf->cpu, e);  in perf_buffer__process_record()
5611 		if (pb->sample_cb)  in perf_buffer__process_record()
5612 			pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);  in perf_buffer__process_record()
5618 		if (pb->lost_cb)  in perf_buffer__process_record()
5619 			pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);  in perf_buffer__process_record()
5629 static int perf_buffer__process_records(struct perf_buffer *pb,  in perf_buffer__process_records()  argument
5634 	ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,  in perf_buffer__process_records()
5635 					 pb->page_size, &cpu_buf->buf,  in perf_buffer__process_records()
5643 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)  in perf_buffer__poll()  argument
5647 	cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);  in perf_buffer__poll()
5649 		struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;  in perf_buffer__poll()
5651 		err = perf_buffer__process_records(pb, cpu_buf);  in perf_buffer__poll()