90 	struct perf_output_handle __percpu	*handle;  member
363 static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len)  in arm_spe_pmu_pad_buf()  argument
365 	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);  in arm_spe_pmu_pad_buf()
366 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_pad_buf()
370 		perf_aux_output_skip(handle, len);  in arm_spe_pmu_pad_buf()
373 static u64 arm_spe_pmu_next_snapshot_off(struct perf_output_handle *handle)  in arm_spe_pmu_next_snapshot_off()  argument
375 	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);  in arm_spe_pmu_next_snapshot_off()
376 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in arm_spe_pmu_next_snapshot_off()
377 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_next_snapshot_off()
394 		arm_spe_pmu_pad_buf(handle, limit - head);  in arm_spe_pmu_next_snapshot_off()
395 		handle->head = PERF_IDX2OFF(limit, buf);  in arm_spe_pmu_next_snapshot_off()
396 		limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head;  in arm_spe_pmu_next_snapshot_off()
402 static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle)  in __arm_spe_pmu_next_off()  argument
404 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in __arm_spe_pmu_next_off()
405 	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);  in __arm_spe_pmu_next_off()
424 	head = PERF_IDX2OFF(handle->head, buf);  in __arm_spe_pmu_next_off()
428 		delta = min(delta, handle->size);  in __arm_spe_pmu_next_off()
429 		arm_spe_pmu_pad_buf(handle, delta);  in __arm_spe_pmu_next_off()
430 		head = PERF_IDX2OFF(handle->head, buf);  in __arm_spe_pmu_next_off()
434 	if (!handle->size)  in __arm_spe_pmu_next_off()
438 	tail = PERF_IDX2OFF(handle->head + handle->size, buf);  in __arm_spe_pmu_next_off()
439 	wakeup = PERF_IDX2OFF(handle->wakeup, buf);  in __arm_spe_pmu_next_off()
459 	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)  in __arm_spe_pmu_next_off()
465 	arm_spe_pmu_pad_buf(handle, handle->size);  in __arm_spe_pmu_next_off()
467 	perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);  in __arm_spe_pmu_next_off()
468 	perf_aux_output_end(handle, 0);  in __arm_spe_pmu_next_off()
472 static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle)  in arm_spe_pmu_next_off()  argument
474 	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);  in arm_spe_pmu_next_off()
475 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in arm_spe_pmu_next_off()
476 	u64 limit = __arm_spe_pmu_next_off(handle);  in arm_spe_pmu_next_off()
477 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_next_off()
484 		arm_spe_pmu_pad_buf(handle, limit - head);  in arm_spe_pmu_next_off()
485 		limit = __arm_spe_pmu_next_off(handle);  in arm_spe_pmu_next_off()
491 static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle,  in arm_spe_perf_aux_output_begin()  argument
498 	buf = perf_aux_output_begin(handle, event);  in arm_spe_perf_aux_output_begin()
509 	limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle)  in arm_spe_perf_aux_output_begin()
510 			      : arm_spe_pmu_next_off(handle);  in arm_spe_perf_aux_output_begin()
515 	base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf);  in arm_spe_perf_aux_output_begin()
522 static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle)  in arm_spe_perf_aux_output_end()  argument
524 	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);  in arm_spe_perf_aux_output_end()
528 	size = offset - PERF_IDX2OFF(handle->head, buf);  in arm_spe_perf_aux_output_end()
531 		handle->head = offset;  in arm_spe_perf_aux_output_end()
533 	perf_aux_output_end(handle, size);  in arm_spe_perf_aux_output_end()
553 arm_spe_pmu_buf_get_fault_act(struct perf_output_handle *handle)  in arm_spe_pmu_buf_get_fault_act()  argument
579 		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED |  in arm_spe_pmu_buf_get_fault_act()
584 		perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);  in arm_spe_pmu_buf_get_fault_act()
618 	arm_spe_perf_aux_output_end(handle);  in arm_spe_pmu_buf_get_fault_act()
624 	struct perf_output_handle *handle = dev;  in arm_spe_pmu_irq_handler()  local
625 	struct perf_event *event = handle->event;  in arm_spe_pmu_irq_handler()
628 	if (!perf_get_aux(handle))  in arm_spe_pmu_irq_handler()
631 	act = arm_spe_pmu_buf_get_fault_act(handle);  in arm_spe_pmu_irq_handler()
659 		if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) {  in arm_spe_pmu_irq_handler()
660 			arm_spe_perf_aux_output_begin(handle, event);  in arm_spe_pmu_irq_handler()
745 	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);  in arm_spe_pmu_start()  local
748 	arm_spe_perf_aux_output_begin(handle, event);  in arm_spe_pmu_start()
778 	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);  in arm_spe_pmu_stop()  local
793 		if (perf_get_aux(handle)) {  in arm_spe_pmu_stop()
796 			act = arm_spe_pmu_buf_get_fault_act(handle);  in arm_spe_pmu_stop()
798 				arm_spe_perf_aux_output_end(handle);  in arm_spe_pmu_stop()
1135 				 spe_pmu->handle);  in arm_spe_pmu_dev_init()
1147 		free_percpu_irq(spe_pmu->irq, spe_pmu->handle);  in arm_spe_pmu_dev_init()
1155 	free_percpu_irq(spe_pmu->irq, spe_pmu->handle);  in arm_spe_pmu_dev_teardown()
1212 	spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle));  in arm_spe_pmu_device_probe()
1213 	if (!spe_pmu->handle)  in arm_spe_pmu_device_probe()
1236 	free_percpu(spe_pmu->handle);  in arm_spe_pmu_device_probe()
1246 	free_percpu(spe_pmu->handle);  in arm_spe_pmu_device_remove()