133 	struct perf_output_handle * __percpu *handle;  member
207 static void trbe_report_wrap_event(struct perf_output_handle *handle)  in trbe_report_wrap_event()  argument
223 	perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);  in trbe_report_wrap_event()
226 static void trbe_stop_and_truncate_event(struct perf_output_handle *handle)  in trbe_stop_and_truncate_event()  argument
228 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_stop_and_truncate_event()
238 	perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);  in trbe_stop_and_truncate_event()
239 	perf_aux_output_end(handle, 0);  in trbe_stop_and_truncate_event()
240 	*this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;  in trbe_stop_and_truncate_event()
291 static void trbe_pad_buf(struct perf_output_handle *handle, int len)  in trbe_pad_buf()  argument
293 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_pad_buf()
294 	u64 head = PERF_IDX2OFF(handle->head, buf);  in trbe_pad_buf()
298 		perf_aux_output_skip(handle, len);  in trbe_pad_buf()
301 static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle)  in trbe_snapshot_offset()  argument
303 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_snapshot_offset()
313 static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle)  in trbe_min_trace_buf_size()  argument
316 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_min_trace_buf_size()
342 static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)  in __trbe_normal_offset()  argument
344 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in __trbe_normal_offset()
350 	head = PERF_IDX2OFF(handle->head, buf);  in __trbe_normal_offset()
371 		delta = min(delta, handle->size);  in __trbe_normal_offset()
372 		trbe_pad_buf(handle, delta);  in __trbe_normal_offset()
373 		head = PERF_IDX2OFF(handle->head, buf);  in __trbe_normal_offset()
385 	if (!handle->size)  in __trbe_normal_offset()
389 	tail = PERF_IDX2OFF(handle->head + handle->size, buf);  in __trbe_normal_offset()
390 	wakeup = PERF_IDX2OFF(handle->wakeup, buf);  in __trbe_normal_offset()
453 	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)  in __trbe_normal_offset()
487 	trbe_pad_buf(handle, handle->size);  in __trbe_normal_offset()
491 static unsigned long trbe_normal_offset(struct perf_output_handle *handle)  in trbe_normal_offset()  argument
493 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_normal_offset()
494 	u64 limit = __trbe_normal_offset(handle);  in trbe_normal_offset()
495 	u64 head = PERF_IDX2OFF(handle->head, buf);  in trbe_normal_offset()
505 	while (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) {  in trbe_normal_offset()
506 		trbe_pad_buf(handle, limit - head);  in trbe_normal_offset()
507 		limit = __trbe_normal_offset(handle);  in trbe_normal_offset()
508 		head = PERF_IDX2OFF(handle->head, buf);  in trbe_normal_offset()
513 static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle)  in compute_trbe_buffer_limit()  argument
515 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in compute_trbe_buffer_limit()
519 		offset = trbe_snapshot_offset(handle);  in compute_trbe_buffer_limit()
521 		offset = trbe_normal_offset(handle);  in compute_trbe_buffer_limit()
596 static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *handle,  in trbe_get_fault_act()  argument
601 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_get_fault_act()
624 static unsigned long trbe_get_trace_size(struct perf_output_handle *handle,  in trbe_get_trace_size()  argument
658 	start_off = PERF_IDX2OFF(handle->head, buf);  in trbe_get_trace_size()
730 					    struct perf_output_handle *handle,  in arm_trbe_update_buffer()  argument
792 		act = trbe_get_fault_act(handle, status);  in arm_trbe_update_buffer()
802 		trbe_report_wrap_event(handle);  in arm_trbe_update_buffer()
806 	size = trbe_get_trace_size(handle, buf, wrap);  in arm_trbe_update_buffer()
812 		handle->head += size;  in arm_trbe_update_buffer()
932 			     struct perf_output_handle *handle)  in __arm_trbe_enable()  argument
936 	perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);  in __arm_trbe_enable()
937 	buf->trbe_limit = compute_trbe_buffer_limit(handle);  in __arm_trbe_enable()
938 	buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf);  in __arm_trbe_enable()
950 	*this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;  in __arm_trbe_enable()
954 	trbe_stop_and_truncate_event(handle);  in __arm_trbe_enable()
962 	struct perf_output_handle *handle = data;  in arm_trbe_enable()  local
963 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in arm_trbe_enable()
974 	return __arm_trbe_enable(buf, handle);  in arm_trbe_enable()
996 static void trbe_handle_spurious(struct perf_output_handle *handle)  in trbe_handle_spurious()  argument
1010 static int trbe_handle_overflow(struct perf_output_handle *handle)  in trbe_handle_overflow()  argument
1012 	struct perf_event *event = handle->event;  in trbe_handle_overflow()
1013 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in trbe_handle_overflow()
1017 	size = trbe_get_trace_size(handle, buf, true);  in trbe_handle_overflow()
1019 		handle->head += size;  in trbe_handle_overflow()
1021 	trbe_report_wrap_event(handle);  in trbe_handle_overflow()
1022 	perf_aux_output_end(handle, size);  in trbe_handle_overflow()
1023 	event_data = perf_aux_output_begin(handle, event);  in trbe_handle_overflow()
1032 		*this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;  in trbe_handle_overflow()
1036 	return __arm_trbe_enable(buf, handle);  in trbe_handle_overflow()
1039 static bool is_perf_trbe(struct perf_output_handle *handle)  in is_perf_trbe()  argument
1041 	struct trbe_buf *buf = etm_perf_sink_config(handle);  in is_perf_trbe()
1064 	struct perf_output_handle *handle = *handle_ptr;  in arm_trbe_irq_handler()  local
1089 	if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle))  in arm_trbe_irq_handler()
1092 	if (!is_perf_trbe(handle))  in arm_trbe_irq_handler()
1095 	act = trbe_get_fault_act(handle, status);  in arm_trbe_irq_handler()
1098 		truncated = !!trbe_handle_overflow(handle);  in arm_trbe_irq_handler()
1101 		trbe_handle_spurious(handle);  in arm_trbe_irq_handler()
1104 		trbe_stop_and_truncate_event(handle);  in arm_trbe_irq_handler()
1402 	drvdata->handle = alloc_percpu(struct perf_output_handle *);  in arm_trbe_probe_irq()
1403 	if (!drvdata->handle)  in arm_trbe_probe_irq()
1406 	ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle);  in arm_trbe_probe_irq()
1408 		free_percpu(drvdata->handle);  in arm_trbe_probe_irq()
1416 	free_percpu_irq(drvdata->irq, drvdata->handle);  in arm_trbe_remove_irq()
1417 	free_percpu(drvdata->handle);  in arm_trbe_remove_irq()