Lines Matching refs:iter
1907 static int wait_on_pipe(struct trace_iterator *iter, int full) in wait_on_pipe() argument
1912 if (trace_buffer_iter(iter, iter->cpu_file)) in wait_on_pipe()
1915 ret = ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, full); in wait_on_pipe()
1922 if (iter->snapshot) in wait_on_pipe()
1923 iter->array_buffer = &iter->tr->max_buffer; in wait_on_pipe()
2869 struct trace_iterator *iter = tracepoint_print_iter; in output_printk() local
2872 if (WARN_ON_ONCE(!iter)) in output_printk()
2889 trace_seq_init(&iter->seq); in output_printk()
2890 iter->ent = fbuffer->entry; in output_printk()
2891 event_call->event.funcs->trace(iter, 0, event); in output_printk()
2892 trace_seq_putc(&iter->seq, 0); in output_printk()
2893 printk("%s", iter->seq.buffer); in output_printk()
3579 static void trace_iterator_increment(struct trace_iterator *iter) in trace_iterator_increment() argument
3581 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); in trace_iterator_increment()
3583 iter->idx++; in trace_iterator_increment()
3589 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, in peek_next_entry() argument
3593 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu); in peek_next_entry()
3601 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts, in peek_next_entry()
3606 iter->ent_size = ring_buffer_event_length(event); in peek_next_entry()
3609 iter->ent_size = 0; in peek_next_entry()
3614 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, in __find_next_entry() argument
3617 struct trace_buffer *buffer = iter->array_buffer->buffer; in __find_next_entry()
3620 int cpu_file = iter->cpu_file; in __find_next_entry()
3633 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events); in __find_next_entry()
3645 ent = peek_next_entry(iter, cpu, &ts, &lost_events); in __find_next_entry()
3655 next_size = iter->ent_size; in __find_next_entry()
3659 iter->ent_size = next_size; in __find_next_entry()
3676 static char *trace_iter_expand_format(struct trace_iterator *iter) in trace_iter_expand_format() argument
3684 if (!iter->tr || iter->fmt == static_fmt_buf) in trace_iter_expand_format()
3687 tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE, in trace_iter_expand_format()
3690 iter->fmt_size += STATIC_FMT_BUF_SIZE; in trace_iter_expand_format()
3691 iter->fmt = tmp; in trace_iter_expand_format()
3698 static bool trace_safe_str(struct trace_iterator *iter, const char *str, in trace_safe_str() argument
3710 if ((addr >= (unsigned long)iter->ent) && in trace_safe_str()
3711 (addr < (unsigned long)iter->ent + iter->ent_size)) in trace_safe_str()
3715 if ((addr >= (unsigned long)iter->tmp_seq.buffer) && in trace_safe_str()
3716 (addr < (unsigned long)iter->tmp_seq.buffer + PAGE_SIZE)) in trace_safe_str()
3730 if (!iter->ent) in trace_safe_str()
3733 trace_event = ftrace_find_event(iter->ent->type); in trace_safe_str()
3802 void trace_check_vprintf(struct trace_iterator *iter, const char *fmt, in trace_check_vprintf() argument
3816 if (iter->fmt == static_fmt_buf) in trace_check_vprintf()
3827 if (i + 1 >= iter->fmt_size) { in trace_check_vprintf()
3832 if (!trace_iter_expand_format(iter)) in trace_check_vprintf()
3863 strncpy(iter->fmt, p, i); in trace_check_vprintf()
3864 iter->fmt[i] = '\0'; in trace_check_vprintf()
3865 trace_seq_vprintf(&iter->seq, iter->fmt, ap); in trace_check_vprintf()
3874 if (iter->seq.full) { in trace_check_vprintf()
3894 if (WARN_ONCE(!trace_safe_str(iter, str, star, len), in trace_check_vprintf()
3896 fmt, show_buffer(&iter->seq))) { in trace_check_vprintf()
3901 if (len + 1 > iter->fmt_size) in trace_check_vprintf()
3902 len = iter->fmt_size - 1; in trace_check_vprintf()
3905 ret = copy_from_kernel_nofault(iter->fmt, str, len); in trace_check_vprintf()
3906 iter->fmt[len] = 0; in trace_check_vprintf()
3909 ret = strncpy_from_kernel_nofault(iter->fmt, str, in trace_check_vprintf()
3910 iter->fmt_size); in trace_check_vprintf()
3913 trace_seq_printf(&iter->seq, "(0x%px)", str); in trace_check_vprintf()
3915 trace_seq_printf(&iter->seq, "(0x%px:%s)", in trace_check_vprintf()
3916 str, iter->fmt); in trace_check_vprintf()
3918 strcpy(iter->fmt, "%s"); in trace_check_vprintf()
3920 strncpy(iter->fmt, p + i, j + 1); in trace_check_vprintf()
3921 iter->fmt[j+1] = '\0'; in trace_check_vprintf()
3924 trace_seq_printf(&iter->seq, iter->fmt, len, str); in trace_check_vprintf()
3926 trace_seq_printf(&iter->seq, iter->fmt, str); in trace_check_vprintf()
3932 trace_seq_vprintf(&iter->seq, p, ap); in trace_check_vprintf()
3935 const char *trace_event_format(struct trace_iterator *iter, const char *fmt) in trace_event_format() argument
3943 if (!iter->tr || iter->tr->trace_flags & TRACE_ITER_HASH_PTR) in trace_event_format()
3947 new_fmt = q = iter->fmt; in trace_event_format()
3949 if (unlikely(q - new_fmt + 3 > iter->fmt_size)) { in trace_event_format()
3950 if (!trace_iter_expand_format(iter)) in trace_event_format()
3953 q += iter->fmt - new_fmt; in trace_event_format()
3954 new_fmt = iter->fmt; in trace_event_format()
3978 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, in trace_find_next_entry() argument
3982 int ent_size = iter->ent_size; in trace_find_next_entry()
3993 if (iter->temp == static_temp_buf && in trace_find_next_entry()
4002 if (iter->ent && iter->ent != iter->temp) { in trace_find_next_entry()
4003 if ((!iter->temp || iter->temp_size < iter->ent_size) && in trace_find_next_entry()
4004 !WARN_ON_ONCE(iter->temp == static_temp_buf)) { in trace_find_next_entry()
4006 temp = kmalloc(iter->ent_size, GFP_KERNEL); in trace_find_next_entry()
4009 kfree(iter->temp); in trace_find_next_entry()
4010 iter->temp = temp; in trace_find_next_entry()
4011 iter->temp_size = iter->ent_size; in trace_find_next_entry()
4013 memcpy(iter->temp, iter->ent, iter->ent_size); in trace_find_next_entry()
4014 iter->ent = iter->temp; in trace_find_next_entry()
4016 entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts); in trace_find_next_entry()
4018 iter->ent_size = ent_size; in trace_find_next_entry()
4024 void *trace_find_next_entry_inc(struct trace_iterator *iter) in trace_find_next_entry_inc() argument
4026 iter->ent = __find_next_entry(iter, &iter->cpu, in trace_find_next_entry_inc()
4027 &iter->lost_events, &iter->ts); in trace_find_next_entry_inc()
4029 if (iter->ent) in trace_find_next_entry_inc()
4030 trace_iterator_increment(iter); in trace_find_next_entry_inc()
4032 return iter->ent ? iter : NULL; in trace_find_next_entry_inc()
4035 static void trace_consume(struct trace_iterator *iter) in trace_consume() argument
4037 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts, in trace_consume()
4038 &iter->lost_events); in trace_consume()
4043 struct trace_iterator *iter = m->private; in s_next() local
4047 WARN_ON_ONCE(iter->leftover); in s_next()
4052 if (iter->idx > i) in s_next()
4055 if (iter->idx < 0) in s_next()
4056 ent = trace_find_next_entry_inc(iter); in s_next()
4058 ent = iter; in s_next()
4060 while (ent && iter->idx < i) in s_next()
4061 ent = trace_find_next_entry_inc(iter); in s_next()
4063 iter->pos = *pos; in s_next()
4068 void tracing_iter_reset(struct trace_iterator *iter, int cpu) in tracing_iter_reset() argument
4074 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0; in tracing_iter_reset()
4076 buf_iter = trace_buffer_iter(iter, cpu); in tracing_iter_reset()
4088 if (ts >= iter->array_buffer->time_start) in tracing_iter_reset()
4094 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries; in tracing_iter_reset()
4103 struct trace_iterator *iter = m->private; in s_start() local
4104 struct trace_array *tr = iter->tr; in s_start()
4105 int cpu_file = iter->cpu_file; in s_start()
4117 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name)) { in s_start()
4119 if (iter->trace->close) in s_start()
4120 iter->trace->close(iter); in s_start()
4121 *iter->trace = *tr->current_trace; in s_start()
4123 if (iter->trace->open) in s_start()
4124 iter->trace->open(iter); in s_start()
4129 if (iter->snapshot && iter->trace->use_max_tr) in s_start()
4133 if (*pos != iter->pos) { in s_start()
4134 iter->ent = NULL; in s_start()
4135 iter->cpu = 0; in s_start()
4136 iter->idx = -1; in s_start()
4140 tracing_iter_reset(iter, cpu); in s_start()
4142 tracing_iter_reset(iter, cpu_file); in s_start()
4144 iter->leftover = 0; in s_start()
4145 for (p = iter; p && l < *pos; p = s_next(m, p, &l)) in s_start()
4153 if (iter->leftover) in s_start()
4154 p = iter; in s_start()
4168 struct trace_iterator *iter = m->private; in s_stop() local
4171 if (iter->snapshot && iter->trace->use_max_tr) in s_stop()
4175 trace_access_unlock(iter->cpu_file); in s_stop()
4297 print_trace_header(struct seq_file *m, struct trace_iterator *iter) in print_trace_header() argument
4300 struct array_buffer *buf = iter->array_buffer; in print_trace_header()
4302 struct tracer *type = iter->trace; in print_trace_header()
4341 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); in print_trace_header()
4342 trace_print_seq(m, &iter->seq); in print_trace_header()
4344 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); in print_trace_header()
4345 trace_print_seq(m, &iter->seq); in print_trace_header()
4352 static void test_cpu_buff_start(struct trace_iterator *iter) in test_cpu_buff_start() argument
4354 struct trace_seq *s = &iter->seq; in test_cpu_buff_start()
4355 struct trace_array *tr = iter->tr; in test_cpu_buff_start()
4360 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE)) in test_cpu_buff_start()
4363 if (cpumask_available(iter->started) && in test_cpu_buff_start()
4364 cpumask_test_cpu(iter->cpu, iter->started)) in test_cpu_buff_start()
4367 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries) in test_cpu_buff_start()
4370 if (cpumask_available(iter->started)) in test_cpu_buff_start()
4371 cpumask_set_cpu(iter->cpu, iter->started); in test_cpu_buff_start()
4374 if (iter->idx > 1) in test_cpu_buff_start()
4376 iter->cpu); in test_cpu_buff_start()
4379 static enum print_line_t print_trace_fmt(struct trace_iterator *iter) in print_trace_fmt() argument
4381 struct trace_array *tr = iter->tr; in print_trace_fmt()
4382 struct trace_seq *s = &iter->seq; in print_trace_fmt()
4387 entry = iter->ent; in print_trace_fmt()
4389 test_cpu_buff_start(iter); in print_trace_fmt()
4394 if (iter->iter_flags & TRACE_FILE_LAT_FMT) in print_trace_fmt()
4395 trace_print_lat_context(iter); in print_trace_fmt()
4397 trace_print_context(iter); in print_trace_fmt()
4404 return event->funcs->trace(iter, sym_flags, event); in print_trace_fmt()
4411 static enum print_line_t print_raw_fmt(struct trace_iterator *iter) in print_raw_fmt() argument
4413 struct trace_array *tr = iter->tr; in print_raw_fmt()
4414 struct trace_seq *s = &iter->seq; in print_raw_fmt()
4418 entry = iter->ent; in print_raw_fmt()
4422 entry->pid, iter->cpu, iter->ts); in print_raw_fmt()
4429 return event->funcs->raw(iter, 0, event); in print_raw_fmt()
4436 static enum print_line_t print_hex_fmt(struct trace_iterator *iter) in print_hex_fmt() argument
4438 struct trace_array *tr = iter->tr; in print_hex_fmt()
4439 struct trace_seq *s = &iter->seq; in print_hex_fmt()
4444 entry = iter->ent; in print_hex_fmt()
4448 SEQ_PUT_HEX_FIELD(s, iter->cpu); in print_hex_fmt()
4449 SEQ_PUT_HEX_FIELD(s, iter->ts); in print_hex_fmt()
4456 enum print_line_t ret = event->funcs->hex(iter, 0, event); in print_hex_fmt()
4466 static enum print_line_t print_bin_fmt(struct trace_iterator *iter) in print_bin_fmt() argument
4468 struct trace_array *tr = iter->tr; in print_bin_fmt()
4469 struct trace_seq *s = &iter->seq; in print_bin_fmt()
4473 entry = iter->ent; in print_bin_fmt()
4477 SEQ_PUT_FIELD(s, iter->cpu); in print_bin_fmt()
4478 SEQ_PUT_FIELD(s, iter->ts); in print_bin_fmt()
4484 return event ? event->funcs->binary(iter, 0, event) : in print_bin_fmt()
4488 int trace_empty(struct trace_iterator *iter) in trace_empty() argument
4494 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { in trace_empty()
4495 cpu = iter->cpu_file; in trace_empty()
4496 buf_iter = trace_buffer_iter(iter, cpu); in trace_empty()
4501 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu)) in trace_empty()
4508 buf_iter = trace_buffer_iter(iter, cpu); in trace_empty()
4513 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu)) in trace_empty()
4522 enum print_line_t print_trace_line(struct trace_iterator *iter) in print_trace_line() argument
4524 struct trace_array *tr = iter->tr; in print_trace_line()
4528 if (iter->lost_events) { in print_trace_line()
4529 if (iter->lost_events == (unsigned long)-1) in print_trace_line()
4530 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n", in print_trace_line()
4531 iter->cpu); in print_trace_line()
4533 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", in print_trace_line()
4534 iter->cpu, iter->lost_events); in print_trace_line()
4535 if (trace_seq_has_overflowed(&iter->seq)) in print_trace_line()
4539 if (iter->trace && iter->trace->print_line) { in print_trace_line()
4540 ret = iter->trace->print_line(iter); in print_trace_line()
4545 if (iter->ent->type == TRACE_BPUTS && in print_trace_line()
4548 return trace_print_bputs_msg_only(iter); in print_trace_line()
4550 if (iter->ent->type == TRACE_BPRINT && in print_trace_line()
4553 return trace_print_bprintk_msg_only(iter); in print_trace_line()
4555 if (iter->ent->type == TRACE_PRINT && in print_trace_line()
4558 return trace_print_printk_msg_only(iter); in print_trace_line()
4561 return print_bin_fmt(iter); in print_trace_line()
4564 return print_hex_fmt(iter); in print_trace_line()
4567 return print_raw_fmt(iter); in print_trace_line()
4569 return print_trace_fmt(iter); in print_trace_line()
4574 struct trace_iterator *iter = m->private; in trace_latency_header() local
4575 struct trace_array *tr = iter->tr; in trace_latency_header()
4578 if (trace_empty(iter)) in trace_latency_header()
4581 if (iter->iter_flags & TRACE_FILE_LAT_FMT) in trace_latency_header()
4582 print_trace_header(m, iter); in trace_latency_header()
4590 struct trace_iterator *iter = m->private; in trace_default_header() local
4591 struct trace_array *tr = iter->tr; in trace_default_header()
4597 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { in trace_default_header()
4599 if (trace_empty(iter)) in trace_default_header()
4601 print_trace_header(m, iter); in trace_default_header()
4607 print_func_help_header_irq(iter->array_buffer, in trace_default_header()
4610 print_func_help_header(iter->array_buffer, m, in trace_default_header()
4650 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) in print_snapshot_help() argument
4652 if (iter->tr->allocated_snapshot) in print_snapshot_help()
4658 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) in print_snapshot_help()
4665 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { } in print_snapshot_help() argument
4670 struct trace_iterator *iter = v; in s_show() local
4673 if (iter->ent == NULL) { in s_show()
4674 if (iter->tr) { in s_show()
4675 seq_printf(m, "# tracer: %s\n", iter->trace->name); in s_show()
4679 if (iter->snapshot && trace_empty(iter)) in s_show()
4680 print_snapshot_help(m, iter); in s_show()
4681 else if (iter->trace && iter->trace->print_header) in s_show()
4682 iter->trace->print_header(m); in s_show()
4686 } else if (iter->leftover) { in s_show()
4691 ret = trace_print_seq(m, &iter->seq); in s_show()
4694 iter->leftover = ret; in s_show()
4697 ret = print_trace_line(iter); in s_show()
4699 iter->seq.full = 0; in s_show()
4700 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n"); in s_show()
4702 ret = trace_print_seq(m, &iter->seq); in s_show()
4710 iter->leftover = ret; in s_show()
4738 struct trace_iterator *iter; in __tracing_open() local
4744 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter)); in __tracing_open()
4745 if (!iter) in __tracing_open()
4748 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter), in __tracing_open()
4750 if (!iter->buffer_iter) in __tracing_open()
4761 iter->temp = kmalloc(128, GFP_KERNEL); in __tracing_open()
4762 if (iter->temp) in __tracing_open()
4763 iter->temp_size = 128; in __tracing_open()
4772 iter->fmt = NULL; in __tracing_open()
4773 iter->fmt_size = 0; in __tracing_open()
4780 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL); in __tracing_open()
4781 if (!iter->trace) in __tracing_open()
4784 *iter->trace = *tr->current_trace; in __tracing_open()
4786 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) in __tracing_open()
4789 iter->tr = tr; in __tracing_open()
4794 iter->array_buffer = &tr->max_buffer; in __tracing_open()
4797 iter->array_buffer = &tr->array_buffer; in __tracing_open()
4798 iter->snapshot = snapshot; in __tracing_open()
4799 iter->pos = -1; in __tracing_open()
4800 iter->cpu_file = tracing_get_cpu(inode); in __tracing_open()
4801 mutex_init(&iter->mutex); in __tracing_open()
4804 if (iter->trace->open) in __tracing_open()
4805 iter->trace->open(iter); in __tracing_open()
4808 if (ring_buffer_overruns(iter->array_buffer->buffer)) in __tracing_open()
4809 iter->iter_flags |= TRACE_FILE_ANNOTATE; in __tracing_open()
4813 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; in __tracing_open()
4819 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE)) in __tracing_open()
4822 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { in __tracing_open()
4824 iter->buffer_iter[cpu] = in __tracing_open()
4825 ring_buffer_read_prepare(iter->array_buffer->buffer, in __tracing_open()
4830 ring_buffer_read_start(iter->buffer_iter[cpu]); in __tracing_open()
4831 tracing_iter_reset(iter, cpu); in __tracing_open()
4834 cpu = iter->cpu_file; in __tracing_open()
4835 iter->buffer_iter[cpu] = in __tracing_open()
4836 ring_buffer_read_prepare(iter->array_buffer->buffer, in __tracing_open()
4839 ring_buffer_read_start(iter->buffer_iter[cpu]); in __tracing_open()
4840 tracing_iter_reset(iter, cpu); in __tracing_open()
4845 return iter; in __tracing_open()
4849 kfree(iter->trace); in __tracing_open()
4850 kfree(iter->temp); in __tracing_open()
4851 kfree(iter->buffer_iter); in __tracing_open()
4935 struct trace_iterator *iter; in tracing_release() local
4944 iter = m->private; in tracing_release()
4948 if (iter->buffer_iter[cpu]) in tracing_release()
4949 ring_buffer_read_finish(iter->buffer_iter[cpu]); in tracing_release()
4952 if (iter->trace && iter->trace->close) in tracing_release()
4953 iter->trace->close(iter); in tracing_release()
4955 if (!iter->snapshot && tr->stop_count) in tracing_release()
4963 mutex_destroy(&iter->mutex); in tracing_release()
4964 free_cpumask_var(iter->started); in tracing_release()
4965 kfree(iter->fmt); in tracing_release()
4966 kfree(iter->temp); in tracing_release()
4967 kfree(iter->trace); in tracing_release()
4968 kfree(iter->buffer_iter); in tracing_release()
4994 struct trace_iterator *iter; in tracing_open() local
5018 iter = __tracing_open(inode, file, false); in tracing_open()
5019 if (IS_ERR(iter)) in tracing_open()
5020 ret = PTR_ERR(iter); in tracing_open()
5022 iter->iter_flags |= TRACE_FILE_LAT_FMT; in tracing_open()
6643 struct trace_iterator *iter; in tracing_open_pipe() local
6653 iter = kzalloc(sizeof(*iter), GFP_KERNEL); in tracing_open_pipe()
6654 if (!iter) { in tracing_open_pipe()
6660 trace_seq_init(&iter->seq); in tracing_open_pipe()
6661 iter->trace = tr->current_trace; in tracing_open_pipe()
6663 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { in tracing_open_pipe()
6669 cpumask_setall(iter->started); in tracing_open_pipe()
6672 iter->iter_flags |= TRACE_FILE_LAT_FMT; in tracing_open_pipe()
6676 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; in tracing_open_pipe()
6678 iter->tr = tr; in tracing_open_pipe()
6679 iter->array_buffer = &tr->array_buffer; in tracing_open_pipe()
6680 iter->cpu_file = tracing_get_cpu(inode); in tracing_open_pipe()
6681 mutex_init(&iter->mutex); in tracing_open_pipe()
6682 filp->private_data = iter; in tracing_open_pipe()
6684 if (iter->trace->pipe_open) in tracing_open_pipe()
6685 iter->trace->pipe_open(iter); in tracing_open_pipe()
6695 kfree(iter); in tracing_open_pipe()
6703 struct trace_iterator *iter = file->private_data; in tracing_release_pipe() local
6710 if (iter->trace->pipe_close) in tracing_release_pipe()
6711 iter->trace->pipe_close(iter); in tracing_release_pipe()
6715 free_cpumask_var(iter->started); in tracing_release_pipe()
6716 kfree(iter->fmt); in tracing_release_pipe()
6717 kfree(iter->temp); in tracing_release_pipe()
6718 mutex_destroy(&iter->mutex); in tracing_release_pipe()
6719 kfree(iter); in tracing_release_pipe()
6727 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table) in trace_poll() argument
6729 struct trace_array *tr = iter->tr; in trace_poll()
6732 if (trace_buffer_iter(iter, iter->cpu_file)) in trace_poll()
6741 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file, in trace_poll()
6742 filp, poll_table, iter->tr->buffer_percent); in trace_poll()
6748 struct trace_iterator *iter = filp->private_data; in tracing_poll_pipe() local
6750 return trace_poll(iter, filp, poll_table); in tracing_poll_pipe()
6756 struct trace_iterator *iter = filp->private_data; in tracing_wait_pipe() local
6759 while (trace_empty(iter)) { in tracing_wait_pipe()
6774 if (!tracer_tracing_is_on(iter->tr) && iter->pos) in tracing_wait_pipe()
6777 mutex_unlock(&iter->mutex); in tracing_wait_pipe()
6779 ret = wait_on_pipe(iter, 0); in tracing_wait_pipe()
6781 mutex_lock(&iter->mutex); in tracing_wait_pipe()
6797 struct trace_iterator *iter = filp->private_data; in tracing_read_pipe() local
6805 mutex_lock(&iter->mutex); in tracing_read_pipe()
6808 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); in tracing_read_pipe()
6812 trace_seq_init(&iter->seq); in tracing_read_pipe()
6814 if (iter->trace->read) { in tracing_read_pipe()
6815 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); in tracing_read_pipe()
6826 if (trace_empty(iter)) { in tracing_read_pipe()
6835 trace_iterator_reset(iter); in tracing_read_pipe()
6836 cpumask_clear(iter->started); in tracing_read_pipe()
6837 trace_seq_init(&iter->seq); in tracing_read_pipe()
6840 trace_access_lock(iter->cpu_file); in tracing_read_pipe()
6841 while (trace_find_next_entry_inc(iter) != NULL) { in tracing_read_pipe()
6843 int save_len = iter->seq.seq.len; in tracing_read_pipe()
6845 ret = print_trace_line(iter); in tracing_read_pipe()
6854 iter->seq.full = 0; in tracing_read_pipe()
6855 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n"); in tracing_read_pipe()
6856 trace_consume(iter); in tracing_read_pipe()
6861 iter->seq.seq.len = save_len; in tracing_read_pipe()
6865 trace_consume(iter); in tracing_read_pipe()
6867 if (trace_seq_used(&iter->seq) >= cnt) in tracing_read_pipe()
6875 WARN_ONCE(iter->seq.full, "full flag set for trace type %d", in tracing_read_pipe()
6876 iter->ent->type); in tracing_read_pipe()
6878 trace_access_unlock(iter->cpu_file); in tracing_read_pipe()
6882 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); in tracing_read_pipe()
6883 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq)) in tracing_read_pipe()
6884 trace_seq_init(&iter->seq); in tracing_read_pipe()
6894 mutex_unlock(&iter->mutex); in tracing_read_pipe()
6906 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter) in tracing_fill_pipe_page() argument
6914 save_len = iter->seq.seq.len; in tracing_fill_pipe_page()
6915 ret = print_trace_line(iter); in tracing_fill_pipe_page()
6917 if (trace_seq_has_overflowed(&iter->seq)) { in tracing_fill_pipe_page()
6918 iter->seq.seq.len = save_len; in tracing_fill_pipe_page()
6928 iter->seq.seq.len = save_len; in tracing_fill_pipe_page()
6932 count = trace_seq_used(&iter->seq) - save_len; in tracing_fill_pipe_page()
6935 iter->seq.seq.len = save_len; in tracing_fill_pipe_page()
6940 trace_consume(iter); in tracing_fill_pipe_page()
6942 if (!trace_find_next_entry_inc(iter)) { in tracing_fill_pipe_page()
6944 iter->ent = NULL; in tracing_fill_pipe_page()
6960 struct trace_iterator *iter = filp->private_data; in tracing_splice_read_pipe() local
6976 mutex_lock(&iter->mutex); in tracing_splice_read_pipe()
6978 if (iter->trace->splice_read) { in tracing_splice_read_pipe()
6979 ret = iter->trace->splice_read(iter, filp, in tracing_splice_read_pipe()
6989 if (!iter->ent && !trace_find_next_entry_inc(iter)) { in tracing_splice_read_pipe()
6995 trace_access_lock(iter->cpu_file); in tracing_splice_read_pipe()
7003 rem = tracing_fill_pipe_page(rem, iter); in tracing_splice_read_pipe()
7006 ret = trace_seq_to_buffer(&iter->seq, in tracing_splice_read_pipe()
7008 trace_seq_used(&iter->seq)); in tracing_splice_read_pipe()
7014 spd.partial[i].len = trace_seq_used(&iter->seq); in tracing_splice_read_pipe()
7016 trace_seq_init(&iter->seq); in tracing_splice_read_pipe()
7019 trace_access_unlock(iter->cpu_file); in tracing_splice_read_pipe()
7021 mutex_unlock(&iter->mutex); in tracing_splice_read_pipe()
7034 mutex_unlock(&iter->mutex); in tracing_splice_read_pipe()
7460 struct trace_iterator iter; member
7470 struct trace_iterator *iter; in tracing_snapshot_open() local
7479 iter = __tracing_open(inode, file, true); in tracing_snapshot_open()
7480 if (IS_ERR(iter)) in tracing_snapshot_open()
7481 ret = PTR_ERR(iter); in tracing_snapshot_open()
7488 iter = kzalloc(sizeof(*iter), GFP_KERNEL); in tracing_snapshot_open()
7489 if (!iter) { in tracing_snapshot_open()
7495 iter->tr = tr; in tracing_snapshot_open()
7496 iter->array_buffer = &tr->max_buffer; in tracing_snapshot_open()
7497 iter->cpu_file = tracing_get_cpu(inode); in tracing_snapshot_open()
7498 m->private = iter; in tracing_snapshot_open()
7518 struct trace_iterator *iter = m->private; in tracing_snapshot_write() local
7519 struct trace_array *tr = iter->tr; in tracing_snapshot_write()
7549 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { in tracing_snapshot_write()
7559 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { in tracing_snapshot_write()
7566 &tr->array_buffer, iter->cpu_file); in tracing_snapshot_write()
7572 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { in tracing_snapshot_write()
7577 smp_call_function_single(iter->cpu_file, tracing_swap_cpu_buffer, in tracing_snapshot_write()
7583 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) in tracing_snapshot_write()
7586 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file); in tracing_snapshot_write()
7637 if (info->iter.trace->use_max_tr) { in snapshot_raw_open()
7642 info->iter.snapshot = true; in snapshot_raw_open()
7643 info->iter.array_buffer = &info->iter.tr->max_buffer; in snapshot_raw_open()
8131 info->iter.tr = tr; in tracing_buffers_open()
8132 info->iter.cpu_file = tracing_get_cpu(inode); in tracing_buffers_open()
8133 info->iter.trace = tr->current_trace; in tracing_buffers_open()
8134 info->iter.array_buffer = &tr->array_buffer; in tracing_buffers_open()
8156 struct trace_iterator *iter = &info->iter; in tracing_buffers_poll() local
8158 return trace_poll(iter, filp, poll_table); in tracing_buffers_poll()
8166 struct trace_iterator *iter = &info->iter; in tracing_buffers_read() local
8174 if (iter->snapshot && iter->tr->current_trace->use_max_tr) in tracing_buffers_read()
8179 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer, in tracing_buffers_read()
8180 iter->cpu_file); in tracing_buffers_read()
8185 info->spare_cpu = iter->cpu_file; in tracing_buffers_read()
8196 trace_access_lock(iter->cpu_file); in tracing_buffers_read()
8197 ret = ring_buffer_read_page(iter->array_buffer->buffer, in tracing_buffers_read()
8200 iter->cpu_file, 0); in tracing_buffers_read()
8201 trace_access_unlock(iter->cpu_file); in tracing_buffers_read()
8204 if (trace_empty(iter)) { in tracing_buffers_read()
8208 ret = wait_on_pipe(iter, 0); in tracing_buffers_read()
8238 struct trace_iterator *iter = &info->iter; in tracing_buffers_release() local
8242 iter->tr->trace_ref--; in tracing_buffers_release()
8244 __trace_array_put(iter->tr); in tracing_buffers_release()
8246 iter->wait_index++; in tracing_buffers_release()
8250 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); in tracing_buffers_release()
8253 ring_buffer_free_read_page(iter->array_buffer->buffer, in tracing_buffers_release()
8323 struct trace_iterator *iter = &info->iter; in tracing_buffers_splice_read() local
8338 if (iter->snapshot && iter->tr->current_trace->use_max_tr) in tracing_buffers_splice_read()
8355 trace_access_lock(iter->cpu_file); in tracing_buffers_splice_read()
8356 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); in tracing_buffers_splice_read()
8369 ref->buffer = iter->array_buffer->buffer; in tracing_buffers_splice_read()
8370 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); in tracing_buffers_splice_read()
8377 ref->cpu = iter->cpu_file; in tracing_buffers_splice_read()
8380 len, iter->cpu_file, 1); in tracing_buffers_splice_read()
8397 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); in tracing_buffers_splice_read()
8400 trace_access_unlock(iter->cpu_file); in tracing_buffers_splice_read()
8414 wait_index = READ_ONCE(iter->wait_index); in tracing_buffers_splice_read()
8416 ret = wait_on_pipe(iter, iter->snapshot ? 0 : iter->tr->buffer_percent); in tracing_buffers_splice_read()
8421 if (!tracer_tracing_is_on(iter->tr)) in tracing_buffers_splice_read()
8426 if (wait_index != iter->wait_index) in tracing_buffers_splice_read()
8443 struct trace_iterator *iter = &info->iter; in tracing_buffers_ioctl() local
8450 iter->wait_index++; in tracing_buffers_ioctl()
8454 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); in tracing_buffers_ioctl()
10041 void trace_init_global_iter(struct trace_iterator *iter) in trace_init_global_iter() argument
10043 iter->tr = &global_trace; in trace_init_global_iter()
10044 iter->trace = iter->tr->current_trace; in trace_init_global_iter()
10045 iter->cpu_file = RING_BUFFER_ALL_CPUS; in trace_init_global_iter()
10046 iter->array_buffer = &global_trace.array_buffer; in trace_init_global_iter()
10048 if (iter->trace && iter->trace->open) in trace_init_global_iter()
10049 iter->trace->open(iter); in trace_init_global_iter()
10052 if (ring_buffer_overruns(iter->array_buffer->buffer)) in trace_init_global_iter()
10053 iter->iter_flags |= TRACE_FILE_ANNOTATE; in trace_init_global_iter()
10056 if (trace_clocks[iter->tr->clock_id].in_ns) in trace_init_global_iter()
10057 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; in trace_init_global_iter()
10060 iter->temp = static_temp_buf; in trace_init_global_iter()
10061 iter->temp_size = STATIC_TEMP_BUF_SIZE; in trace_init_global_iter()
10062 iter->fmt = static_fmt_buf; in trace_init_global_iter()
10063 iter->fmt_size = STATIC_FMT_BUF_SIZE; in trace_init_global_iter()
10069 static struct trace_iterator iter; in ftrace_dump() local
10097 trace_init_global_iter(&iter); in ftrace_dump()
10100 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); in ftrace_dump()
10101 size = ring_buffer_size(iter.array_buffer->buffer, cpu); in ftrace_dump()
10115 iter.cpu_file = RING_BUFFER_ALL_CPUS; in ftrace_dump()
10118 iter.cpu_file = raw_smp_processor_id(); in ftrace_dump()
10124 iter.cpu_file = RING_BUFFER_ALL_CPUS; in ftrace_dump()
10142 while (!trace_empty(&iter)) { in ftrace_dump()
10150 trace_iterator_reset(&iter); in ftrace_dump()
10153 iter.iter_flags |= TRACE_FILE_LAT_FMT; in ftrace_dump()
10155 if (trace_find_next_entry_inc(&iter) != NULL) { in ftrace_dump()
10158 ret = print_trace_line(&iter); in ftrace_dump()
10160 trace_consume(&iter); in ftrace_dump()
10164 trace_printk_seq(&iter.seq); in ftrace_dump()
10176 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); in ftrace_dump()