1918 rb_iter_head_event(struct ring_buffer_iter *iter)  in rb_iter_head_event()  argument
1920 	return __rb_page_index(iter->head_page, iter->head);  in rb_iter_head_event()
1948 static void rb_inc_iter(struct ring_buffer_iter *iter)  in rb_inc_iter()  argument
1950 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in rb_inc_iter()
1958 	if (iter->head_page == cpu_buffer->reader_page)  in rb_inc_iter()
1959 		iter->head_page = rb_set_head_page(cpu_buffer);  in rb_inc_iter()
1961 		rb_inc_page(cpu_buffer, &iter->head_page);  in rb_inc_iter()
1963 	iter->read_stamp = iter->head_page->page->time_stamp;  in rb_inc_iter()
1964 	iter->head = 0;  in rb_inc_iter()
3544 static void rb_iter_reset(struct ring_buffer_iter *iter)  in rb_iter_reset()  argument
3546 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in rb_iter_reset()
3549 	iter->head_page = cpu_buffer->reader_page;  in rb_iter_reset()
3550 	iter->head = cpu_buffer->reader_page->read;  in rb_iter_reset()
3552 	iter->cache_reader_page = iter->head_page;  in rb_iter_reset()
3553 	iter->cache_read = cpu_buffer->read;  in rb_iter_reset()
3555 	if (iter->head)  in rb_iter_reset()
3556 		iter->read_stamp = cpu_buffer->read_stamp;  in rb_iter_reset()
3558 		iter->read_stamp = iter->head_page->page->time_stamp;  in rb_iter_reset()
3568 void ring_buffer_iter_reset(struct ring_buffer_iter *iter)  in ring_buffer_iter_reset()  argument
3573 	if (!iter)  in ring_buffer_iter_reset()
3576 	cpu_buffer = iter->cpu_buffer;  in ring_buffer_iter_reset()
3579 	rb_iter_reset(iter);  in ring_buffer_iter_reset()
3588 int ring_buffer_iter_empty(struct ring_buffer_iter *iter)  in ring_buffer_iter_empty()  argument
3596 	cpu_buffer = iter->cpu_buffer;  in ring_buffer_iter_empty()
3604 	return ((iter->head_page == commit_page && iter->head == commit) ||  in ring_buffer_iter_empty()
3605 		(iter->head_page == reader && commit_page == head_page &&  in ring_buffer_iter_empty()
3607 		 iter->head == rb_page_commit(cpu_buffer->reader_page)));  in ring_buffer_iter_empty()
3642 rb_update_iter_read_stamp(struct ring_buffer_iter *iter,  in rb_update_iter_read_stamp()  argument
3653 		iter->read_stamp += delta;  in rb_update_iter_read_stamp()
3658 		iter->read_stamp = delta;  in rb_update_iter_read_stamp()
3662 		iter->read_stamp += event->time_delta;  in rb_update_iter_read_stamp()
3829 static void rb_advance_iter(struct ring_buffer_iter *iter)  in rb_advance_iter()  argument
3835 	cpu_buffer = iter->cpu_buffer;  in rb_advance_iter()
3840 	if (iter->head >= rb_page_size(iter->head_page)) {  in rb_advance_iter()
3842 		if (iter->head_page == cpu_buffer->commit_page)  in rb_advance_iter()
3844 		rb_inc_iter(iter);  in rb_advance_iter()
3848 	event = rb_iter_head_event(iter);  in rb_advance_iter()
3857 		       (iter->head_page == cpu_buffer->commit_page) &&  in rb_advance_iter()
3858 		       (iter->head + length > rb_commit_index(cpu_buffer))))  in rb_advance_iter()
3861 	rb_update_iter_read_stamp(iter, event);  in rb_advance_iter()
3863 	iter->head += length;  in rb_advance_iter()
3866 	if ((iter->head >= rb_page_size(iter->head_page)) &&  in rb_advance_iter()
3867 	    (iter->head_page != cpu_buffer->commit_page))  in rb_advance_iter()
3868 		rb_inc_iter(iter);  in rb_advance_iter()
3950 rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)  in rb_iter_peek()  argument
3960 	cpu_buffer = iter->cpu_buffer;  in rb_iter_peek()
3968 	if (unlikely(iter->cache_read != cpu_buffer->read ||  in rb_iter_peek()
3969 		     iter->cache_reader_page != cpu_buffer->reader_page))  in rb_iter_peek()
3970 		rb_iter_reset(iter);  in rb_iter_peek()
3973 	if (ring_buffer_iter_empty(iter))  in rb_iter_peek()
3990 	if (iter->head >= rb_page_size(iter->head_page)) {  in rb_iter_peek()
3991 		rb_inc_iter(iter);  in rb_iter_peek()
3995 	event = rb_iter_head_event(iter);  in rb_iter_peek()
4000 			rb_inc_iter(iter);  in rb_iter_peek()
4003 		rb_advance_iter(iter);  in rb_iter_peek()
4008 		rb_advance_iter(iter);  in rb_iter_peek()
4018 		rb_advance_iter(iter);  in rb_iter_peek()
4023 			*ts = iter->read_stamp + event->time_delta;  in rb_iter_peek()
4115 ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)  in ring_buffer_iter_peek()  argument
4117 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in ring_buffer_iter_peek()
4123 	event = rb_iter_peek(iter, ts);  in ring_buffer_iter_peek()
4207 	struct ring_buffer_iter *iter;  in ring_buffer_read_prepare()  local
4212 	iter = kmalloc(sizeof(*iter), flags);  in ring_buffer_read_prepare()
4213 	if (!iter)  in ring_buffer_read_prepare()
4218 	iter->cpu_buffer = cpu_buffer;  in ring_buffer_read_prepare()
4223 	return iter;  in ring_buffer_read_prepare()
4253 ring_buffer_read_start(struct ring_buffer_iter *iter)  in ring_buffer_read_start()  argument
4258 	if (!iter)  in ring_buffer_read_start()
4261 	cpu_buffer = iter->cpu_buffer;  in ring_buffer_read_start()
4265 	rb_iter_reset(iter);  in ring_buffer_read_start()
4279 ring_buffer_read_finish(struct ring_buffer_iter *iter)  in ring_buffer_read_finish()  argument
4281 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in ring_buffer_read_finish()
4296 	kfree(iter);  in ring_buffer_read_finish()
4308 ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)  in ring_buffer_read()  argument
4311 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in ring_buffer_read()
4316 	event = rb_iter_peek(iter, ts);  in ring_buffer_read()
4323 	rb_advance_iter(iter);  in ring_buffer_read()