Lines Matching refs:page
337 struct buffer_data_page *page; /* Actual data page */ member
362 return local_read(&bpage->page->commit); in rb_page_commit()
369 free_pages((unsigned long)bpage->page, bpage->order); in free_buffer_page()
595 struct buffer_page *page = cpu_buffer->commit_page; in verify_event() local
605 if (page == tail_page || WARN_ON_ONCE(stop++ > 100)) in verify_event()
607 commit = local_read(&page->page->commit); in verify_event()
608 write = local_read(&page->write); in verify_event()
609 if (addr >= (unsigned long)&page->page->data[commit] && in verify_event()
610 addr < (unsigned long)&page->page->data[write]) in verify_event()
613 next = rb_list_head(page->list.next); in verify_event()
614 page = list_entry(next, struct buffer_page, list); in verify_event()
673 return rb_fix_abs_ts(ts, cpu_buffer->tail_page->page->time_stamp); in ring_buffer_event_time_stamp()
1188 rb_is_head_page(struct buffer_page *page, struct list_head *list) in rb_is_head_page() argument
1194 if ((val & ~RB_FLAG_MASK) != (unsigned long)&page->list) in rb_is_head_page()
1207 static bool rb_is_reader_page(struct buffer_page *page) in rb_is_reader_page() argument
1209 struct list_head *list = page->list.prev; in rb_is_reader_page()
1211 return rb_list_head(list->next) != &page->list; in rb_is_reader_page()
1244 meta->head_buffer = (unsigned long)head->page; in rb_head_page_activate()
1331 struct buffer_page *page; in rb_set_head_page() local
1343 page = head = cpu_buffer->head_page; in rb_set_head_page()
1352 if (rb_is_head_page(page, page->list.prev)) { in rb_set_head_page()
1353 cpu_buffer->head_page = page; in rb_set_head_page()
1354 return page; in rb_set_head_page()
1356 rb_inc_page(&page); in rb_set_head_page()
1357 } while (page != head); in rb_set_head_page()
1433 local_set(&next_page->page->commit, 0); in rb_tail_page_update()
1793 ret = rb_validate_buffer(cpu_buffer->reader_page->page, cpu_buffer->cpu); in rb_meta_validate_events()
1799 entry_bytes += local_read(&cpu_buffer->reader_page->page->commit); in rb_meta_validate_events()
1805 if (meta->commit_buffer == (unsigned long)cpu_buffer->reader_page->page) { in rb_meta_validate_events()
1818 ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu); in rb_meta_validate_events()
1830 entry_bytes += local_read(&head_page->page->commit); in rb_meta_validate_events()
1856 local_set(&cpu_buffer->reader_page->page->commit, 0); in rb_meta_validate_events()
1861 local_set(&head_page->page->commit, 0); in rb_meta_validate_events()
2021 if (meta->head_buffer == (unsigned long)bpage->page) in rb_meta_buffer_update()
2024 if (meta->commit_buffer == (unsigned long)bpage->page) { in rb_meta_buffer_update()
2074 struct page *page; in __rb_allocate_pages() local
2091 bpage->page = rb_range_buffer(cpu_buffer, i + 1); in __rb_allocate_pages()
2092 if (!bpage->page) in __rb_allocate_pages()
2100 page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), in __rb_allocate_pages()
2103 if (!page) in __rb_allocate_pages()
2105 bpage->page = page_address(page); in __rb_allocate_pages()
2106 rb_init_page(bpage->page); in __rb_allocate_pages()
2186 struct page *page; in rb_allocate_cpu_buffer() local
2228 bpage->page = rb_page_desc_page(pdesc, in rb_allocate_cpu_buffer()
2230 if (!bpage->page) in rb_allocate_cpu_buffer()
2247 bpage->page = rb_range_buffer(cpu_buffer, 0); in rb_allocate_cpu_buffer()
2248 if (!bpage->page) in rb_allocate_cpu_buffer()
2254 page = alloc_pages_node(cpu_to_node(cpu), in rb_allocate_cpu_buffer()
2257 if (!page) in rb_allocate_cpu_buffer()
2259 bpage->page = page_address(page); in rb_allocate_cpu_buffer()
2260 rb_init_page(bpage->page); in rb_allocate_cpu_buffer()
2319 cpu_buffer->reader_page->page = NULL; in rb_free_cpu_buffer()
3030 return bpage->page->data + index; in __rb_page_index()
3084 if (iter->page_stamp != iter_head_page->page->time_stamp || in rb_iter_head_event()
3092 iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp; in rb_iter_head_event()
3136 iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp; in rb_inc_iter()
3155 unsigned long old_head = (unsigned long)next_page->page; in rb_update_meta_head()
3159 new_head = (unsigned long)next_page->page; in rb_update_meta_head()
3172 void *old_reader = cpu_buffer->reader_page->page; in rb_update_meta_reader()
3173 void *new_reader = reader->page; in rb_update_meta_reader()
3717 if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) { in rb_try_to_discard()
3796 local_set(&cpu_buffer->commit_page->page->commit, in rb_set_commit_to_write()
3801 meta->commit_buffer = (unsigned long)cpu_buffer->commit_page->page; in rb_set_commit_to_write()
3811 local_set(&cpu_buffer->commit_page->page->commit, in rb_set_commit_to_write()
3814 local_read(&cpu_buffer->commit_page->page->commit) & in rb_set_commit_to_write()
4254 bpage = info->tail_page->page; in check_buffer()
4440 tail_page->page->time_stamp = info->ts; in __rb_reserve_next()
4604 if (likely(bpage->page == (void *)addr)) { in rb_decrement_entry()
4616 if (bpage->page == (void *)addr) { in rb_decrement_entry()
4934 ret = bpage->page->time_stamp; in ring_buffer_oldest_event_ts()
5124 iter->page_stamp = cpu_buffer->reader_page->page->time_stamp; in rb_iter_reset()
5126 iter->read_stamp = iter->head_page->page->time_stamp; in rb_iter_reset()
5173 commit_ts = commit_page->page->time_stamp; in ring_buffer_iter_empty()
5187 curr_commit_ts = READ_ONCE(curr_commit_page->page->time_stamp); in ring_buffer_iter_empty()
5287 cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp; in __rb_get_reader_page_from_writer()
5301 cpu_buffer->reader_page->page = in __rb_get_reader_page_from_writer()
5305 cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp; in __rb_get_reader_page_from_writer()
5367 local_set(&cpu_buffer->reader_page->page->commit, 0); in __rb_get_reader_page()
5449 cpu_buffer->read_stamp = reader->page->time_stamp; in __rb_get_reader_page()
5602 *ts = rb_fix_abs_ts(*ts, reader->page->time_stamp); in rb_buffer_peek()
5695 *ts = rb_fix_abs_ts(*ts, iter->head_page->page->time_stamp); in rb_iter_peek()
5995 static void rb_clear_buffer_page(struct buffer_page *page) in rb_clear_buffer_page() argument
5997 local_set(&page->write, 0); in rb_clear_buffer_page()
5998 local_set(&page->entries, 0); in rb_clear_buffer_page()
5999 rb_init_page(page->page); in rb_clear_buffer_page()
6000 page->read = 0; in rb_clear_buffer_page()
6025 struct buffer_page *page; in rb_reset_cpu() local
6049 list_for_each_entry(page, cpu_buffer->pages, list) { in rb_reset_cpu()
6050 rb_clear_buffer_page(page); in rb_reset_cpu()
6450 struct page *page; in ring_buffer_alloc_read_page() local
6475 page = alloc_pages_node(cpu_to_node(cpu), in ring_buffer_alloc_read_page()
6478 if (!page) { in ring_buffer_alloc_read_page()
6483 bpage->data = page_address(page); in ring_buffer_alloc_read_page()
6505 struct page *page = virt_to_page(bpage); in ring_buffer_free_read_page() local
6518 if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order) in ring_buffer_free_read_page()
6635 struct buffer_data_page *rpage = cpu_buffer->reader_page->page; in ring_buffer_read_page()
6701 bpage = reader->page; in ring_buffer_read_page()
6702 reader->page = data_page->data; in ring_buffer_read_page()
6757 void *ring_buffer_read_page_data(struct buffer_data_read_page *page) in ring_buffer_read_page_data() argument
6759 return page->data; in ring_buffer_read_page_data()
6972 struct page *page; in rb_alloc_meta_page() local
6977 page = alloc_page(GFP_USER | __GFP_ZERO); in rb_alloc_meta_page()
6978 if (!page) in rb_alloc_meta_page()
6981 cpu_buffer->meta_page = page_to_virt(page); in rb_alloc_meta_page()
7002 subbuf_ids[id] = (unsigned long)cpu_buffer->reader_page->page; in rb_setup_ids_meta_page()
7010 subbuf_ids[id] = (unsigned long)subbuf->page; in rb_setup_ids_meta_page()
7108 struct page **pages; in __rb_map_vma()
7173 struct page *page; in __rb_map_vma() local
7181 page = virt_to_page((void *)cpu_buffer->subbuf_ids[s]); in __rb_map_vma()
7183 for (; off < (1 << (subbuf_order)); off++, page++) { in __rb_map_vma()
7187 pages[p++] = page; in __rb_map_vma()
7360 struct buffer_data_page *bpage = reader->page; in ring_buffer_map_get_reader()
7395 cpu, missed_events, cpu_buffer->reader_page->page->time_stamp); in ring_buffer_map_get_reader()
7405 flush_kernel_vmap_range(cpu_buffer->reader_page->page, in ring_buffer_map_get_reader()