Lines Matching refs:bpage
326 static void rb_init_page(struct buffer_data_page *bpage) in rb_init_page() argument
328 local_set(&bpage->commit, 0); in rb_init_page()
331 static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage) in rb_page_commit() argument
333 return local_read(&bpage->page->commit); in rb_page_commit()
336 static void free_buffer_page(struct buffer_page *bpage) in free_buffer_page() argument
338 free_page((unsigned long)bpage->page); in free_buffer_page()
339 kfree(bpage); in free_buffer_page()
1388 static inline void rb_inc_page(struct buffer_page **bpage) in rb_inc_page() argument
1390 struct list_head *p = rb_list_head((*bpage)->list.next); in rb_inc_page()
1392 *bpage = list_entry(p, struct buffer_page, list); in rb_inc_page()
1513 struct buffer_page *bpage) in rb_check_bpage() argument
1515 unsigned long val = (unsigned long)bpage; in rb_check_bpage()
1559 struct buffer_page *bpage, *tmp; in __rb_allocate_pages() local
1596 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), in __rb_allocate_pages()
1598 if (!bpage) in __rb_allocate_pages()
1601 rb_check_bpage(cpu_buffer, bpage); in __rb_allocate_pages()
1603 list_add(&bpage->list, pages); in __rb_allocate_pages()
1608 bpage->page = page_address(page); in __rb_allocate_pages()
1609 rb_init_page(bpage->page); in __rb_allocate_pages()
1620 list_for_each_entry_safe(bpage, tmp, pages, list) { in __rb_allocate_pages()
1621 list_del_init(&bpage->list); in __rb_allocate_pages()
1622 free_buffer_page(bpage); in __rb_allocate_pages()
1659 struct buffer_page *bpage; in rb_allocate_cpu_buffer() local
1679 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), in rb_allocate_cpu_buffer()
1681 if (!bpage) in rb_allocate_cpu_buffer()
1684 rb_check_bpage(cpu_buffer, bpage); in rb_allocate_cpu_buffer()
1686 cpu_buffer->reader_page = bpage; in rb_allocate_cpu_buffer()
1690 bpage->page = page_address(page); in rb_allocate_cpu_buffer()
1691 rb_init_page(bpage->page); in rb_allocate_cpu_buffer()
1719 struct buffer_page *bpage, *tmp; in rb_free_cpu_buffer() local
1728 list_for_each_entry_safe(bpage, tmp, head, list) { in rb_free_cpu_buffer()
1729 list_del_init(&bpage->list); in rb_free_cpu_buffer()
1730 free_buffer_page(bpage); in rb_free_cpu_buffer()
1732 bpage = list_entry(head, struct buffer_page, list); in rb_free_cpu_buffer()
1733 free_buffer_page(bpage); in rb_free_cpu_buffer()
1883 static inline unsigned long rb_page_entries(struct buffer_page *bpage) in rb_page_entries() argument
1885 return local_read(&bpage->entries) & RB_WRITE_MASK; in rb_page_entries()
1888 static inline unsigned long rb_page_write(struct buffer_page *bpage) in rb_page_write() argument
1890 return local_read(&bpage->write) & RB_WRITE_MASK; in rb_page_write()
2073 struct buffer_page *bpage, *tmp; in rb_insert_pages() local
2074 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, in rb_insert_pages()
2076 list_del_init(&bpage->list); in rb_insert_pages()
2077 free_buffer_page(bpage); in rb_insert_pages()
2290 struct buffer_page *bpage, *tmp; in ring_buffer_resize() local
2298 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, in ring_buffer_resize()
2300 list_del_init(&bpage->list); in ring_buffer_resize()
2301 free_buffer_page(bpage); in ring_buffer_resize()
2322 static __always_inline void *__rb_page_index(struct buffer_page *bpage, unsigned index) in __rb_page_index() argument
2324 return bpage->page->data + index; in __rb_page_index()
2394 static __always_inline unsigned rb_page_size(struct buffer_page *bpage) in rb_page_size() argument
2396 return rb_page_commit(bpage); in rb_page_size()
2945 struct buffer_page *bpage; in rb_try_to_discard() local
2954 bpage = READ_ONCE(cpu_buffer->tail_page); in rb_try_to_discard()
2960 if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) { in rb_try_to_discard()
2962 local_read(&bpage->write) & ~RB_WRITE_MASK; in rb_try_to_discard()
2994 index = local_cmpxchg(&bpage->write, old_index, new_index); in rb_try_to_discard()
3341 static void dump_buffer_page(struct buffer_data_page *bpage, in dump_buffer_page() argument
3349 ts = bpage->time_stamp; in dump_buffer_page()
3354 event = (struct ring_buffer_event *)(bpage->data + e); in dump_buffer_page()
3398 struct buffer_data_page *bpage; in check_buffer() local
3403 bpage = info->tail_page->page; in check_buffer()
3407 tail = local_read(&bpage->commit); in check_buffer()
3418 if (tail <= 8 || tail > local_read(&bpage->commit)) in check_buffer()
3427 ts = bpage->time_stamp; in check_buffer()
3431 event = (struct ring_buffer_event *)(bpage->data + e); in check_buffer()
3472 dump_buffer_page(bpage, info, tail); in check_buffer()
3781 struct buffer_page *bpage = cpu_buffer->commit_page; in rb_decrement_entry() local
3787 if (likely(bpage->page == (void *)addr)) { in rb_decrement_entry()
3788 local_dec(&bpage->entries); in rb_decrement_entry()
3796 rb_inc_page(&bpage); in rb_decrement_entry()
3797 start = bpage; in rb_decrement_entry()
3799 if (bpage->page == (void *)addr) { in rb_decrement_entry()
3800 local_dec(&bpage->entries); in rb_decrement_entry()
3803 rb_inc_page(&bpage); in rb_decrement_entry()
3804 } while (bpage != start); in rb_decrement_entry()
4140 struct buffer_page *bpage; in ring_buffer_oldest_event_ts() local
4153 bpage = cpu_buffer->reader_page; in ring_buffer_oldest_event_ts()
4155 bpage = rb_set_head_page(cpu_buffer); in ring_buffer_oldest_event_ts()
4156 if (bpage) in ring_buffer_oldest_event_ts()
4157 ret = bpage->page->time_stamp; in ring_buffer_oldest_event_ts()
4493 static int __read_footer_reader_status(struct buffer_page *bpage) in __read_footer_reader_status() argument
4495 struct rb_ext_page_footer *footer = rb_ext_page_get_footer(bpage->page); in __read_footer_reader_status()
4500 static int __read_footer_writer_status(struct buffer_page *bpage) in __read_footer_writer_status() argument
4502 struct rb_ext_page_footer *footer = rb_ext_page_get_footer(bpage->page); in __read_footer_writer_status()
4511 struct buffer_page *bpage = start; in ring_buffer_search_footer() local
4516 status = search_writer ? __read_footer_writer_status(bpage) : in ring_buffer_search_footer()
4517 __read_footer_reader_status(bpage); in ring_buffer_search_footer()
4519 return bpage; in ring_buffer_search_footer()
4521 rb_inc_page(&bpage); in ring_buffer_search_footer()
4522 } while (bpage != start); in ring_buffer_search_footer()
5161 struct buffer_page *bpage; in ring_buffer_update_view() local
5171 bpage = cpu_buffer->reader_page; in ring_buffer_update_view()
5172 footer = rb_ext_page_get_footer(bpage->page); in ring_buffer_update_view()
5179 bpage = ring_buffer_search_footer(cpu_buffer->commit_page, in ring_buffer_update_view()
5181 if (!bpage) { in ring_buffer_update_view()
5185 cpu_buffer->commit_page = bpage; in ring_buffer_update_view()
5188 bpage = ring_buffer_search_footer(cpu_buffer->head_page, in ring_buffer_update_view()
5190 if (!bpage) { in ring_buffer_update_view()
5199 __set_head_page_flag(bpage, RB_PAGE_HEAD); in ring_buffer_update_view()
5203 cpu_buffer->head_page = bpage; in ring_buffer_update_view()
5738 struct buffer_data_page *bpage = NULL; in ring_buffer_alloc_read_page() local
5750 bpage = cpu_buffer->free_page; in ring_buffer_alloc_read_page()
5757 if (bpage) in ring_buffer_alloc_read_page()
5765 bpage = page_address(page); in ring_buffer_alloc_read_page()
5768 rb_init_page(bpage); in ring_buffer_alloc_read_page()
5770 return bpage; in ring_buffer_alloc_read_page()
5785 struct buffer_data_page *bpage = data; in ring_buffer_free_read_page() local
5786 struct page *page = virt_to_page(bpage); in ring_buffer_free_read_page()
5802 cpu_buffer->free_page = bpage; in ring_buffer_free_read_page()
5803 bpage = NULL; in ring_buffer_free_read_page()
5810 free_page((unsigned long)bpage); in ring_buffer_free_read_page()
5852 struct buffer_data_page *bpage; in ring_buffer_read_page() local
5876 bpage = *data_page; in ring_buffer_read_page()
5877 if (!bpage) in ring_buffer_read_page()
5941 memcpy(bpage->data + pos, rpage->data + rpos, size); in ring_buffer_read_page()
5958 local_set(&bpage->commit, pos); in ring_buffer_read_page()
5959 bpage->time_stamp = save_timestamp; in ring_buffer_read_page()
5969 rb_init_page(bpage); in ring_buffer_read_page()
5970 bpage = reader->page; in ring_buffer_read_page()
5975 *data_page = bpage; in ring_buffer_read_page()
5983 local_set(&bpage->commit, reader->real_end); in ring_buffer_read_page()
5989 commit = local_read(&bpage->commit); in ring_buffer_read_page()
5998 memcpy(&bpage->data[commit], &missed_events, in ring_buffer_read_page()
6000 local_add(RB_MISSED_STORED, &bpage->commit); in ring_buffer_read_page()
6003 local_add(RB_MISSED_EVENTS, &bpage->commit); in ring_buffer_read_page()
6010 memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit); in ring_buffer_read_page()
6097 struct buffer_page *bpage; in trace_buffer_pack() local
6110 bpage = rb->head_page; in trace_buffer_pack()
6120 cpu_pack->page_va[j] = (unsigned long)bpage->page; in trace_buffer_pack()
6121 rb_inc_page(&bpage); in trace_buffer_pack()