Lines Matching refs:bpage
352 static void rb_init_page(struct buffer_data_page *bpage) in rb_init_page() argument
354 local_set(&bpage->commit, 0); in rb_init_page()
361 static void free_buffer_page(struct buffer_page *bpage) in free_buffer_page() argument
363 free_page((unsigned long)bpage->page); in free_buffer_page()
364 kfree(bpage); in free_buffer_page()
1023 struct buffer_page **bpage) in rb_inc_page() argument
1025 struct list_head *p = rb_list_head((*bpage)->list.next); in rb_inc_page()
1027 *bpage = list_entry(p, struct buffer_page, list); in rb_inc_page()
1148 struct buffer_page *bpage) in rb_check_bpage() argument
1150 unsigned long val = (unsigned long)bpage; in rb_check_bpage()
1181 struct buffer_page *bpage, *tmp; in rb_check_pages() local
1197 list_for_each_entry_safe(bpage, tmp, head, list) { in rb_check_pages()
1199 bpage->list.next->prev != &bpage->list)) in rb_check_pages()
1202 bpage->list.prev->next != &bpage->list)) in rb_check_pages()
1204 if (rb_check_list(cpu_buffer, &bpage->list)) in rb_check_pages()
1215 struct buffer_page *bpage, *tmp; in __rb_allocate_pages() local
1252 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), in __rb_allocate_pages()
1254 if (!bpage) in __rb_allocate_pages()
1257 list_add(&bpage->list, pages); in __rb_allocate_pages()
1262 bpage->page = page_address(page); in __rb_allocate_pages()
1263 rb_init_page(bpage->page); in __rb_allocate_pages()
1274 list_for_each_entry_safe(bpage, tmp, pages, list) { in __rb_allocate_pages()
1275 list_del_init(&bpage->list); in __rb_allocate_pages()
1276 free_buffer_page(bpage); in __rb_allocate_pages()
1313 struct buffer_page *bpage; in rb_allocate_cpu_buffer() local
1333 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), in rb_allocate_cpu_buffer()
1335 if (!bpage) in rb_allocate_cpu_buffer()
1338 rb_check_bpage(cpu_buffer, bpage); in rb_allocate_cpu_buffer()
1340 cpu_buffer->reader_page = bpage; in rb_allocate_cpu_buffer()
1344 bpage->page = page_address(page); in rb_allocate_cpu_buffer()
1345 rb_init_page(bpage->page); in rb_allocate_cpu_buffer()
1373 struct buffer_page *bpage, *tmp; in rb_free_cpu_buffer() local
1382 list_for_each_entry_safe(bpage, tmp, head, list) { in rb_free_cpu_buffer()
1383 list_del_init(&bpage->list); in rb_free_cpu_buffer()
1384 free_buffer_page(bpage); in rb_free_cpu_buffer()
1386 bpage = list_entry(head, struct buffer_page, list); in rb_free_cpu_buffer()
1387 free_buffer_page(bpage); in rb_free_cpu_buffer()
1514 static inline unsigned long rb_page_entries(struct buffer_page *bpage) in rb_page_entries() argument
1516 return local_read(&bpage->entries) & RB_WRITE_MASK; in rb_page_entries()
1519 static inline unsigned long rb_page_write(struct buffer_page *bpage) in rb_page_write() argument
1521 return local_read(&bpage->write) & RB_WRITE_MASK; in rb_page_write()
1700 struct buffer_page *bpage, *tmp; in rb_insert_pages() local
1701 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, in rb_insert_pages()
1703 list_del_init(&bpage->list); in rb_insert_pages()
1704 free_buffer_page(bpage); in rb_insert_pages()
1904 struct buffer_page *bpage, *tmp; in ring_buffer_resize() local
1912 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, in ring_buffer_resize()
1914 list_del_init(&bpage->list); in ring_buffer_resize()
1915 free_buffer_page(bpage); in ring_buffer_resize()
1934 static __always_inline void *__rb_page_index(struct buffer_page *bpage, unsigned index) in __rb_page_index() argument
1936 return bpage->page->data + index; in __rb_page_index()
1952 static __always_inline unsigned rb_page_commit(struct buffer_page *bpage) in rb_page_commit() argument
1954 return local_read(&bpage->page->commit); in rb_page_commit()
1958 static __always_inline unsigned rb_page_size(struct buffer_page *bpage) in rb_page_size() argument
1960 return rb_page_commit(bpage); in rb_page_size()
2455 struct buffer_page *bpage; in rb_try_to_discard() local
2464 bpage = READ_ONCE(cpu_buffer->tail_page); in rb_try_to_discard()
2466 if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) { in rb_try_to_discard()
2468 local_read(&bpage->write) & ~RB_WRITE_MASK; in rb_try_to_discard()
2478 index = local_cmpxchg(&bpage->write, old_index, new_index); in rb_try_to_discard()
3097 struct buffer_page *bpage = cpu_buffer->commit_page; in rb_decrement_entry() local
3103 if (likely(bpage->page == (void *)addr)) { in rb_decrement_entry()
3104 local_dec(&bpage->entries); in rb_decrement_entry()
3112 rb_inc_page(cpu_buffer, &bpage); in rb_decrement_entry()
3113 start = bpage; in rb_decrement_entry()
3115 if (bpage->page == (void *)addr) { in rb_decrement_entry()
3116 local_dec(&bpage->entries); in rb_decrement_entry()
3119 rb_inc_page(cpu_buffer, &bpage); in rb_decrement_entry()
3120 } while (bpage != start); in rb_decrement_entry()
3452 struct buffer_page *bpage; in ring_buffer_oldest_event_ts() local
3465 bpage = cpu_buffer->reader_page; in ring_buffer_oldest_event_ts()
3467 bpage = rb_set_head_page(cpu_buffer); in ring_buffer_oldest_event_ts()
3468 if (bpage) in ring_buffer_oldest_event_ts()
3469 ret = bpage->page->time_stamp; in ring_buffer_oldest_event_ts()
4742 struct buffer_data_page *bpage = NULL; in ring_buffer_alloc_read_page() local
4754 bpage = cpu_buffer->free_page; in ring_buffer_alloc_read_page()
4761 if (bpage) in ring_buffer_alloc_read_page()
4769 bpage = page_address(page); in ring_buffer_alloc_read_page()
4772 rb_init_page(bpage); in ring_buffer_alloc_read_page()
4774 return bpage; in ring_buffer_alloc_read_page()
4789 struct buffer_data_page *bpage = data; in ring_buffer_free_read_page() local
4790 struct page *page = virt_to_page(bpage); in ring_buffer_free_read_page()
4806 cpu_buffer->free_page = bpage; in ring_buffer_free_read_page()
4807 bpage = NULL; in ring_buffer_free_read_page()
4814 free_page((unsigned long)bpage); in ring_buffer_free_read_page()
4856 struct buffer_data_page *bpage; in ring_buffer_read_page() local
4880 bpage = *data_page; in ring_buffer_read_page()
4881 if (!bpage) in ring_buffer_read_page()
4944 memcpy(bpage->data + pos, rpage->data + rpos, size); in ring_buffer_read_page()
4961 local_set(&bpage->commit, pos); in ring_buffer_read_page()
4962 bpage->time_stamp = save_timestamp; in ring_buffer_read_page()
4972 rb_init_page(bpage); in ring_buffer_read_page()
4973 bpage = reader->page; in ring_buffer_read_page()
4978 *data_page = bpage; in ring_buffer_read_page()
4986 local_set(&bpage->commit, reader->real_end); in ring_buffer_read_page()
4992 commit = local_read(&bpage->commit); in ring_buffer_read_page()
5001 memcpy(&bpage->data[commit], &missed_events, in ring_buffer_read_page()
5003 local_add(RB_MISSED_STORED, &bpage->commit); in ring_buffer_read_page()
5006 local_add(RB_MISSED_EVENTS, &bpage->commit); in ring_buffer_read_page()
5013 memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit); in ring_buffer_read_page()