Lines Matching refs:buf
36 struct rchan_buf *buf = vmf->vma->vm_private_data; in relay_buf_fault() local
39 if (!buf) in relay_buf_fault()
42 page = vmalloc_to_page(buf->start + (pgoff << PAGE_SHIFT)); in relay_buf_fault()
83 static int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma) in relay_mmap_buf() argument
87 if (!buf) in relay_mmap_buf()
90 if (length != (unsigned long)buf->chan->alloc_size) in relay_mmap_buf()
95 vma->vm_private_data = buf; in relay_mmap_buf()
108 static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size) in relay_alloc_buf() argument
116 buf->page_array = relay_alloc_page_array(n_pages); in relay_alloc_buf()
117 if (!buf->page_array) in relay_alloc_buf()
121 buf->page_array[i] = alloc_page(GFP_KERNEL); in relay_alloc_buf()
122 if (unlikely(!buf->page_array[i])) in relay_alloc_buf()
124 set_page_private(buf->page_array[i], (unsigned long)buf); in relay_alloc_buf()
126 mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL); in relay_alloc_buf()
131 buf->page_count = n_pages; in relay_alloc_buf()
136 __free_page(buf->page_array[j]); in relay_alloc_buf()
137 relay_free_page_array(buf->page_array); in relay_alloc_buf()
149 struct rchan_buf *buf; in relay_create_buf() local
154 buf = kzalloc(sizeof(struct rchan_buf), GFP_KERNEL); in relay_create_buf()
155 if (!buf) in relay_create_buf()
157 buf->padding = kmalloc_array(chan->n_subbufs, sizeof(size_t), in relay_create_buf()
159 if (!buf->padding) in relay_create_buf()
162 buf->start = relay_alloc_buf(buf, &chan->alloc_size); in relay_create_buf()
163 if (!buf->start) in relay_create_buf()
166 buf->chan = chan; in relay_create_buf()
167 kref_get(&buf->chan->kref); in relay_create_buf()
168 return buf; in relay_create_buf()
171 kfree(buf->padding); in relay_create_buf()
172 kfree(buf); in relay_create_buf()
185 free_percpu(chan->buf); in relay_destroy_channel()
193 static void relay_destroy_buf(struct rchan_buf *buf) in relay_destroy_buf() argument
195 struct rchan *chan = buf->chan; in relay_destroy_buf()
198 if (likely(buf->start)) { in relay_destroy_buf()
199 vunmap(buf->start); in relay_destroy_buf()
200 for (i = 0; i < buf->page_count; i++) in relay_destroy_buf()
201 __free_page(buf->page_array[i]); in relay_destroy_buf()
202 relay_free_page_array(buf->page_array); in relay_destroy_buf()
204 *per_cpu_ptr(chan->buf, buf->cpu) = NULL; in relay_destroy_buf()
205 kfree(buf->padding); in relay_destroy_buf()
206 kfree(buf); in relay_destroy_buf()
220 struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref); in relay_remove_buf() local
221 relay_destroy_buf(buf); in relay_remove_buf()
230 static int relay_buf_empty(struct rchan_buf *buf) in relay_buf_empty() argument
232 return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1; in relay_buf_empty()
241 int relay_buf_full(struct rchan_buf *buf) in relay_buf_full() argument
243 size_t ready = buf->subbufs_produced - buf->subbufs_consumed; in relay_buf_full()
244 return (ready >= buf->chan->n_subbufs) ? 1 : 0; in relay_buf_full()
252 static int relay_subbuf_start(struct rchan_buf *buf, void *subbuf, in relay_subbuf_start() argument
255 if (!buf->chan->cb->subbuf_start) in relay_subbuf_start()
256 return !relay_buf_full(buf); in relay_subbuf_start()
258 return buf->chan->cb->subbuf_start(buf, subbuf, in relay_subbuf_start()
270 struct rchan_buf *buf; in wakeup_readers() local
272 buf = container_of(work, struct rchan_buf, wakeup_work); in wakeup_readers()
273 wake_up_interruptible(&buf->read_wait); in wakeup_readers()
283 static void __relay_reset(struct rchan_buf *buf, unsigned int init) in __relay_reset() argument
288 init_waitqueue_head(&buf->read_wait); in __relay_reset()
289 kref_init(&buf->kref); in __relay_reset()
290 init_irq_work(&buf->wakeup_work, wakeup_readers); in __relay_reset()
292 irq_work_sync(&buf->wakeup_work); in __relay_reset()
295 buf->subbufs_produced = 0; in __relay_reset()
296 buf->subbufs_consumed = 0; in __relay_reset()
297 buf->bytes_consumed = 0; in __relay_reset()
298 buf->finalized = 0; in __relay_reset()
299 buf->data = buf->start; in __relay_reset()
300 buf->offset = 0; in __relay_reset()
302 for (i = 0; i < buf->chan->n_subbufs; i++) in __relay_reset()
303 buf->padding[i] = 0; in __relay_reset()
305 relay_subbuf_start(buf, buf->data, NULL, 0); in __relay_reset()
321 struct rchan_buf *buf; in relay_reset() local
327 if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) { in relay_reset()
328 __relay_reset(buf, 0); in relay_reset()
334 if ((buf = *per_cpu_ptr(chan->buf, i))) in relay_reset()
335 __relay_reset(buf, 0); in relay_reset()
340 static inline void relay_set_buf_dentry(struct rchan_buf *buf, in relay_set_buf_dentry() argument
343 buf->dentry = dentry; in relay_set_buf_dentry()
344 d_inode(buf->dentry)->i_size = buf->early_bytes; in relay_set_buf_dentry()
348 struct rchan_buf *buf, in relay_create_buf_file() argument
361 S_IRUSR, buf, in relay_create_buf_file()
378 struct rchan_buf *buf = NULL; in relay_open_buf() local
382 return *per_cpu_ptr(chan->buf, 0); in relay_open_buf()
384 buf = relay_create_buf(chan); in relay_open_buf()
385 if (!buf) in relay_open_buf()
389 dentry = relay_create_buf_file(chan, buf, cpu); in relay_open_buf()
392 relay_set_buf_dentry(buf, dentry); in relay_open_buf()
396 S_IRUSR, buf, in relay_open_buf()
402 buf->cpu = cpu; in relay_open_buf()
403 __relay_reset(buf, 1); in relay_open_buf()
406 *per_cpu_ptr(chan->buf, 0) = buf; in relay_open_buf()
407 buf->cpu = 0; in relay_open_buf()
410 return buf; in relay_open_buf()
413 relay_destroy_buf(buf); in relay_open_buf()
425 static void relay_close_buf(struct rchan_buf *buf) in relay_close_buf() argument
427 buf->finalized = 1; in relay_close_buf()
428 irq_work_sync(&buf->wakeup_work); in relay_close_buf()
429 buf->chan->cb->remove_buf_file(buf->dentry); in relay_close_buf()
430 kref_put(&buf->kref, relay_remove_buf); in relay_close_buf()
436 struct rchan_buf *buf; in relay_prepare_cpu() local
440 if (*per_cpu_ptr(chan->buf, cpu)) in relay_prepare_cpu()
442 buf = relay_open_buf(chan, cpu); in relay_prepare_cpu()
443 if (!buf) { in relay_prepare_cpu()
448 *per_cpu_ptr(chan->buf, cpu) = buf; in relay_prepare_cpu()
483 struct rchan_buf *buf; in relay_open() local
496 chan->buf = alloc_percpu(struct rchan_buf *); in relay_open()
497 if (!chan->buf) { in relay_open()
517 buf = relay_open_buf(chan, i); in relay_open()
518 if (!buf) in relay_open()
520 *per_cpu_ptr(chan->buf, i) = buf; in relay_open()
529 if ((buf = *per_cpu_ptr(chan->buf, i))) in relay_open()
530 relay_close_buf(buf); in relay_open()
540 struct rchan_buf *buf; member
549 relay_set_buf_dentry(p->buf, p->dentry); in __relay_set_buf_dentry()
575 struct rchan_buf *buf; in relay_late_setup_files() local
594 buf = *per_cpu_ptr(chan->buf, 0); in relay_late_setup_files()
595 if (!WARN_ON_ONCE(!buf)) { in relay_late_setup_files()
596 dentry = relay_create_buf_file(chan, buf, 0); in relay_late_setup_files()
598 relay_set_buf_dentry(buf, dentry); in relay_late_setup_files()
613 buf = *per_cpu_ptr(chan->buf, i); in relay_late_setup_files()
614 if (unlikely(!buf)) { in relay_late_setup_files()
620 dentry = relay_create_buf_file(chan, buf, i); in relay_late_setup_files()
628 relay_set_buf_dentry(buf, dentry); in relay_late_setup_files()
631 disp.buf = buf; in relay_late_setup_files()
659 size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length) in relay_switch_subbuf() argument
664 if (unlikely(length > buf->chan->subbuf_size)) in relay_switch_subbuf()
667 if (buf->offset != buf->chan->subbuf_size + 1) { in relay_switch_subbuf()
668 buf->prev_padding = buf->chan->subbuf_size - buf->offset; in relay_switch_subbuf()
669 old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; in relay_switch_subbuf()
670 buf->padding[old_subbuf] = buf->prev_padding; in relay_switch_subbuf()
671 buf->subbufs_produced++; in relay_switch_subbuf()
672 if (buf->dentry) in relay_switch_subbuf()
673 d_inode(buf->dentry)->i_size += in relay_switch_subbuf()
674 buf->chan->subbuf_size - in relay_switch_subbuf()
675 buf->padding[old_subbuf]; in relay_switch_subbuf()
677 buf->early_bytes += buf->chan->subbuf_size - in relay_switch_subbuf()
678 buf->padding[old_subbuf]; in relay_switch_subbuf()
680 if (waitqueue_active(&buf->read_wait)) { in relay_switch_subbuf()
687 irq_work_queue(&buf->wakeup_work); in relay_switch_subbuf()
691 old = buf->data; in relay_switch_subbuf()
692 new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; in relay_switch_subbuf()
693 new = buf->start + new_subbuf * buf->chan->subbuf_size; in relay_switch_subbuf()
694 buf->offset = 0; in relay_switch_subbuf()
695 if (!relay_subbuf_start(buf, new, old, buf->prev_padding)) { in relay_switch_subbuf()
696 buf->offset = buf->chan->subbuf_size + 1; in relay_switch_subbuf()
699 buf->data = new; in relay_switch_subbuf()
700 buf->padding[new_subbuf] = 0; in relay_switch_subbuf()
702 if (unlikely(length + buf->offset > buf->chan->subbuf_size)) in relay_switch_subbuf()
708 buf->chan->last_toobig = length; in relay_switch_subbuf()
730 struct rchan_buf *buf; in relay_subbufs_consumed() local
735 buf = *per_cpu_ptr(chan->buf, cpu); in relay_subbufs_consumed()
736 if (!buf || subbufs_consumed > chan->n_subbufs) in relay_subbufs_consumed()
739 if (subbufs_consumed > buf->subbufs_produced - buf->subbufs_consumed) in relay_subbufs_consumed()
740 buf->subbufs_consumed = buf->subbufs_produced; in relay_subbufs_consumed()
742 buf->subbufs_consumed += subbufs_consumed; in relay_subbufs_consumed()
754 struct rchan_buf *buf; in relay_close() local
761 if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) in relay_close()
762 relay_close_buf(buf); in relay_close()
765 if ((buf = *per_cpu_ptr(chan->buf, i))) in relay_close()
766 relay_close_buf(buf); in relay_close()
787 struct rchan_buf *buf; in relay_flush() local
793 if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) { in relay_flush()
794 relay_switch_subbuf(buf, 0); in relay_flush()
800 if ((buf = *per_cpu_ptr(chan->buf, i))) in relay_flush()
801 relay_switch_subbuf(buf, 0); in relay_flush()
815 struct rchan_buf *buf = inode->i_private; in relay_file_open() local
816 kref_get(&buf->kref); in relay_file_open()
817 filp->private_data = buf; in relay_file_open()
831 struct rchan_buf *buf = filp->private_data; in relay_file_mmap() local
832 return relay_mmap_buf(buf, vma); in relay_file_mmap()
845 struct rchan_buf *buf = filp->private_data; in relay_file_poll() local
847 if (buf->finalized) in relay_file_poll()
851 poll_wait(filp, &buf->read_wait, wait); in relay_file_poll()
852 if (!relay_buf_empty(buf)) in relay_file_poll()
869 struct rchan_buf *buf = filp->private_data; in relay_file_release() local
870 kref_put(&buf->kref, relay_remove_buf); in relay_file_release()
878 static void relay_file_read_consume(struct rchan_buf *buf, in relay_file_read_consume() argument
882 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_consume()
883 size_t n_subbufs = buf->chan->n_subbufs; in relay_file_read_consume()
886 if (buf->subbufs_produced == buf->subbufs_consumed && in relay_file_read_consume()
887 buf->offset == buf->bytes_consumed) in relay_file_read_consume()
890 if (buf->bytes_consumed + bytes_consumed > subbuf_size) { in relay_file_read_consume()
891 relay_subbufs_consumed(buf->chan, buf->cpu, 1); in relay_file_read_consume()
892 buf->bytes_consumed = 0; in relay_file_read_consume()
895 buf->bytes_consumed += bytes_consumed; in relay_file_read_consume()
897 read_subbuf = buf->subbufs_consumed % n_subbufs; in relay_file_read_consume()
899 read_subbuf = read_pos / buf->chan->subbuf_size; in relay_file_read_consume()
900 if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) { in relay_file_read_consume()
901 if ((read_subbuf == buf->subbufs_produced % n_subbufs) && in relay_file_read_consume()
902 (buf->offset == subbuf_size)) in relay_file_read_consume()
904 relay_subbufs_consumed(buf->chan, buf->cpu, 1); in relay_file_read_consume()
905 buf->bytes_consumed = 0; in relay_file_read_consume()
912 static int relay_file_read_avail(struct rchan_buf *buf) in relay_file_read_avail() argument
914 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_avail()
915 size_t n_subbufs = buf->chan->n_subbufs; in relay_file_read_avail()
916 size_t produced = buf->subbufs_produced; in relay_file_read_avail()
919 relay_file_read_consume(buf, 0, 0); in relay_file_read_avail()
921 consumed = buf->subbufs_consumed; in relay_file_read_avail()
923 if (unlikely(buf->offset > subbuf_size)) { in relay_file_read_avail()
931 buf->subbufs_consumed = consumed; in relay_file_read_avail()
932 buf->bytes_consumed = 0; in relay_file_read_avail()
935 produced = (produced % n_subbufs) * subbuf_size + buf->offset; in relay_file_read_avail()
936 consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed; in relay_file_read_avail()
942 if (buf->offset == subbuf_size && in relay_file_read_avail()
943 buf->subbufs_produced > buf->subbufs_consumed) in relay_file_read_avail()
957 struct rchan_buf *buf) in relay_file_read_subbuf_avail() argument
961 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_subbuf_avail()
963 write_subbuf = (buf->data - buf->start) / subbuf_size; in relay_file_read_subbuf_avail()
964 write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset; in relay_file_read_subbuf_avail()
967 padding = buf->padding[read_subbuf]; in relay_file_read_subbuf_avail()
986 static size_t relay_file_read_start_pos(struct rchan_buf *buf) in relay_file_read_start_pos() argument
989 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_start_pos()
990 size_t n_subbufs = buf->chan->n_subbufs; in relay_file_read_start_pos()
991 size_t consumed = buf->subbufs_consumed % n_subbufs; in relay_file_read_start_pos()
992 size_t read_pos = (consumed * subbuf_size + buf->bytes_consumed) in relay_file_read_start_pos()
996 padding = buf->padding[read_subbuf]; in relay_file_read_start_pos()
1013 static size_t relay_file_read_end_pos(struct rchan_buf *buf, in relay_file_read_end_pos() argument
1018 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_end_pos()
1019 size_t n_subbufs = buf->chan->n_subbufs; in relay_file_read_end_pos()
1022 padding = buf->padding[read_subbuf]; in relay_file_read_end_pos()
1038 struct rchan_buf *buf = filp->private_data; in relay_file_read() local
1050 if (!relay_file_read_avail(buf)) in relay_file_read()
1053 read_start = relay_file_read_start_pos(buf); in relay_file_read()
1054 avail = relay_file_read_subbuf_avail(read_start, buf); in relay_file_read()
1059 from = buf->start + read_start; in relay_file_read()
1068 relay_file_read_consume(buf, read_start, ret); in relay_file_read()
1069 *ppos = relay_file_read_end_pos(buf, read_start, ret); in relay_file_read()
1087 struct pipe_buffer *buf) in relay_pipe_buf_release() argument
1091 rbuf = (struct rchan_buf *)page_private(buf->page); in relay_pipe_buf_release()
1092 relay_consume_bytes(rbuf, buf->private); in relay_pipe_buf_release()