Lines Matching refs:buf
35 struct rchan_buf *buf = vma->vm_private_data; in relay_file_mmap_close() local
36 buf->chan->cb->buf_unmapped(buf, vma->vm_file); in relay_file_mmap_close()
45 struct rchan_buf *buf = vmf->vma->vm_private_data; in relay_buf_fault() local
48 if (!buf) in relay_buf_fault()
51 page = vmalloc_to_page(buf->start + (pgoff << PAGE_SHIFT)); in relay_buf_fault()
96 static int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma) in relay_mmap_buf() argument
101 if (!buf) in relay_mmap_buf()
104 if (length != (unsigned long)buf->chan->alloc_size) in relay_mmap_buf()
109 vma->vm_private_data = buf; in relay_mmap_buf()
110 buf->chan->cb->buf_mapped(buf, filp); in relay_mmap_buf()
123 static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size) in relay_alloc_buf() argument
131 buf->page_array = relay_alloc_page_array(n_pages); in relay_alloc_buf()
132 if (!buf->page_array) in relay_alloc_buf()
136 buf->page_array[i] = alloc_page(GFP_KERNEL); in relay_alloc_buf()
137 if (unlikely(!buf->page_array[i])) in relay_alloc_buf()
139 set_page_private(buf->page_array[i], (unsigned long)buf); in relay_alloc_buf()
141 mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL); in relay_alloc_buf()
146 buf->page_count = n_pages; in relay_alloc_buf()
151 __free_page(buf->page_array[j]); in relay_alloc_buf()
152 relay_free_page_array(buf->page_array); in relay_alloc_buf()
164 struct rchan_buf *buf; in relay_create_buf() local
169 buf = kzalloc(sizeof(struct rchan_buf), GFP_KERNEL); in relay_create_buf()
170 if (!buf) in relay_create_buf()
172 buf->padding = kmalloc_array(chan->n_subbufs, sizeof(size_t *), in relay_create_buf()
174 if (!buf->padding) in relay_create_buf()
177 buf->start = relay_alloc_buf(buf, &chan->alloc_size); in relay_create_buf()
178 if (!buf->start) in relay_create_buf()
181 buf->chan = chan; in relay_create_buf()
182 kref_get(&buf->chan->kref); in relay_create_buf()
183 return buf; in relay_create_buf()
186 kfree(buf->padding); in relay_create_buf()
187 kfree(buf); in relay_create_buf()
207 static void relay_destroy_buf(struct rchan_buf *buf) in relay_destroy_buf() argument
209 struct rchan *chan = buf->chan; in relay_destroy_buf()
212 if (likely(buf->start)) { in relay_destroy_buf()
213 vunmap(buf->start); in relay_destroy_buf()
214 for (i = 0; i < buf->page_count; i++) in relay_destroy_buf()
215 __free_page(buf->page_array[i]); in relay_destroy_buf()
216 relay_free_page_array(buf->page_array); in relay_destroy_buf()
218 *per_cpu_ptr(chan->buf, buf->cpu) = NULL; in relay_destroy_buf()
219 kfree(buf->padding); in relay_destroy_buf()
220 kfree(buf); in relay_destroy_buf()
234 struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref); in relay_remove_buf() local
235 relay_destroy_buf(buf); in relay_remove_buf()
244 static int relay_buf_empty(struct rchan_buf *buf) in relay_buf_empty() argument
246 return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1; in relay_buf_empty()
255 int relay_buf_full(struct rchan_buf *buf) in relay_buf_full() argument
257 size_t ready = buf->subbufs_produced - buf->subbufs_consumed; in relay_buf_full()
258 return (ready >= buf->chan->n_subbufs) ? 1 : 0; in relay_buf_full()
274 static int subbuf_start_default_callback (struct rchan_buf *buf, in subbuf_start_default_callback() argument
279 if (relay_buf_full(buf)) in subbuf_start_default_callback()
288 static void buf_mapped_default_callback(struct rchan_buf *buf, in buf_mapped_default_callback() argument
296 static void buf_unmapped_default_callback(struct rchan_buf *buf, in buf_unmapped_default_callback() argument
307 struct rchan_buf *buf, in create_buf_file_default_callback() argument
338 struct rchan_buf *buf; in wakeup_readers() local
340 buf = container_of(work, struct rchan_buf, wakeup_work); in wakeup_readers()
341 wake_up_interruptible(&buf->read_wait); in wakeup_readers()
351 static void __relay_reset(struct rchan_buf *buf, unsigned int init) in __relay_reset() argument
356 init_waitqueue_head(&buf->read_wait); in __relay_reset()
357 kref_init(&buf->kref); in __relay_reset()
358 init_irq_work(&buf->wakeup_work, wakeup_readers); in __relay_reset()
360 irq_work_sync(&buf->wakeup_work); in __relay_reset()
363 buf->subbufs_produced = 0; in __relay_reset()
364 buf->subbufs_consumed = 0; in __relay_reset()
365 buf->bytes_consumed = 0; in __relay_reset()
366 buf->finalized = 0; in __relay_reset()
367 buf->data = buf->start; in __relay_reset()
368 buf->offset = 0; in __relay_reset()
370 for (i = 0; i < buf->chan->n_subbufs; i++) in __relay_reset()
371 buf->padding[i] = 0; in __relay_reset()
373 buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0); in __relay_reset()
389 struct rchan_buf *buf; in relay_reset() local
395 if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) { in relay_reset()
396 __relay_reset(buf, 0); in relay_reset()
402 if ((buf = *per_cpu_ptr(chan->buf, i))) in relay_reset()
403 __relay_reset(buf, 0); in relay_reset()
408 static inline void relay_set_buf_dentry(struct rchan_buf *buf, in relay_set_buf_dentry() argument
411 buf->dentry = dentry; in relay_set_buf_dentry()
412 d_inode(buf->dentry)->i_size = buf->early_bytes; in relay_set_buf_dentry()
416 struct rchan_buf *buf, in relay_create_buf_file() argument
429 S_IRUSR, buf, in relay_create_buf_file()
446 struct rchan_buf *buf = NULL; in relay_open_buf() local
450 return *per_cpu_ptr(chan->buf, 0); in relay_open_buf()
452 buf = relay_create_buf(chan); in relay_open_buf()
453 if (!buf) in relay_open_buf()
457 dentry = relay_create_buf_file(chan, buf, cpu); in relay_open_buf()
460 relay_set_buf_dentry(buf, dentry); in relay_open_buf()
464 S_IRUSR, buf, in relay_open_buf()
470 buf->cpu = cpu; in relay_open_buf()
471 __relay_reset(buf, 1); in relay_open_buf()
474 *per_cpu_ptr(chan->buf, 0) = buf; in relay_open_buf()
475 buf->cpu = 0; in relay_open_buf()
478 return buf; in relay_open_buf()
481 relay_destroy_buf(buf); in relay_open_buf()
493 static void relay_close_buf(struct rchan_buf *buf) in relay_close_buf() argument
495 buf->finalized = 1; in relay_close_buf()
496 irq_work_sync(&buf->wakeup_work); in relay_close_buf()
497 buf->chan->cb->remove_buf_file(buf->dentry); in relay_close_buf()
498 kref_put(&buf->kref, relay_remove_buf); in relay_close_buf()
525 struct rchan_buf *buf; in relay_prepare_cpu() local
529 if ((buf = *per_cpu_ptr(chan->buf, cpu))) in relay_prepare_cpu()
531 buf = relay_open_buf(chan, cpu); in relay_prepare_cpu()
532 if (!buf) { in relay_prepare_cpu()
537 *per_cpu_ptr(chan->buf, cpu) = buf; in relay_prepare_cpu()
572 struct rchan_buf *buf; in relay_open() local
583 chan->buf = alloc_percpu(struct rchan_buf *); in relay_open()
599 buf = relay_open_buf(chan, i); in relay_open()
600 if (!buf) in relay_open()
602 *per_cpu_ptr(chan->buf, i) = buf; in relay_open()
611 if ((buf = *per_cpu_ptr(chan->buf, i))) in relay_open()
612 relay_close_buf(buf); in relay_open()
622 struct rchan_buf *buf; member
631 relay_set_buf_dentry(p->buf, p->dentry); in __relay_set_buf_dentry()
657 struct rchan_buf *buf; in relay_late_setup_files() local
676 buf = *per_cpu_ptr(chan->buf, 0); in relay_late_setup_files()
677 if (!WARN_ON_ONCE(!buf)) { in relay_late_setup_files()
678 dentry = relay_create_buf_file(chan, buf, 0); in relay_late_setup_files()
680 relay_set_buf_dentry(buf, dentry); in relay_late_setup_files()
695 buf = *per_cpu_ptr(chan->buf, i); in relay_late_setup_files()
696 if (unlikely(!buf)) { in relay_late_setup_files()
702 dentry = relay_create_buf_file(chan, buf, i); in relay_late_setup_files()
710 relay_set_buf_dentry(buf, dentry); in relay_late_setup_files()
713 disp.buf = buf; in relay_late_setup_files()
741 size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length) in relay_switch_subbuf() argument
746 if (unlikely(length > buf->chan->subbuf_size)) in relay_switch_subbuf()
749 if (buf->offset != buf->chan->subbuf_size + 1) { in relay_switch_subbuf()
750 buf->prev_padding = buf->chan->subbuf_size - buf->offset; in relay_switch_subbuf()
751 old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; in relay_switch_subbuf()
752 buf->padding[old_subbuf] = buf->prev_padding; in relay_switch_subbuf()
753 buf->subbufs_produced++; in relay_switch_subbuf()
754 if (buf->dentry) in relay_switch_subbuf()
755 d_inode(buf->dentry)->i_size += in relay_switch_subbuf()
756 buf->chan->subbuf_size - in relay_switch_subbuf()
757 buf->padding[old_subbuf]; in relay_switch_subbuf()
759 buf->early_bytes += buf->chan->subbuf_size - in relay_switch_subbuf()
760 buf->padding[old_subbuf]; in relay_switch_subbuf()
762 if (waitqueue_active(&buf->read_wait)) { in relay_switch_subbuf()
769 irq_work_queue(&buf->wakeup_work); in relay_switch_subbuf()
773 old = buf->data; in relay_switch_subbuf()
774 new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; in relay_switch_subbuf()
775 new = buf->start + new_subbuf * buf->chan->subbuf_size; in relay_switch_subbuf()
776 buf->offset = 0; in relay_switch_subbuf()
777 if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) { in relay_switch_subbuf()
778 buf->offset = buf->chan->subbuf_size + 1; in relay_switch_subbuf()
781 buf->data = new; in relay_switch_subbuf()
782 buf->padding[new_subbuf] = 0; in relay_switch_subbuf()
784 if (unlikely(length + buf->offset > buf->chan->subbuf_size)) in relay_switch_subbuf()
790 buf->chan->last_toobig = length; in relay_switch_subbuf()
812 struct rchan_buf *buf; in relay_subbufs_consumed() local
817 buf = *per_cpu_ptr(chan->buf, cpu); in relay_subbufs_consumed()
818 if (!buf || subbufs_consumed > chan->n_subbufs) in relay_subbufs_consumed()
821 if (subbufs_consumed > buf->subbufs_produced - buf->subbufs_consumed) in relay_subbufs_consumed()
822 buf->subbufs_consumed = buf->subbufs_produced; in relay_subbufs_consumed()
824 buf->subbufs_consumed += subbufs_consumed; in relay_subbufs_consumed()
836 struct rchan_buf *buf; in relay_close() local
843 if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) in relay_close()
844 relay_close_buf(buf); in relay_close()
847 if ((buf = *per_cpu_ptr(chan->buf, i))) in relay_close()
848 relay_close_buf(buf); in relay_close()
869 struct rchan_buf *buf; in relay_flush() local
875 if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) { in relay_flush()
876 relay_switch_subbuf(buf, 0); in relay_flush()
882 if ((buf = *per_cpu_ptr(chan->buf, i))) in relay_flush()
883 relay_switch_subbuf(buf, 0); in relay_flush()
897 struct rchan_buf *buf = inode->i_private; in relay_file_open() local
898 kref_get(&buf->kref); in relay_file_open()
899 filp->private_data = buf; in relay_file_open()
913 struct rchan_buf *buf = filp->private_data; in relay_file_mmap() local
914 return relay_mmap_buf(buf, vma); in relay_file_mmap()
927 struct rchan_buf *buf = filp->private_data; in relay_file_poll() local
929 if (buf->finalized) in relay_file_poll()
933 poll_wait(filp, &buf->read_wait, wait); in relay_file_poll()
934 if (!relay_buf_empty(buf)) in relay_file_poll()
951 struct rchan_buf *buf = filp->private_data; in relay_file_release() local
952 kref_put(&buf->kref, relay_remove_buf); in relay_file_release()
960 static void relay_file_read_consume(struct rchan_buf *buf, in relay_file_read_consume() argument
964 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_consume()
965 size_t n_subbufs = buf->chan->n_subbufs; in relay_file_read_consume()
968 if (buf->subbufs_produced == buf->subbufs_consumed && in relay_file_read_consume()
969 buf->offset == buf->bytes_consumed) in relay_file_read_consume()
972 if (buf->bytes_consumed + bytes_consumed > subbuf_size) { in relay_file_read_consume()
973 relay_subbufs_consumed(buf->chan, buf->cpu, 1); in relay_file_read_consume()
974 buf->bytes_consumed = 0; in relay_file_read_consume()
977 buf->bytes_consumed += bytes_consumed; in relay_file_read_consume()
979 read_subbuf = buf->subbufs_consumed % n_subbufs; in relay_file_read_consume()
981 read_subbuf = read_pos / buf->chan->subbuf_size; in relay_file_read_consume()
982 if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) { in relay_file_read_consume()
983 if ((read_subbuf == buf->subbufs_produced % n_subbufs) && in relay_file_read_consume()
984 (buf->offset == subbuf_size)) in relay_file_read_consume()
986 relay_subbufs_consumed(buf->chan, buf->cpu, 1); in relay_file_read_consume()
987 buf->bytes_consumed = 0; in relay_file_read_consume()
994 static int relay_file_read_avail(struct rchan_buf *buf, size_t read_pos) in relay_file_read_avail() argument
996 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_avail()
997 size_t n_subbufs = buf->chan->n_subbufs; in relay_file_read_avail()
998 size_t produced = buf->subbufs_produced; in relay_file_read_avail()
999 size_t consumed = buf->subbufs_consumed; in relay_file_read_avail()
1001 relay_file_read_consume(buf, read_pos, 0); in relay_file_read_avail()
1003 consumed = buf->subbufs_consumed; in relay_file_read_avail()
1005 if (unlikely(buf->offset > subbuf_size)) { in relay_file_read_avail()
1013 buf->subbufs_consumed = consumed; in relay_file_read_avail()
1014 buf->bytes_consumed = 0; in relay_file_read_avail()
1017 produced = (produced % n_subbufs) * subbuf_size + buf->offset; in relay_file_read_avail()
1018 consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed; in relay_file_read_avail()
1024 if (buf->offset == subbuf_size && in relay_file_read_avail()
1025 buf->subbufs_produced > buf->subbufs_consumed) in relay_file_read_avail()
1039 struct rchan_buf *buf) in relay_file_read_subbuf_avail() argument
1043 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_subbuf_avail()
1045 write_subbuf = (buf->data - buf->start) / subbuf_size; in relay_file_read_subbuf_avail()
1046 write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset; in relay_file_read_subbuf_avail()
1049 padding = buf->padding[read_subbuf]; in relay_file_read_subbuf_avail()
1070 struct rchan_buf *buf) in relay_file_read_start_pos() argument
1073 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_start_pos()
1074 size_t n_subbufs = buf->chan->n_subbufs; in relay_file_read_start_pos()
1075 size_t consumed = buf->subbufs_consumed % n_subbufs; in relay_file_read_start_pos()
1078 read_pos = consumed * subbuf_size + buf->bytes_consumed; in relay_file_read_start_pos()
1080 padding = buf->padding[read_subbuf]; in relay_file_read_start_pos()
1097 static size_t relay_file_read_end_pos(struct rchan_buf *buf, in relay_file_read_end_pos() argument
1102 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_end_pos()
1103 size_t n_subbufs = buf->chan->n_subbufs; in relay_file_read_end_pos()
1106 padding = buf->padding[read_subbuf]; in relay_file_read_end_pos()
1122 struct rchan_buf *buf = filp->private_data; in relay_file_read() local
1134 if (!relay_file_read_avail(buf, *ppos)) in relay_file_read()
1137 read_start = relay_file_read_start_pos(*ppos, buf); in relay_file_read()
1138 avail = relay_file_read_subbuf_avail(read_start, buf); in relay_file_read()
1143 from = buf->start + read_start; in relay_file_read()
1152 relay_file_read_consume(buf, read_start, ret); in relay_file_read()
1153 *ppos = relay_file_read_end_pos(buf, read_start, ret); in relay_file_read()
1171 struct pipe_buffer *buf) in relay_pipe_buf_release() argument
1175 rbuf = (struct rchan_buf *)page_private(buf->page); in relay_pipe_buf_release()
1176 relay_consume_bytes(rbuf, buf->private); in relay_pipe_buf_release()