Lines Matching +full:async +full:- +full:prefix
1 // SPDX-License-Identifier: GPL-2.0-only
6 * Copyright (C) 2007-2017 Google, Inc.
52 return list_entry(buffer->entry.next, struct binder_buffer, entry); in binder_buffer_next()
57 return list_entry(buffer->entry.prev, struct binder_buffer, entry); in binder_buffer_prev()
63 if (list_is_last(&buffer->entry, &alloc->buffers)) in binder_alloc_buffer_size()
64 return alloc->buffer + alloc->buffer_size - buffer->user_data; in binder_alloc_buffer_size()
65 return binder_buffer_next(buffer)->user_data - buffer->user_data; in binder_alloc_buffer_size()
71 struct rb_node **p = &alloc->free_buffers.rb_node; in binder_insert_free_buffer()
77 BUG_ON(!new_buffer->free); in binder_insert_free_buffer()
83 alloc->pid, new_buffer_size, new_buffer); in binder_insert_free_buffer()
88 BUG_ON(!buffer->free); in binder_insert_free_buffer()
93 p = &parent->rb_left; in binder_insert_free_buffer()
95 p = &parent->rb_right; in binder_insert_free_buffer()
97 rb_link_node(&new_buffer->rb_node, parent, p); in binder_insert_free_buffer()
98 rb_insert_color(&new_buffer->rb_node, &alloc->free_buffers); in binder_insert_free_buffer()
104 struct rb_node **p = &alloc->allocated_buffers.rb_node; in binder_insert_allocated_buffer_locked()
108 BUG_ON(new_buffer->free); in binder_insert_allocated_buffer_locked()
113 BUG_ON(buffer->free); in binder_insert_allocated_buffer_locked()
115 if (new_buffer->user_data < buffer->user_data) in binder_insert_allocated_buffer_locked()
116 p = &parent->rb_left; in binder_insert_allocated_buffer_locked()
117 else if (new_buffer->user_data > buffer->user_data) in binder_insert_allocated_buffer_locked()
118 p = &parent->rb_right; in binder_insert_allocated_buffer_locked()
122 rb_link_node(&new_buffer->rb_node, parent, p); in binder_insert_allocated_buffer_locked()
123 rb_insert_color(&new_buffer->rb_node, &alloc->allocated_buffers); in binder_insert_allocated_buffer_locked()
130 struct rb_node *n = alloc->allocated_buffers.rb_node; in binder_alloc_prepare_to_free_locked()
138 BUG_ON(buffer->free); in binder_alloc_prepare_to_free_locked()
140 if (uptr < buffer->user_data) in binder_alloc_prepare_to_free_locked()
141 n = n->rb_left; in binder_alloc_prepare_to_free_locked()
142 else if (uptr > buffer->user_data) in binder_alloc_prepare_to_free_locked()
143 n = n->rb_right; in binder_alloc_prepare_to_free_locked()
150 if (!buffer->allow_user_free) in binder_alloc_prepare_to_free_locked()
151 return ERR_PTR(-EPERM); in binder_alloc_prepare_to_free_locked()
152 buffer->allow_user_free = 0; in binder_alloc_prepare_to_free_locked()
160 * binder_alloc_prepare_to_free() - get buffer given user ptr
175 mutex_lock(&alloc->mutex); in binder_alloc_prepare_to_free()
177 mutex_unlock(&alloc->mutex); in binder_alloc_prepare_to_free()
192 "%d: %s pages %pK-%pK\n", alloc->pid, in binder_update_page_range()
204 page = &alloc->pages[(page_addr - alloc->buffer) / PAGE_SIZE]; in binder_update_page_range()
205 if (!page->page_ptr) { in binder_update_page_range()
211 if (need_mm && mmget_not_zero(alloc->vma_vm_mm)) in binder_update_page_range()
212 mm = alloc->vma_vm_mm; in binder_update_page_range()
216 vma = alloc->vma; in binder_update_page_range()
222 alloc->pid); in binder_update_page_range()
231 index = (page_addr - alloc->buffer) / PAGE_SIZE; in binder_update_page_range()
232 page = &alloc->pages[index]; in binder_update_page_range()
234 if (page->page_ptr) { in binder_update_page_range()
237 on_lru = list_lru_del(&binder_alloc_lru, &page->lru); in binder_update_page_range()
248 page->page_ptr = alloc_page(GFP_KERNEL | in binder_update_page_range()
251 if (!page->page_ptr) { in binder_update_page_range()
253 alloc->pid, page_addr); in binder_update_page_range()
256 page->alloc = alloc; in binder_update_page_range()
257 INIT_LIST_HEAD(&page->lru); in binder_update_page_range()
263 alloc->pid, user_page_addr); in binder_update_page_range()
267 if (index + 1 > alloc->pages_high) in binder_update_page_range()
268 alloc->pages_high = index + 1; in binder_update_page_range()
279 for (page_addr = end - PAGE_SIZE; 1; page_addr -= PAGE_SIZE) { in binder_update_page_range()
283 index = (page_addr - alloc->buffer) / PAGE_SIZE; in binder_update_page_range()
284 page = &alloc->pages[index]; in binder_update_page_range()
288 ret = list_lru_add(&binder_alloc_lru, &page->lru); in binder_update_page_range()
297 __free_page(page->page_ptr); in binder_update_page_range()
298 page->page_ptr = NULL; in binder_update_page_range()
309 return vma ? -ENOMEM : -ESRCH; in binder_update_page_range()
317 alloc->vma_vm_mm = vma->vm_mm; in binder_alloc_set_vma()
319 * If we see alloc->vma is not NULL, buffer data structures set up in binder_alloc_set_vma()
321 * We also want to guarantee new alloc->vma_vm_mm is always visible in binder_alloc_set_vma()
322 * if alloc->vma is set. in binder_alloc_set_vma()
325 alloc->vma = vma; in binder_alloc_set_vma()
333 if (alloc->vma) { in binder_alloc_get_vma()
336 vma = alloc->vma; in binder_alloc_get_vma()
346 * for the low async space is likely to try to send another async txn, in debug_low_async_space_locked()
355 for (n = rb_first(&alloc->allocated_buffers); n != NULL; in debug_low_async_space_locked()
358 if (buffer->pid != pid) in debug_low_async_space_locked()
360 if (!buffer->async_transaction) in debug_low_async_space_locked()
369 * async space (which is 25% of total buffer size). in debug_low_async_space_locked()
371 if (num_buffers > 50 || total_alloc_size > alloc->buffer_size / 4) { in debug_low_async_space_locked()
374 alloc->pid, pid, num_buffers, total_alloc_size); in debug_low_async_space_locked()
386 struct rb_node *n = alloc->free_buffers.rb_node; in binder_alloc_new_buf_locked()
398 alloc->pid); in binder_alloc_new_buf_locked()
399 return ERR_PTR(-ESRCH); in binder_alloc_new_buf_locked()
407 "%d: got transaction with invalid size %zd-%zd\n", in binder_alloc_new_buf_locked()
408 alloc->pid, data_size, offsets_size); in binder_alloc_new_buf_locked()
409 return ERR_PTR(-EINVAL); in binder_alloc_new_buf_locked()
415 alloc->pid, extra_buffers_size); in binder_alloc_new_buf_locked()
416 return ERR_PTR(-EINVAL); in binder_alloc_new_buf_locked()
419 alloc->free_async_space < size + sizeof(struct binder_buffer)) { in binder_alloc_new_buf_locked()
421 "%d: binder_alloc_buf size %zd failed, no async space left\n", in binder_alloc_new_buf_locked()
422 alloc->pid, size); in binder_alloc_new_buf_locked()
423 return ERR_PTR(-ENOSPC); in binder_alloc_new_buf_locked()
426 /* Pad 0-size buffers so they get assigned unique addresses */ in binder_alloc_new_buf_locked()
431 BUG_ON(!buffer->free); in binder_alloc_new_buf_locked()
436 n = n->rb_left; in binder_alloc_new_buf_locked()
438 n = n->rb_right; in binder_alloc_new_buf_locked()
452 for (n = rb_first(&alloc->allocated_buffers); n != NULL; in binder_alloc_new_buf_locked()
461 for (n = rb_first(&alloc->free_buffers); n != NULL; in binder_alloc_new_buf_locked()
472 alloc->pid, size); in binder_alloc_new_buf_locked()
478 return ERR_PTR(-ENOSPC); in binder_alloc_new_buf_locked()
487 alloc->pid, size, buffer, buffer_size); in binder_alloc_new_buf_locked()
490 (((uintptr_t)buffer->user_data + buffer_size) & PAGE_MASK); in binder_alloc_new_buf_locked()
493 (void __user *)PAGE_ALIGN((uintptr_t)buffer->user_data + size); in binder_alloc_new_buf_locked()
497 PAGE_ALIGN((uintptr_t)buffer->user_data), end_page_addr); in binder_alloc_new_buf_locked()
507 __func__, alloc->pid); in binder_alloc_new_buf_locked()
510 new_buffer->user_data = (u8 __user *)buffer->user_data + size; in binder_alloc_new_buf_locked()
511 list_add(&new_buffer->entry, &buffer->entry); in binder_alloc_new_buf_locked()
512 new_buffer->free = 1; in binder_alloc_new_buf_locked()
516 rb_erase(best_fit, &alloc->free_buffers); in binder_alloc_new_buf_locked()
517 buffer->free = 0; in binder_alloc_new_buf_locked()
518 buffer->allow_user_free = 0; in binder_alloc_new_buf_locked()
522 alloc->pid, size, buffer); in binder_alloc_new_buf_locked()
523 buffer->data_size = data_size; in binder_alloc_new_buf_locked()
524 buffer->offsets_size = offsets_size; in binder_alloc_new_buf_locked()
525 buffer->async_transaction = is_async; in binder_alloc_new_buf_locked()
526 buffer->extra_buffers_size = extra_buffers_size; in binder_alloc_new_buf_locked()
527 buffer->pid = pid; in binder_alloc_new_buf_locked()
529 alloc->free_async_space -= size + sizeof(struct binder_buffer); in binder_alloc_new_buf_locked()
531 "%d: binder_alloc_buf size %zd async free %zd\n", in binder_alloc_new_buf_locked()
532 alloc->pid, size, alloc->free_async_space); in binder_alloc_new_buf_locked()
533 if (alloc->free_async_space < alloc->buffer_size / 10) { in binder_alloc_new_buf_locked()
536 * of async space left (which is less than 10% of total in binder_alloc_new_buf_locked()
546 PAGE_ALIGN((uintptr_t)buffer->user_data), in binder_alloc_new_buf_locked()
548 return ERR_PTR(-ENOMEM); in binder_alloc_new_buf_locked()
552 * binder_alloc_new_buf() - Allocate a new binder buffer
556 * @extra_buffers_size: size of extra space for meta-data (eg, security context)
557 * @is_async: buffer for async transaction
563 * pointer-sized boundary)
565 * Return: The allocated buffer or %ERR_PTR(-errno) if error
576 mutex_lock(&alloc->mutex); in binder_alloc_new_buf()
579 mutex_unlock(&alloc->mutex); in binder_alloc_new_buf()
585 return (void __user *)((uintptr_t)buffer->user_data & PAGE_MASK); in buffer_start_page()
591 (((uintptr_t)(buffer->user_data) - 1) & PAGE_MASK); in prev_buffer_end_page()
600 BUG_ON(alloc->buffers.next == &buffer->entry); in binder_delete_free_buffer()
602 BUG_ON(!prev->free); in binder_delete_free_buffer()
607 alloc->pid, buffer->user_data, in binder_delete_free_buffer()
608 prev->user_data); in binder_delete_free_buffer()
611 if (!list_is_last(&buffer->entry, &alloc->buffers)) { in binder_delete_free_buffer()
617 alloc->pid, in binder_delete_free_buffer()
618 buffer->user_data, in binder_delete_free_buffer()
619 next->user_data); in binder_delete_free_buffer()
623 if (PAGE_ALIGNED(buffer->user_data)) { in binder_delete_free_buffer()
626 alloc->pid, buffer->user_data); in binder_delete_free_buffer()
633 alloc->pid, buffer->user_data, in binder_delete_free_buffer()
634 prev->user_data, in binder_delete_free_buffer()
635 next ? next->user_data : NULL); in binder_delete_free_buffer()
639 list_del(&buffer->entry); in binder_delete_free_buffer()
650 size = ALIGN(buffer->data_size, sizeof(void *)) + in binder_free_buf_locked()
651 ALIGN(buffer->offsets_size, sizeof(void *)) + in binder_free_buf_locked()
652 ALIGN(buffer->extra_buffers_size, sizeof(void *)); in binder_free_buf_locked()
656 alloc->pid, buffer, size, buffer_size); in binder_free_buf_locked()
658 BUG_ON(buffer->free); in binder_free_buf_locked()
660 BUG_ON(buffer->transaction != NULL); in binder_free_buf_locked()
661 BUG_ON(buffer->user_data < alloc->buffer); in binder_free_buf_locked()
662 BUG_ON(buffer->user_data > alloc->buffer + alloc->buffer_size); in binder_free_buf_locked()
664 if (buffer->async_transaction) { in binder_free_buf_locked()
665 alloc->free_async_space += buffer_size + sizeof(struct binder_buffer); in binder_free_buf_locked()
668 "%d: binder_free_buf size %zd async free %zd\n", in binder_free_buf_locked()
669 alloc->pid, size, alloc->free_async_space); in binder_free_buf_locked()
673 (void __user *)PAGE_ALIGN((uintptr_t)buffer->user_data), in binder_free_buf_locked()
675 buffer->user_data + buffer_size) & PAGE_MASK)); in binder_free_buf_locked()
677 rb_erase(&buffer->rb_node, &alloc->allocated_buffers); in binder_free_buf_locked()
678 buffer->free = 1; in binder_free_buf_locked()
679 if (!list_is_last(&buffer->entry, &alloc->buffers)) { in binder_free_buf_locked()
682 if (next->free) { in binder_free_buf_locked()
683 rb_erase(&next->rb_node, &alloc->free_buffers); in binder_free_buf_locked()
687 if (alloc->buffers.next != &buffer->entry) { in binder_free_buf_locked()
690 if (prev->free) { in binder_free_buf_locked()
692 rb_erase(&prev->rb_node, &alloc->free_buffers); in binder_free_buf_locked()
702 * binder_alloc_free_buf() - free a binder buffer
719 if (buffer->clear_on_free) { in binder_alloc_free_buf()
721 buffer->clear_on_free = false; in binder_alloc_free_buf()
723 mutex_lock(&alloc->mutex); in binder_alloc_free_buf()
725 mutex_unlock(&alloc->mutex); in binder_alloc_free_buf()
729 * binder_alloc_mmap_handler() - map virtual address space for proc
738 * -EBUSY = address space already mapped
739 * -ENOMEM = failed to map memory to given address space
749 if (alloc->buffer_size) { in binder_alloc_mmap_handler()
750 ret = -EBUSY; in binder_alloc_mmap_handler()
754 alloc->buffer_size = min_t(unsigned long, vma->vm_end - vma->vm_start, in binder_alloc_mmap_handler()
758 alloc->buffer = (void __user *)vma->vm_start; in binder_alloc_mmap_handler()
760 alloc->pages = kcalloc(alloc->buffer_size / PAGE_SIZE, in binder_alloc_mmap_handler()
761 sizeof(alloc->pages[0]), in binder_alloc_mmap_handler()
763 if (alloc->pages == NULL) { in binder_alloc_mmap_handler()
764 ret = -ENOMEM; in binder_alloc_mmap_handler()
771 ret = -ENOMEM; in binder_alloc_mmap_handler()
776 buffer->user_data = alloc->buffer; in binder_alloc_mmap_handler()
777 list_add(&buffer->entry, &alloc->buffers); in binder_alloc_mmap_handler()
778 buffer->free = 1; in binder_alloc_mmap_handler()
780 alloc->free_async_space = alloc->buffer_size / 2; in binder_alloc_mmap_handler()
782 mmgrab(alloc->vma_vm_mm); in binder_alloc_mmap_handler()
787 kfree(alloc->pages); in binder_alloc_mmap_handler()
788 alloc->pages = NULL; in binder_alloc_mmap_handler()
790 alloc->buffer = NULL; in binder_alloc_mmap_handler()
792 alloc->buffer_size = 0; in binder_alloc_mmap_handler()
796 "%s: %d %lx-%lx %s failed %d\n", __func__, in binder_alloc_mmap_handler()
797 alloc->pid, vma->vm_start, vma->vm_end, in binder_alloc_mmap_handler()
810 mutex_lock(&alloc->mutex); in binder_alloc_deferred_release()
811 BUG_ON(alloc->vma); in binder_alloc_deferred_release()
813 while ((n = rb_first(&alloc->allocated_buffers))) { in binder_alloc_deferred_release()
817 BUG_ON(buffer->transaction); in binder_alloc_deferred_release()
819 if (buffer->clear_on_free) { in binder_alloc_deferred_release()
821 buffer->clear_on_free = false; in binder_alloc_deferred_release()
827 while (!list_empty(&alloc->buffers)) { in binder_alloc_deferred_release()
828 buffer = list_first_entry(&alloc->buffers, in binder_alloc_deferred_release()
830 WARN_ON(!buffer->free); in binder_alloc_deferred_release()
832 list_del(&buffer->entry); in binder_alloc_deferred_release()
833 WARN_ON_ONCE(!list_empty(&alloc->buffers)); in binder_alloc_deferred_release()
838 if (alloc->pages) { in binder_alloc_deferred_release()
841 for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { in binder_alloc_deferred_release()
845 if (!alloc->pages[i].page_ptr) in binder_alloc_deferred_release()
849 &alloc->pages[i].lru); in binder_alloc_deferred_release()
850 page_addr = alloc->buffer + i * PAGE_SIZE; in binder_alloc_deferred_release()
853 __func__, alloc->pid, i, page_addr, in binder_alloc_deferred_release()
855 __free_page(alloc->pages[i].page_ptr); in binder_alloc_deferred_release()
858 kfree(alloc->pages); in binder_alloc_deferred_release()
860 mutex_unlock(&alloc->mutex); in binder_alloc_deferred_release()
861 if (alloc->vma_vm_mm) in binder_alloc_deferred_release()
862 mmdrop(alloc->vma_vm_mm); in binder_alloc_deferred_release()
866 __func__, alloc->pid, buffers, page_count); in binder_alloc_deferred_release()
869 static void print_binder_buffer(struct seq_file *m, const char *prefix, in print_binder_buffer() argument
873 prefix, buffer->debug_id, buffer->user_data, in print_binder_buffer()
874 buffer->data_size, buffer->offsets_size, in print_binder_buffer()
875 buffer->extra_buffers_size, in print_binder_buffer()
876 buffer->transaction ? "active" : "delivered"); in print_binder_buffer()
880 * binder_alloc_print_allocated() - print buffer info
892 mutex_lock(&alloc->mutex); in binder_alloc_print_allocated()
893 for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n)) in binder_alloc_print_allocated()
896 mutex_unlock(&alloc->mutex); in binder_alloc_print_allocated()
900 * binder_alloc_print_pages() - print page usage
913 mutex_lock(&alloc->mutex); in binder_alloc_print_pages()
919 for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { in binder_alloc_print_pages()
920 page = &alloc->pages[i]; in binder_alloc_print_pages()
921 if (!page->page_ptr) in binder_alloc_print_pages()
923 else if (list_empty(&page->lru)) in binder_alloc_print_pages()
929 mutex_unlock(&alloc->mutex); in binder_alloc_print_pages()
931 seq_printf(m, " pages high watermark: %zu\n", alloc->pages_high); in binder_alloc_print_pages()
935 * binder_alloc_get_allocated_count() - return count of buffers
945 mutex_lock(&alloc->mutex); in binder_alloc_get_allocated_count()
946 for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n)) in binder_alloc_get_allocated_count()
948 mutex_unlock(&alloc->mutex); in binder_alloc_get_allocated_count()
954 * binder_alloc_vma_close() - invalidate address space
958 * Clears alloc->vma to prevent new incoming transactions from
967 * binder_alloc_free_page() - shrinker callback to free pages
990 alloc = page->alloc; in binder_alloc_free_page()
991 if (!mutex_trylock(&alloc->mutex)) in binder_alloc_free_page()
994 if (!page->page_ptr) in binder_alloc_free_page()
997 index = page - alloc->pages; in binder_alloc_free_page()
998 page_addr = (uintptr_t)alloc->buffer + index * PAGE_SIZE; in binder_alloc_free_page()
1000 mm = alloc->vma_vm_mm; in binder_alloc_free_page()
1024 __free_page(page->page_ptr); in binder_alloc_free_page()
1025 page->page_ptr = NULL; in binder_alloc_free_page()
1030 mutex_unlock(&alloc->mutex); in binder_alloc_free_page()
1039 mutex_unlock(&alloc->mutex); in binder_alloc_free_page()
1057 NULL, sc->nr_to_scan); in binder_shrink_scan()
1068 * binder_alloc_init() - called by binder_open() for per-proc initialization
1076 alloc->pid = current->group_leader->pid; in binder_alloc_init()
1077 mutex_init(&alloc->mutex); in binder_alloc_init()
1078 INIT_LIST_HEAD(&alloc->buffers); in binder_alloc_init()
1100 * check_buffer() - verify that buffer/offset is safe to access
1112 * (buffer->free == 0 && buffer->allow_user_free == 0)
1114 * (buffer->free == 0 && buffer->transaction == NULL).
1125 offset <= buffer_size - bytes && in check_buffer()
1127 !buffer->free && in check_buffer()
1128 (!buffer->allow_user_free || !buffer->transaction); in check_buffer()
1132 * binder_alloc_get_page() - get kernel pointer for given buffer offset
1139 * at @buffer_offset into @buffer->user_data. If @pgoffp is not
1140 * NULL, the byte-offset into the page is written there.
1145 * guaranteed that the corresponding elements of @alloc->pages[]
1156 (buffer->user_data - alloc->buffer); in binder_alloc_get_page()
1161 lru_page = &alloc->pages[index]; in binder_alloc_get_page()
1163 return lru_page->page_ptr; in binder_alloc_get_page()
1167 * binder_alloc_clear_buf() - zero out buffer
1187 size = min_t(size_t, bytes, PAGE_SIZE - pgoff); in binder_alloc_clear_buf()
1191 bytes -= size; in binder_alloc_clear_buf()
1197 * binder_alloc_copy_user_to_buffer() - copy src user to tgt user
1227 size = min_t(size_t, bytes, PAGE_SIZE - pgoff); in binder_alloc_copy_user_to_buffer()
1232 return bytes - size + ret; in binder_alloc_copy_user_to_buffer()
1233 bytes -= size; in binder_alloc_copy_user_to_buffer()
1247 /* All copies must be 32-bit aligned and 32-bit size */ in binder_alloc_do_buffer_copy()
1249 return -EINVAL; in binder_alloc_do_buffer_copy()
1260 size = min_t(size_t, bytes, PAGE_SIZE - pgoff); in binder_alloc_do_buffer_copy()
1272 bytes -= size; in binder_alloc_do_buffer_copy()