| /kernel/dma/ |
| D | direct.c | 33 static inline struct page *dma_direct_to_page(struct device *dev, in dma_direct_to_page()
98 static void __dma_direct_free_pages(struct device *dev, struct page *page, in __dma_direct_free_pages() argument
102 swiotlb_free(dev, page, size)) in __dma_direct_free_pages()
104 dma_free_contiguous(dev, page, size); in __dma_direct_free_pages()
107 static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, in __dma_direct_alloc_pages()
111 struct page *page = NULL; in __dma_direct_alloc_pages() local
120 page = swiotlb_alloc(dev, size); in __dma_direct_alloc_pages()
121 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { in __dma_direct_alloc_pages()
122 __dma_direct_free_pages(dev, page, size); in __dma_direct_alloc_pages()
125 return page; in __dma_direct_alloc_pages()
[all …]
|
| D | ops_helpers.c | 8 static struct page *dma_common_vaddr_to_page(void *cpu_addr) in dma_common_vaddr_to_page()
22 struct page *page = dma_common_vaddr_to_page(cpu_addr); in dma_common_get_sgtable() local
27 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); in dma_common_get_sgtable()
42 struct page *page = dma_common_vaddr_to_page(cpu_addr); in dma_common_mmap() local
54 page_to_pfn(page) + vma->vm_pgoff, in dma_common_mmap()
61 struct page *dma_common_alloc_pages(struct device *dev, size_t size, in dma_common_alloc_pages()
65 struct page *page; in dma_common_alloc_pages() local
67 page = dma_alloc_contiguous(dev, size, gfp); in dma_common_alloc_pages()
68 if (!page) in dma_common_alloc_pages()
69 page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size)); in dma_common_alloc_pages()
[all …]
|
| D | pool.c | 83 struct page *page = NULL; in atomic_pool_expand() local
93 page = dma_alloc_from_contiguous(NULL, 1 << order, in atomic_pool_expand()
95 if (!page) in atomic_pool_expand()
96 page = alloc_pages(gfp, order); in atomic_pool_expand()
97 } while (!page && order-- > 0); in atomic_pool_expand()
98 if (!page) in atomic_pool_expand()
101 arch_dma_prep_coherent(page, pool_size); in atomic_pool_expand()
104 addr = dma_common_contiguous_remap(page, pool_size, in atomic_pool_expand()
110 addr = page_to_virt(page); in atomic_pool_expand()
116 ret = set_memory_decrypted((unsigned long)page_to_virt(page), in atomic_pool_expand()
[all …]
|
| D | contiguous.c | 258 struct page *dma_alloc_from_contiguous(struct device *dev, size_t count, in dma_alloc_from_contiguous()
278 bool dma_release_from_contiguous(struct device *dev, struct page *pages, in dma_release_from_contiguous()
284 static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp) in cma_alloc_aligned()
307 struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) in dma_alloc_contiguous()
324 struct page *page; in dma_alloc_contiguous() local
327 page = cma_alloc_aligned(cma, size, gfp); in dma_alloc_contiguous()
328 if (page) in dma_alloc_contiguous()
329 return page; in dma_alloc_contiguous()
350 void dma_free_contiguous(struct device *dev, struct page *page, size_t size) in dma_free_contiguous() argument
356 if (cma_release(dev->cma_area, page, count)) in dma_free_contiguous()
[all …]
|
| D | remap.c | 9 struct page **dma_common_find_pages(void *cpu_addr) in dma_common_find_pages()
22 void *dma_common_pages_remap(struct page **pages, size_t size, in dma_common_pages_remap()
38 void *dma_common_contiguous_remap(struct page *page, size_t size, in dma_common_contiguous_remap() argument
42 struct page **pages; in dma_common_contiguous_remap()
46 pages = kvmalloc_array(count, sizeof(struct page *), GFP_KERNEL); in dma_common_contiguous_remap()
50 pages[i] = nth_page(page, i); in dma_common_contiguous_remap()
|
| D | mapping.c | 142 dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, in dma_map_page_attrs() argument
155 arch_dma_map_page_direct(dev, page_to_phys(page) + offset + size)) in dma_map_page_attrs()
156 addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); in dma_map_page_attrs()
158 addr = ops->map_page(dev, page, offset, size, dir, attrs); in dma_map_page_attrs()
159 debug_dma_map_page(dev, page, offset, size, dir, addr, attrs); in dma_map_page_attrs()
544 static struct page *__dma_alloc_pages(struct device *dev, size_t size, in __dma_alloc_pages()
562 struct page *dma_alloc_pages(struct device *dev, size_t size, in dma_alloc_pages()
565 struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp); in dma_alloc_pages() local
567 if (page) in dma_alloc_pages()
568 debug_dma_map_page(dev, page, 0, size, dir, *dma_handle, 0); in dma_alloc_pages()
[all …]
|
| D | debug.h | 12 extern void debug_dma_map_page(struct device *dev, struct page *page,
58 static inline void debug_dma_map_page(struct device *dev, struct page *page, in debug_dma_map_page() argument
|
| D | dummy.c | 14 static dma_addr_t dma_dummy_map_page(struct device *dev, struct page *page, in dma_dummy_map_page() argument
|
| D | direct.h | 84 struct page *page, unsigned long offset, size_t size, in dma_direct_map_page() argument
87 phys_addr_t phys = page_to_phys(page) + offset; in dma_direct_map_page()
|
| D | debug.c | 1052 struct page *page, size_t offset) in check_for_stack() argument
1059 if (PageHighMem(page)) in check_for_stack()
1061 addr = page_address(page) + offset; in check_for_stack()
1069 if (page != stack_vm_area->pages[i]) in check_for_stack()
1203 void debug_dma_map_page(struct device *dev, struct page *page, size_t offset, in debug_dma_map_page() argument
1221 entry->pfn = page_to_pfn(page); in debug_dma_map_page()
1228 check_for_stack(dev, page, offset); in debug_dma_map_page()
1230 if (!PageHighMem(page)) { in debug_dma_map_page()
1231 void *addr = page_address(page) + offset; in debug_dma_map_page()
|
| /kernel/power/ |
| D | snapshot.c | 86 static inline void hibernate_map_page(struct page *page) in hibernate_map_page() argument
89 int ret = set_direct_map_default_noflush(page); in hibernate_map_page()
94 debug_pagealloc_map_pages(page, 1); in hibernate_map_page()
98 static inline void hibernate_unmap_page(struct page *page) in hibernate_unmap_page() argument
101 unsigned long addr = (unsigned long)page_address(page); in hibernate_unmap_page()
102 int ret = set_direct_map_invalid_noflush(page); in hibernate_unmap_page()
109 debug_pagealloc_unmap_pages(page, 1); in hibernate_unmap_page()
113 static int swsusp_page_is_free(struct page *);
114 static void swsusp_set_page_forbidden(struct page *);
115 static void swsusp_unset_page_forbidden(struct page *);
[all …]
|
| D | swap.c | 248 struct page *page = bio_first_page_all(bio); in hib_end_io() local
257 put_page(page); in hib_end_io()
259 flush_icache_range((unsigned long)page_address(page), in hib_end_io()
260 (unsigned long)page_address(page) + PAGE_SIZE); in hib_end_io()
273 struct page *page = virt_to_page(addr); in hib_submit_io() local
282 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { in hib_submit_io()
688 unsigned char *page = NULL; in save_image_lzo() local
701 page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH); in save_image_lzo()
702 if (!page) { in save_image_lzo()
845 memcpy(page, data[thr].cmp + off, PAGE_SIZE); in save_image_lzo()
[all …]
|
| /kernel/ |
| D | kexec_core.c | 145 static struct page *kimage_alloc_page(struct kimage *image,
300 static struct page *kimage_alloc_pages(gfp_t gfp_mask, unsigned int order) in kimage_alloc_pages()
302 struct page *pages; in kimage_alloc_pages()
327 static void kimage_free_pages(struct page *page) in kimage_free_pages() argument
331 order = page_private(page); in kimage_free_pages()
334 arch_kexec_pre_free_pages(page_address(page), count); in kimage_free_pages()
337 ClearPageReserved(page + i); in kimage_free_pages()
338 __free_pages(page, order); in kimage_free_pages()
343 struct page *page, *next; in kimage_free_page_list() local
345 list_for_each_entry_safe(page, next, list, lru) { in kimage_free_page_list()
[all …]
|
| D | crash_core.c | 435 VMCOREINFO_STRUCT_SIZE(page); in crash_save_vmcoreinfo_init()
441 VMCOREINFO_OFFSET(page, flags); in crash_save_vmcoreinfo_init()
442 VMCOREINFO_OFFSET(page, _refcount); in crash_save_vmcoreinfo_init()
443 VMCOREINFO_OFFSET(page, mapping); in crash_save_vmcoreinfo_init()
444 VMCOREINFO_OFFSET(page, lru); in crash_save_vmcoreinfo_init()
445 VMCOREINFO_OFFSET(page, _mapcount); in crash_save_vmcoreinfo_init()
446 VMCOREINFO_OFFSET(page, private); in crash_save_vmcoreinfo_init()
447 VMCOREINFO_OFFSET(page, compound_dtor); in crash_save_vmcoreinfo_init()
448 VMCOREINFO_OFFSET(page, compound_order); in crash_save_vmcoreinfo_init()
449 VMCOREINFO_OFFSET(page, compound_head); in crash_save_vmcoreinfo_init()
|
| D | watch_queue.c | 62 struct page *page; in watch_queue_pipe_buf_release() local
74 page = buf->page; in watch_queue_pipe_buf_release()
75 bit += page->index; in watch_queue_pipe_buf_release()
104 struct page *page; in post_one_notification() local
123 page = wqueue->notes[note / WATCH_QUEUE_NOTES_PER_PAGE]; in post_one_notification()
125 get_page(page); in post_one_notification()
127 p = kmap_atomic(page); in post_one_notification()
132 buf->page = page; in post_one_notification()
245 struct page **pages; in watch_queue_set_size()
278 pages = kcalloc(sizeof(struct page *), nr_pages, GFP_KERNEL); in watch_queue_set_size()
|
| D | profile.c | 344 struct page *page; in profile_dead_cpu() local
352 page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[i]); in profile_dead_cpu()
354 __free_page(page); in profile_dead_cpu()
363 struct page *page; in profile_prepare_cpu() local
371 page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); in profile_prepare_cpu()
372 if (!page) { in profile_prepare_cpu()
376 per_cpu(cpu_profile_hits, cpu)[i] = page_address(page); in profile_prepare_cpu()
|
| D | relay.c | 35 struct page *page; in relay_buf_fault() local
42 page = vmalloc_to_page(buf->start + (pgoff << PAGE_SHIFT)); in relay_buf_fault()
43 if (!page) in relay_buf_fault()
45 get_page(page); in relay_buf_fault()
46 vmf->page = page; in relay_buf_fault()
61 static struct page **relay_alloc_page_array(unsigned int n_pages) in relay_alloc_page_array()
63 const size_t pa_size = n_pages * sizeof(struct page *); in relay_alloc_page_array()
72 static void relay_free_page_array(struct page **array) in relay_free_page_array()
1094 rbuf = (struct rchan_buf *)page_private(buf->page); in relay_pipe_buf_release()
1127 struct page *pages[PIPE_DEF_BUFFERS]; in subbuf_splice_actor()
|
| D | cfi.c | 74 unsigned long page = ptr >> PAGE_SHIFT; in ptr_to_shadow() local
76 if (unlikely(page < s->base)) in ptr_to_shadow()
79 index = page - s->base; in ptr_to_shadow()
|
| /kernel/events/ |
| D | ring_buffer.c | 235 handle->page = (offset >> page_shift) & (rb->nr_pages - 1); in __perf_output_begin()
237 handle->addr = rb->data_pages[handle->page] + offset; in __perf_output_begin()
605 static struct page *rb_alloc_aux_page(int node, int order) in rb_alloc_aux_page()
607 struct page *page; in rb_alloc_aux_page() local
613 page = alloc_pages_node(node, PERF_AUX_GFP, order); in rb_alloc_aux_page()
614 } while (!page && order--); in rb_alloc_aux_page()
616 if (page && order) { in rb_alloc_aux_page()
623 split_page(page, order); in rb_alloc_aux_page()
624 SetPagePrivate(page); in rb_alloc_aux_page()
625 set_page_private(page, order); in rb_alloc_aux_page()
[all …]
|
| D | uprobes.c | 104 struct page *pages[2];
155 struct page *old_page, struct page *new_page) in __replace_page()
159 .page = compound_head(old_page), in __replace_page()
244 static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len) in copy_from_page() argument
246 void *kaddr = kmap_atomic(page); in copy_from_page()
251 static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len) in copy_to_page() argument
253 void *kaddr = kmap_atomic(page); in copy_to_page()
258 static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode) in verify_opcode() argument
272 copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE); in verify_opcode()
371 struct page *page; in __update_ref_ctr() local
[all …]
|
| D | internal.h | 96 extern struct page *
151 handle->page++; \
152 handle->page &= rb->nr_pages - 1; \
153 handle->addr = rb->data_pages[handle->page]; \
|
| /kernel/trace/ |
| D | ring_buffer.c | 300 struct buffer_data_page *page; /* Actual data page */ member
325 return local_read(&bpage->page->commit); in rb_page_commit()
330 free_page((unsigned long)bpage->page); in free_buffer_page()
691 struct buffer_page *page = cpu_buffer->commit_page; in verify_event() local
701 if (page == tail_page || WARN_ON_ONCE(stop++ > 100)) in verify_event()
703 commit = local_read(&page->page->commit); in verify_event()
704 write = local_read(&page->write); in verify_event()
705 if (addr >= (unsigned long)&page->page->data[commit] && in verify_event()
706 addr < (unsigned long)&page->page->data[write]) in verify_event()
709 next = rb_list_head(page->list.next); in verify_event()
[all …]
|
| /kernel/bpf/ |
| D | ringbuf.c | 37 struct page **pages;
68 struct page **pages, *page; in bpf_ringbuf_area_alloc() local
96 page = alloc_pages_node(numa_node, flags, 0); in bpf_ringbuf_area_alloc()
97 if (!page) { in bpf_ringbuf_area_alloc()
101 pages[i] = page; in bpf_ringbuf_area_alloc()
103 pages[nr_data_pages + i] = page; in bpf_ringbuf_area_alloc()
186 struct page **pages = rb->pages; in bpf_ringbuf_free()
|
| /kernel/debug/kdb/ |
| D | kdb_support.c | 322 struct page *page; in kdb_getphys() local
327 page = pfn_to_page(pfn); in kdb_getphys()
328 vaddr = kmap_atomic(page); in kdb_getphys()
|
| /kernel/futex/ |
| D | core.c | 548 struct page *page, *tail; in get_futex_key() local
594 err = get_user_pages_fast(address, 1, FOLL_WRITE, &page); in get_futex_key()
600 err = get_user_pages_fast(address, 1, 0, &page); in get_futex_key()
626 tail = page; in get_futex_key()
627 page = compound_head(page); in get_futex_key()
628 mapping = READ_ONCE(page->mapping); in get_futex_key()
653 lock_page(page); in get_futex_key()
654 shmem_swizzled = PageSwapCache(page) || page->mapping; in get_futex_key()
655 unlock_page(page); in get_futex_key()
656 put_page(page); in get_futex_key()
[all …]
|