Lines Matching +full:chg +full:- +full:int
1 // SPDX-License-Identifier: GPL-2.0-only
46 int hugetlb_max_hstate __read_mostly;
47 unsigned int default_hstate_idx;
59 static unsigned int minimum_order __read_mostly = UINT_MAX;
79 static int num_fault_mutexes;
84 return page_private(head + 4) == -1UL; in PageHugeFreed()
89 set_page_private(head + 4, -1UL); in SetPageHugeFreed()
98 static int hugetlb_acct_memory(struct hstate *h, long delta);
102 bool free = (spool->count == 0) && (spool->used_hpages == 0); in unlock_or_release_subpool()
104 spin_unlock(&spool->lock); in unlock_or_release_subpool()
110 if (spool->min_hpages != -1) in unlock_or_release_subpool()
111 hugetlb_acct_memory(spool->hstate, in unlock_or_release_subpool()
112 -spool->min_hpages); in unlock_or_release_subpool()
126 spin_lock_init(&spool->lock); in hugepage_new_subpool()
127 spool->count = 1; in hugepage_new_subpool()
128 spool->max_hpages = max_hpages; in hugepage_new_subpool()
129 spool->hstate = h; in hugepage_new_subpool()
130 spool->min_hpages = min_hpages; in hugepage_new_subpool()
132 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) { in hugepage_new_subpool()
136 spool->rsv_hpages = min_hpages; in hugepage_new_subpool()
143 spin_lock(&spool->lock); in hugepage_put_subpool()
144 BUG_ON(!spool->count); in hugepage_put_subpool()
145 spool->count--; in hugepage_put_subpool()
151 * Return -ENOMEM if there are not enough resources to satisfy the
165 spin_lock(&spool->lock); in hugepage_subpool_get_pages()
167 if (spool->max_hpages != -1) { /* maximum size accounting */ in hugepage_subpool_get_pages()
168 if ((spool->used_hpages + delta) <= spool->max_hpages) in hugepage_subpool_get_pages()
169 spool->used_hpages += delta; in hugepage_subpool_get_pages()
171 ret = -ENOMEM; in hugepage_subpool_get_pages()
177 if (spool->min_hpages != -1 && spool->rsv_hpages) { in hugepage_subpool_get_pages()
178 if (delta > spool->rsv_hpages) { in hugepage_subpool_get_pages()
183 ret = delta - spool->rsv_hpages; in hugepage_subpool_get_pages()
184 spool->rsv_hpages = 0; in hugepage_subpool_get_pages()
187 spool->rsv_hpages -= delta; in hugepage_subpool_get_pages()
192 spin_unlock(&spool->lock); in hugepage_subpool_get_pages()
210 spin_lock(&spool->lock); in hugepage_subpool_put_pages()
212 if (spool->max_hpages != -1) /* maximum size accounting */ in hugepage_subpool_put_pages()
213 spool->used_hpages -= delta; in hugepage_subpool_put_pages()
216 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) { in hugepage_subpool_put_pages()
217 if (spool->rsv_hpages + delta <= spool->min_hpages) in hugepage_subpool_put_pages()
220 ret = spool->rsv_hpages + delta - spool->min_hpages; in hugepage_subpool_put_pages()
222 spool->rsv_hpages += delta; in hugepage_subpool_put_pages()
223 if (spool->rsv_hpages > spool->min_hpages) in hugepage_subpool_put_pages()
224 spool->rsv_hpages = spool->min_hpages; in hugepage_subpool_put_pages()
238 return HUGETLBFS_SB(inode->i_sb)->spool; in subpool_inode()
243 return subpool_inode(file_inode(vma->vm_file)); in subpool_vma()
254 VM_BUG_ON(resv->region_cache_count <= 0); in get_file_region_entry_from_cache()
256 resv->region_cache_count--; in get_file_region_entry_from_cache()
257 nrg = list_first_entry(&resv->region_cache, struct file_region, link); in get_file_region_entry_from_cache()
258 list_del(&nrg->link); in get_file_region_entry_from_cache()
260 nrg->from = from; in get_file_region_entry_from_cache()
261 nrg->to = to; in get_file_region_entry_from_cache()
270 nrg->reservation_counter = rg->reservation_counter; in copy_hugetlb_cgroup_uncharge_info()
271 nrg->css = rg->css; in copy_hugetlb_cgroup_uncharge_info()
272 if (rg->css) in copy_hugetlb_cgroup_uncharge_info()
273 css_get(rg->css); in copy_hugetlb_cgroup_uncharge_info()
285 nrg->reservation_counter = in record_hugetlb_cgroup_uncharge_info()
286 &h_cg->rsvd_hugepage[hstate_index(h)]; in record_hugetlb_cgroup_uncharge_info()
287 nrg->css = &h_cg->css; in record_hugetlb_cgroup_uncharge_info()
289 * The caller will hold exactly one h_cg->css reference for the in record_hugetlb_cgroup_uncharge_info()
294 * exactly one h_cg->css reference, we should do css_get for in record_hugetlb_cgroup_uncharge_info()
298 css_get(&h_cg->css); in record_hugetlb_cgroup_uncharge_info()
299 if (!resv->pages_per_hpage) in record_hugetlb_cgroup_uncharge_info()
300 resv->pages_per_hpage = pages_per_huge_page(h); in record_hugetlb_cgroup_uncharge_info()
304 VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h)); in record_hugetlb_cgroup_uncharge_info()
306 nrg->reservation_counter = NULL; in record_hugetlb_cgroup_uncharge_info()
307 nrg->css = NULL; in record_hugetlb_cgroup_uncharge_info()
315 if (rg->css) in put_uncharge_info()
316 css_put(rg->css); in put_uncharge_info()
325 rg->reservation_counter == org->reservation_counter && in has_same_uncharge_info()
326 rg->css == org->css; in has_same_uncharge_info()
338 if (&prg->link != &resv->regions && prg->to == rg->from && in coalesce_file_region()
340 prg->to = rg->to; in coalesce_file_region()
342 list_del(&rg->link); in coalesce_file_region()
350 if (&nrg->link != &resv->regions && nrg->from == rg->to && in coalesce_file_region()
352 nrg->from = rg->from; in coalesce_file_region()
354 list_del(&rg->link); in coalesce_file_region()
361 * Must be called with resv->lock held.
373 struct list_head *head = &resv->regions; in add_reservation_in_range()
381 * [last_accounted_offset, rg->from), at every iteration, with some in add_reservation_in_range()
386 if (rg->from < f) { in add_reservation_in_range()
390 if (rg->to > last_accounted_offset) in add_reservation_in_range()
391 last_accounted_offset = rg->to; in add_reservation_in_range()
398 if (rg->from > t) in add_reservation_in_range()
401 /* Add an entry for last_accounted_offset -> rg->from, and in add_reservation_in_range()
404 if (rg->from > last_accounted_offset) { in add_reservation_in_range()
405 add += rg->from - last_accounted_offset; in add_reservation_in_range()
408 resv, last_accounted_offset, rg->from); in add_reservation_in_range()
411 list_add(&nrg->link, rg->link.prev); in add_reservation_in_range()
417 last_accounted_offset = rg->to; in add_reservation_in_range()
424 add += t - last_accounted_offset; in add_reservation_in_range()
429 list_add(&nrg->link, rg->link.prev); in add_reservation_in_range()
439 /* Must be called with resv->lock acquired. Will drop lock to allocate entries.
441 static int allocate_file_region_entries(struct resv_map *resv, in allocate_file_region_entries()
442 int regions_needed) in allocate_file_region_entries()
443 __must_hold(&resv->lock) in allocate_file_region_entries()
446 int to_allocate = 0, i = 0; in allocate_file_region_entries()
462 while (resv->region_cache_count < in allocate_file_region_entries()
463 (resv->adds_in_progress + regions_needed)) { in allocate_file_region_entries()
464 to_allocate = resv->adds_in_progress + regions_needed - in allocate_file_region_entries()
465 resv->region_cache_count; in allocate_file_region_entries()
471 VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress); in allocate_file_region_entries()
473 spin_unlock(&resv->lock); in allocate_file_region_entries()
478 list_add(&trg->link, &allocated_regions); in allocate_file_region_entries()
481 spin_lock(&resv->lock); in allocate_file_region_entries()
483 list_splice(&allocated_regions, &resv->region_cache); in allocate_file_region_entries()
484 resv->region_cache_count += to_allocate; in allocate_file_region_entries()
491 list_del(&rg->link); in allocate_file_region_entries()
494 return -ENOMEM; in allocate_file_region_entries()
509 * this operation and we were not able to allocate, it returns -ENOMEM.
520 spin_lock(&resv->lock); in region_add()
537 resv->region_cache_count < in region_add()
538 resv->adds_in_progress + in region_add()
539 (actual_regions_needed - in_regions_needed)) { in region_add()
543 VM_BUG_ON(t - f <= 1); in region_add()
546 resv, actual_regions_needed - in_regions_needed)) { in region_add()
547 return -ENOMEM; in region_add()
555 resv->adds_in_progress -= in_regions_needed; in region_add()
557 spin_unlock(&resv->lock); in region_add()
574 * resv->adds_in_progress. This value needs to be provided to a follow up call
579 * zero. -ENOMEM is returned if a new file_region structure or cache entry
585 long chg = 0; in region_chg() local
587 spin_lock(&resv->lock); in region_chg()
590 chg = add_reservation_in_range(resv, f, t, NULL, NULL, in region_chg()
597 return -ENOMEM; in region_chg()
599 resv->adds_in_progress += *out_regions_needed; in region_chg()
601 spin_unlock(&resv->lock); in region_chg()
602 return chg; in region_chg()
621 spin_lock(&resv->lock); in region_abort()
622 VM_BUG_ON(!resv->region_cache_count); in region_abort()
623 resv->adds_in_progress -= regions_needed; in region_abort()
624 spin_unlock(&resv->lock); in region_abort()
636 * be allocated. If the allocation fails, -ENOMEM will be returned.
638 * a region and possibly return -ENOMEM. Callers specifying
639 * t == LONG_MAX do not need to check for -ENOMEM error.
643 struct list_head *head = &resv->regions; in region_del()
649 spin_lock(&resv->lock); in region_del()
658 if (rg->to <= f && (rg->to != rg->from || rg->to != f)) in region_del()
661 if (rg->from >= t) in region_del()
664 if (f > rg->from && t < rg->to) { /* Must split region */ in region_del()
670 resv->region_cache_count > resv->adds_in_progress) { in region_del()
671 nrg = list_first_entry(&resv->region_cache, in region_del()
674 list_del(&nrg->link); in region_del()
675 resv->region_cache_count--; in region_del()
679 spin_unlock(&resv->lock); in region_del()
682 return -ENOMEM; in region_del()
686 del += t - f; in region_del()
688 resv, rg, t - f, false); in region_del()
691 nrg->from = t; in region_del()
692 nrg->to = rg->to; in region_del()
696 INIT_LIST_HEAD(&nrg->link); in region_del()
699 rg->to = f; in region_del()
701 list_add(&nrg->link, &rg->link); in region_del()
706 if (f <= rg->from && t >= rg->to) { /* Remove entire region */ in region_del()
707 del += rg->to - rg->from; in region_del()
709 rg->to - rg->from, true); in region_del()
710 list_del(&rg->link); in region_del()
715 if (f <= rg->from) { /* Trim beginning of region */ in region_del()
717 t - rg->from, false); in region_del()
719 del += t - rg->from; in region_del()
720 rg->from = t; in region_del()
723 rg->to - f, false); in region_del()
725 del += rg->to - f; in region_del()
726 rg->to = f; in region_del()
730 spin_unlock(&resv->lock); in region_del()
770 struct list_head *head = &resv->regions; in region_count()
772 long chg = 0; in region_count() local
774 spin_lock(&resv->lock); in region_count()
780 if (rg->to <= f) in region_count()
782 if (rg->from >= t) in region_count()
785 seg_from = max(rg->from, f); in region_count()
786 seg_to = min(rg->to, t); in region_count()
788 chg += seg_to - seg_from; in region_count()
790 spin_unlock(&resv->lock); in region_count()
792 return chg; in region_count()
802 return ((address - vma->vm_start) >> huge_page_shift(h)) + in vma_hugecache_offset()
803 (vma->vm_pgoff >> huge_page_order(h)); in vma_hugecache_offset()
819 if (vma->vm_ops && vma->vm_ops->pagesize) in vma_kernel_pagesize()
820 return vma->vm_ops->pagesize(vma); in vma_kernel_pagesize()
828 * architectures where it differs, an architecture-specific 'strong'
866 return (unsigned long)vma->vm_private_data; in get_vma_private_data()
872 vma->vm_private_data = (void *)value; in set_vma_private_data()
882 resv_map->reservation_counter = NULL; in resv_map_set_hugetlb_cgroup_uncharge_info()
883 resv_map->pages_per_hpage = 0; in resv_map_set_hugetlb_cgroup_uncharge_info()
884 resv_map->css = NULL; in resv_map_set_hugetlb_cgroup_uncharge_info()
886 resv_map->reservation_counter = in resv_map_set_hugetlb_cgroup_uncharge_info()
887 &h_cg->rsvd_hugepage[hstate_index(h)]; in resv_map_set_hugetlb_cgroup_uncharge_info()
888 resv_map->pages_per_hpage = pages_per_huge_page(h); in resv_map_set_hugetlb_cgroup_uncharge_info()
889 resv_map->css = &h_cg->css; in resv_map_set_hugetlb_cgroup_uncharge_info()
905 kref_init(&resv_map->refs); in resv_map_alloc()
906 spin_lock_init(&resv_map->lock); in resv_map_alloc()
907 INIT_LIST_HEAD(&resv_map->regions); in resv_map_alloc()
909 resv_map->adds_in_progress = 0; in resv_map_alloc()
913 * re-initialized to the proper values, to indicate that hugetlb cgroup in resv_map_alloc()
914 * reservations are to be un-charged from here. in resv_map_alloc()
918 INIT_LIST_HEAD(&resv_map->region_cache); in resv_map_alloc()
919 list_add(&rg->link, &resv_map->region_cache); in resv_map_alloc()
920 resv_map->region_cache_count = 1; in resv_map_alloc()
928 struct list_head *head = &resv_map->region_cache; in resv_map_release()
936 list_del(&rg->link); in resv_map_release()
940 VM_BUG_ON(resv_map->adds_in_progress); in resv_map_release()
952 * The VERY common case is inode->mapping == &inode->i_data but, in inode_resv_map()
955 return (struct resv_map *)(&inode->i_data)->private_data; in inode_resv_map()
961 if (vma->vm_flags & VM_MAYSHARE) { in vma_resv_map()
962 struct address_space *mapping = vma->vm_file->f_mapping; in vma_resv_map()
963 struct inode *inode = mapping->host; in vma_resv_map()
976 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); in set_vma_resv_map()
985 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); in set_vma_resv_flags()
990 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag) in is_vma_resv_set()
1001 if (!(vma->vm_flags & VM_MAYSHARE)) in reset_vma_resv_huge_pages()
1002 vma->vm_private_data = (void *)0; in reset_vma_resv_huge_pages()
1006 static bool vma_has_reserves(struct vm_area_struct *vma, long chg) in vma_has_reserves() argument
1008 if (vma->vm_flags & VM_NORESERVE) { in vma_has_reserves()
1010 * This address is already reserved by other process(chg == 0), in vma_has_reserves()
1016 * properly, so add work-around here. in vma_has_reserves()
1018 if (vma->vm_flags & VM_MAYSHARE && chg == 0) in vma_has_reserves()
1025 if (vma->vm_flags & VM_MAYSHARE) { in vma_has_reserves()
1031 * use. This situation is indicated if chg != 0. in vma_has_reserves()
1033 if (chg) in vma_has_reserves()
1047 * Examine the value of chg to determine if reserves in vma_has_reserves()
1049 * Very Subtle - The value of chg comes from a previous in vma_has_reserves()
1054 * account. Therefore, the meaning of chg is the same in vma_has_reserves()
1059 if (chg) in vma_has_reserves()
1070 int nid = page_to_nid(page); in enqueue_huge_page()
1071 list_move(&page->lru, &h->hugepage_freelists[nid]); in enqueue_huge_page()
1072 h->free_huge_pages++; in enqueue_huge_page()
1073 h->free_huge_pages_node[nid]++; in enqueue_huge_page()
1077 static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid) in dequeue_huge_page_node_exact()
1080 bool nocma = !!(current->flags & PF_MEMALLOC_NOCMA); in dequeue_huge_page_node_exact()
1082 list_for_each_entry(page, &h->hugepage_freelists[nid], lru) { in dequeue_huge_page_node_exact()
1089 list_move(&page->lru, &h->hugepage_activelist); in dequeue_huge_page_node_exact()
1092 h->free_huge_pages--; in dequeue_huge_page_node_exact()
1093 h->free_huge_pages_node[nid]--; in dequeue_huge_page_node_exact()
1100 static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid, in dequeue_huge_page_nodemask()
1103 unsigned int cpuset_mems_cookie; in dequeue_huge_page_nodemask()
1107 int node = NUMA_NO_NODE; in dequeue_huge_page_nodemask()
1138 unsigned long address, int avoid_reserve, in dequeue_huge_page_vma()
1139 long chg) in dequeue_huge_page_vma() argument
1145 int nid; in dequeue_huge_page_vma()
1152 if (!vma_has_reserves(vma, chg) && in dequeue_huge_page_vma()
1153 h->free_huge_pages - h->resv_huge_pages == 0) in dequeue_huge_page_vma()
1157 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0) in dequeue_huge_page_vma()
1163 if (page && !avoid_reserve && vma_has_reserves(vma, chg)) { in dequeue_huge_page_vma()
1165 h->resv_huge_pages--; in dequeue_huge_page_vma()
1178 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
1182 static int next_node_allowed(int nid, nodemask_t *nodes_allowed) in next_node_allowed()
1190 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed) in get_valid_node_allowed()
1203 static int hstate_next_node_to_alloc(struct hstate *h, in hstate_next_node_to_alloc()
1206 int nid; in hstate_next_node_to_alloc()
1210 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed); in hstate_next_node_to_alloc()
1211 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed); in hstate_next_node_to_alloc()
1217 * helper for free_pool_huge_page() - return the previously saved
1222 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed) in hstate_next_node_to_free()
1224 int nid; in hstate_next_node_to_free()
1228 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed); in hstate_next_node_to_free()
1229 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed); in hstate_next_node_to_free()
1238 nr_nodes--)
1244 nr_nodes--)
1248 unsigned int order) in destroy_compound_gigantic_page()
1250 int i; in destroy_compound_gigantic_page()
1251 int nr_pages = 1 << order; in destroy_compound_gigantic_page()
1267 static void free_gigantic_page(struct page *page, unsigned int order) in free_gigantic_page()
1283 int nid, nodemask_t *nodemask) in alloc_gigantic_page()
1292 int node; in alloc_gigantic_page()
1320 int nid, nodemask_t *nodemask) in alloc_gigantic_page()
1328 int nid, nodemask_t *nodemask) in alloc_gigantic_page()
1332 static inline void free_gigantic_page(struct page *page, unsigned int order) { } in free_gigantic_page()
1334 unsigned int order) { } in destroy_compound_gigantic_page()
1339 int i; in update_and_free_page()
1345 h->nr_huge_pages--; in update_and_free_page()
1346 h->nr_huge_pages_node[page_to_nid(page)]--; in update_and_free_page()
1349 subpage->flags &= ~(1 << PG_locked | 1 << PG_error | in update_and_free_page()
1359 * For non-gigantic pages set the destructor to the normal compound in update_and_free_page()
1402 * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
1403 * to hstate->hugepage_activelist.)
1434 return (unsigned long)page[2].mapping == -1U; in PageHugeTemporary()
1439 page[2].mapping = (void *)-1U; in SetPageHugeTemporary()
1454 int nid = page_to_nid(page); in __free_huge_page()
1463 page->mapping = NULL; in __free_huge_page()
1493 h->resv_huge_pages++; in __free_huge_page()
1496 list_del(&page->lru); in __free_huge_page()
1499 } else if (h->surplus_huge_pages_node[nid]) { in __free_huge_page()
1501 list_del(&page->lru); in __free_huge_page()
1503 h->surplus_huge_pages--; in __free_huge_page()
1504 h->surplus_huge_pages_node[nid]--; in __free_huge_page()
1513 * As free_huge_page() can be called from a non-task context, we have
1518 * be freed and frees them one-by-one. As the page->mapping pointer is
1534 node = node->next; in free_hpage_workfn()
1543 * Defer freeing if in non-task context to avoid hugetlb_lock deadlock. in free_huge_page()
1551 if (llist_add((struct llist_node *)&page->mapping, in free_huge_page()
1560 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) in prep_new_huge_page()
1562 INIT_LIST_HEAD(&page->lru); in prep_new_huge_page()
1567 h->nr_huge_pages++; in prep_new_huge_page()
1568 h->nr_huge_pages_node[nid]++; in prep_new_huge_page()
1573 static void prep_compound_gigantic_page(struct page *page, unsigned int order) in prep_compound_gigantic_page()
1575 int i; in prep_compound_gigantic_page()
1576 int nr_pages = 1 << order; in prep_compound_gigantic_page()
1586 * boot, it's safer to be consistent with the not-gigantic in prep_compound_gigantic_page()
1600 atomic_set(compound_mapcount_ptr(page), -1); in prep_compound_gigantic_page()
1609 int PageHuge(struct page *page) in PageHuge()
1623 int PageHeadHuge(struct page *page_head) in PageHeadHuge()
1658 compound_idx = page_to_pfn(page) - page_to_pfn(page_head); in hugetlb_basepage_index()
1660 compound_idx = page - page_head; in hugetlb_basepage_index()
1666 gfp_t gfp_mask, int nid, nodemask_t *nmask, in alloc_buddy_huge_page()
1669 int order = huge_page_order(h); in alloc_buddy_huge_page()
1717 gfp_t gfp_mask, int nid, nodemask_t *nmask, in alloc_fresh_huge_page()
1741 static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, in alloc_pool_huge_page()
1745 int nr_nodes, node; in alloc_pool_huge_page()
1769 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, in free_pool_huge_page()
1772 int nr_nodes, node; in free_pool_huge_page()
1773 int ret = 0; in free_pool_huge_page()
1780 if ((!acct_surplus || h->surplus_huge_pages_node[node]) && in free_pool_huge_page()
1781 !list_empty(&h->hugepage_freelists[node])) { in free_pool_huge_page()
1783 list_entry(h->hugepage_freelists[node].next, in free_pool_huge_page()
1785 list_del(&page->lru); in free_pool_huge_page()
1786 h->free_huge_pages--; in free_pool_huge_page()
1787 h->free_huge_pages_node[node]--; in free_pool_huge_page()
1789 h->surplus_huge_pages--; in free_pool_huge_page()
1790 h->surplus_huge_pages_node[node]--; in free_pool_huge_page()
1803 * nothing for in-use hugepages and non-hugepages.
1806 * -EBUSY: failed to dissolved free hugepages or the hugepage is in-use
1811 int dissolve_free_huge_page(struct page *page) in dissolve_free_huge_page()
1813 int rc = -EBUSY; in dissolve_free_huge_page()
1829 int nid = page_to_nid(head); in dissolve_free_huge_page()
1830 if (h->free_huge_pages - h->resv_huge_pages == 0) in dissolve_free_huge_page()
1842 * Theoretically, we should return -EBUSY when we in dissolve_free_huge_page()
1860 list_del(&head->lru); in dissolve_free_huge_page()
1861 h->free_huge_pages--; in dissolve_free_huge_page()
1862 h->free_huge_pages_node[nid]--; in dissolve_free_huge_page()
1863 h->max_huge_pages--; in dissolve_free_huge_page()
1880 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) in dissolve_free_huge_pages()
1884 int rc = 0; in dissolve_free_huge_pages()
1903 int nid, nodemask_t *nmask) in alloc_surplus_huge_page()
1911 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) in alloc_surplus_huge_page()
1927 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { in alloc_surplus_huge_page()
1933 h->surplus_huge_pages++; in alloc_surplus_huge_page()
1934 h->surplus_huge_pages_node[page_to_nid(page)]++; in alloc_surplus_huge_page()
1944 int nid, nodemask_t *nmask) in alloc_migrate_huge_page()
1974 int nid; in alloc_buddy_huge_page_with_mpol()
1985 struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid, in alloc_huge_page_nodemask()
1989 if (h->free_huge_pages - h->resv_huge_pages > 0) { in alloc_huge_page_nodemask()
2011 int node; in alloc_huge_page_vma()
2025 static int gather_surplus_pages(struct hstate *h, long delta) in gather_surplus_pages()
2030 int ret; in gather_surplus_pages()
2035 needed = (h->resv_huge_pages + delta) - h->free_huge_pages; in gather_surplus_pages()
2037 h->resv_huge_pages += delta; in gather_surplus_pages()
2044 ret = -ENOMEM; in gather_surplus_pages()
2054 list_add(&page->lru, &surplus_list); in gather_surplus_pages()
2064 needed = (h->resv_huge_pages + delta) - in gather_surplus_pages()
2065 (h->free_huge_pages + allocated); in gather_surplus_pages()
2085 h->resv_huge_pages += delta; in gather_surplus_pages()
2090 if ((--needed) < 0) in gather_surplus_pages()
2094 * no users -- drop the buddy allocator's reference. in gather_surplus_pages()
2136 * by pre-allocated pages. Only free surplus pages. in return_unused_surplus_pages()
2138 nr_pages = min(unused_resv_pages, h->surplus_huge_pages); in return_unused_surplus_pages()
2146 * on-line nodes with memory and will handle the hstate accounting. in return_unused_surplus_pages()
2152 while (nr_pages--) { in return_unused_surplus_pages()
2153 h->resv_huge_pages--; in return_unused_surplus_pages()
2154 unused_resv_pages--; in return_unused_surplus_pages()
2162 h->resv_huge_pages -= unused_resv_pages; in return_unused_surplus_pages()
2229 if (vma->vm_flags & VM_MAYSHARE) { in __vma_reservation_common()
2242 if (vma->vm_flags & VM_MAYSHARE) in __vma_reservation_common()
2250 * Subtle - The reserve map for private mappings has the in __vma_reservation_common()
2336 unsigned long addr, int avoid_reserve) in alloc_huge_page()
2343 int ret, idx; in alloc_huge_page()
2355 return ERR_PTR(-ENOMEM); in alloc_huge_page()
2368 return ERR_PTR(-ENOSPC); in alloc_huge_page()
2412 h->resv_huge_pages--; in alloc_huge_page()
2414 list_add(&page->lru, &h->hugepage_activelist); in alloc_huge_page()
2444 hugetlb_acct_memory(h, -rsv_adjust); in alloc_huge_page()
2461 return ERR_PTR(-ENOSPC); in alloc_huge_page()
2464 int alloc_bootmem_huge_page(struct hstate *h)
2466 int __alloc_bootmem_huge_page(struct hstate *h) in __alloc_bootmem_huge_page()
2469 int nr_nodes, node; in __alloc_bootmem_huge_page()
2492 INIT_LIST_HEAD(&m->list); in __alloc_bootmem_huge_page()
2493 list_add(&m->list, &huge_boot_pages); in __alloc_bootmem_huge_page()
2494 m->hstate = h; in __alloc_bootmem_huge_page()
2508 struct hstate *h = m->hstate; in gather_bootmem_prealloc()
2520 * other side-effects, like CommitLimit going negative. in gather_bootmem_prealloc()
2534 * Bit mask controlling how hard we retry per-node allocations. in hugetlb_hstate_alloc_pages()
2546 /* bit mask controlling how hard we retry per-node allocations */ in hugetlb_hstate_alloc_pages()
2550 for (i = 0; i < h->max_huge_pages; ++i) { in hugetlb_hstate_alloc_pages()
2564 if (i < h->max_huge_pages) { in hugetlb_hstate_alloc_pages()
2569 h->max_huge_pages, buf, i); in hugetlb_hstate_alloc_pages()
2570 h->max_huge_pages = i; in hugetlb_hstate_alloc_pages()
2599 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n", in report_hugepages()
2600 buf, h->free_huge_pages); in report_hugepages()
2608 int i; in try_to_free_low()
2615 struct list_head *freel = &h->hugepage_freelists[i]; in try_to_free_low()
2617 if (count >= h->nr_huge_pages) in try_to_free_low()
2621 list_del(&page->lru); in try_to_free_low()
2623 h->free_huge_pages--; in try_to_free_low()
2624 h->free_huge_pages_node[page_to_nid(page)]--; in try_to_free_low()
2636 * Increment or decrement surplus_huge_pages. Keep node-specific counters
2637 * balanced by operating on them in a round-robin fashion.
2640 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed, in adjust_pool_surplus()
2641 int delta) in adjust_pool_surplus()
2643 int nr_nodes, node; in adjust_pool_surplus()
2645 VM_BUG_ON(delta != -1 && delta != 1); in adjust_pool_surplus()
2649 if (h->surplus_huge_pages_node[node]) in adjust_pool_surplus()
2654 if (h->surplus_huge_pages_node[node] < in adjust_pool_surplus()
2655 h->nr_huge_pages_node[node]) in adjust_pool_surplus()
2662 h->surplus_huge_pages += delta; in adjust_pool_surplus()
2663 h->surplus_huge_pages_node[node] += delta; in adjust_pool_surplus()
2667 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2668 static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid, in set_max_huge_pages()
2675 * Bit mask controlling how hard we retry per-node allocations. in set_max_huge_pages()
2682 return -ENOMEM; in set_max_huge_pages()
2695 count += h->nr_huge_pages - h->nr_huge_pages_node[nid]; in set_max_huge_pages()
2717 return -EINVAL; in set_max_huge_pages()
2733 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) { in set_max_huge_pages()
2734 if (!adjust_pool_surplus(h, nodes_allowed, -1)) in set_max_huge_pages()
2755 /* Bail for signals. Probably ctrl-c from user */ in set_max_huge_pages()
2775 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages; in set_max_huge_pages()
2788 h->max_huge_pages = persistent_huge_pages(h); in set_max_huge_pages()
2806 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
2808 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp) in kobj_to_hstate()
2810 int i; in kobj_to_hstate()
2827 int nid; in nr_hugepages_show_common()
2831 nr_huge_pages = h->nr_huge_pages; in nr_hugepages_show_common()
2833 nr_huge_pages = h->nr_huge_pages_node[nid]; in nr_hugepages_show_common()
2839 struct hstate *h, int nid, in __nr_hugepages_store_common()
2842 int err; in __nr_hugepages_store_common()
2846 return -EINVAL; in __nr_hugepages_store_common()
2877 int nid; in nr_hugepages_store_common()
2878 int err; in nr_hugepages_store_common()
2904 * hstate attribute for optionally mempolicy-based constraint on persistent
2926 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages); in nr_overcommit_hugepages_show()
2932 int err; in nr_overcommit_hugepages_store()
2937 return -EINVAL; in nr_overcommit_hugepages_store()
2944 h->nr_overcommit_huge_pages = input; in nr_overcommit_hugepages_store()
2956 int nid; in free_hugepages_show()
2960 free_huge_pages = h->free_huge_pages; in free_hugepages_show()
2962 free_huge_pages = h->free_huge_pages_node[nid]; in free_hugepages_show()
2972 return sprintf(buf, "%lu\n", h->resv_huge_pages); in resv_hugepages_show()
2981 int nid; in surplus_hugepages_show()
2985 surplus_huge_pages = h->surplus_huge_pages; in surplus_hugepages_show()
2987 surplus_huge_pages = h->surplus_huge_pages_node[nid]; in surplus_hugepages_show()
3009 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, in hugetlb_sysfs_add_hstate()
3013 int retval; in hugetlb_sysfs_add_hstate()
3014 int hi = hstate_index(h); in hugetlb_sysfs_add_hstate()
3016 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); in hugetlb_sysfs_add_hstate()
3018 return -ENOMEM; in hugetlb_sysfs_add_hstate()
3032 int err; in hugetlb_sysfs_init()
3042 pr_err("HugeTLB: Unable to add hstate %s", h->name); in hugetlb_sysfs_init()
3049 * node_hstate/s - associate per node hstate attributes, via their kobjects,
3076 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
3077 * Returns node id via non-NULL nidp.
3079 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) in kobj_to_node_hstate()
3081 int nid; in kobj_to_node_hstate()
3085 int i; in kobj_to_node_hstate()
3087 if (nhs->hstate_kobjs[i] == kobj) { in kobj_to_node_hstate()
3100 * No-op if no hstate attributes attached.
3105 struct node_hstate *nhs = &node_hstates[node->dev.id]; in hugetlb_unregister_node()
3107 if (!nhs->hugepages_kobj) in hugetlb_unregister_node()
3111 int idx = hstate_index(h); in hugetlb_unregister_node()
3112 if (nhs->hstate_kobjs[idx]) { in hugetlb_unregister_node()
3113 kobject_put(nhs->hstate_kobjs[idx]); in hugetlb_unregister_node()
3114 nhs->hstate_kobjs[idx] = NULL; in hugetlb_unregister_node()
3118 kobject_put(nhs->hugepages_kobj); in hugetlb_unregister_node()
3119 nhs->hugepages_kobj = NULL; in hugetlb_unregister_node()
3125 * No-op if attributes already registered.
3130 struct node_hstate *nhs = &node_hstates[node->dev.id]; in hugetlb_register_node()
3131 int err; in hugetlb_register_node()
3133 if (nhs->hugepages_kobj) in hugetlb_register_node()
3136 nhs->hugepages_kobj = kobject_create_and_add("hugepages", in hugetlb_register_node()
3137 &node->dev.kobj); in hugetlb_register_node()
3138 if (!nhs->hugepages_kobj) in hugetlb_register_node()
3142 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, in hugetlb_register_node()
3143 nhs->hstate_kobjs, in hugetlb_register_node()
3147 h->name, node->dev.id); in hugetlb_register_node()
3156 * devices of nodes that have memory. All on-line nodes should have
3161 int nid; in hugetlb_register_all_nodes()
3165 if (node->dev.id == nid) in hugetlb_register_all_nodes()
3178 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) in kobj_to_node_hstate()
3182 *nidp = -1; in kobj_to_node_hstate()
3190 static int __init hugetlb_init(void) in hugetlb_init()
3192 int i; in hugetlb_init()
3196 pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n"); in hugetlb_init()
3262 void __init hugetlb_add_hstate(unsigned int order) in hugetlb_add_hstate()
3273 h->order = order; in hugetlb_add_hstate()
3274 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1); in hugetlb_add_hstate()
3275 h->nr_huge_pages = 0; in hugetlb_add_hstate()
3276 h->free_huge_pages = 0; in hugetlb_add_hstate()
3278 INIT_LIST_HEAD(&h->hugepage_freelists[i]); in hugetlb_add_hstate()
3279 INIT_LIST_HEAD(&h->hugepage_activelist); in hugetlb_add_hstate()
3280 h->next_nid_to_alloc = first_memory_node; in hugetlb_add_hstate()
3281 h->next_nid_to_free = first_memory_node; in hugetlb_add_hstate()
3282 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", in hugetlb_add_hstate()
3295 static int __init hugepages_setup(char *s) in hugepages_setup()
3315 mhp = &parsed_hstate->max_huge_pages; in hugepages_setup()
3330 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER) in hugepages_setup()
3346 static int __init hugepagesz_setup(char *s) in hugepagesz_setup()
3384 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); in hugepagesz_setup()
3394 static int __init default_hugepagesz_setup(char *s) in default_hugepagesz_setup()
3411 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); in default_hugepagesz_setup()
3434 static unsigned int allowed_mems_nr(struct hstate *h) in allowed_mems_nr()
3436 int node; in allowed_mems_nr()
3437 unsigned int nr = 0; in allowed_mems_nr()
3439 unsigned int *array = h->free_huge_pages_node; in allowed_mems_nr()
3454 static int proc_hugetlb_doulongvec_minmax(struct ctl_table *table, int write, in proc_hugetlb_doulongvec_minmax()
3470 static int hugetlb_sysctl_handler_common(bool obey_mempolicy, in hugetlb_sysctl_handler_common()
3471 struct ctl_table *table, int write, in hugetlb_sysctl_handler_common()
3475 unsigned long tmp = h->max_huge_pages; in hugetlb_sysctl_handler_common()
3476 int ret; in hugetlb_sysctl_handler_common()
3479 return -EOPNOTSUPP; in hugetlb_sysctl_handler_common()
3493 int hugetlb_sysctl_handler(struct ctl_table *table, int write, in hugetlb_sysctl_handler()
3502 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, in hugetlb_mempolicy_sysctl_handler()
3510 int hugetlb_overcommit_handler(struct ctl_table *table, int write, in hugetlb_overcommit_handler()
3515 int ret; in hugetlb_overcommit_handler()
3518 return -EOPNOTSUPP; in hugetlb_overcommit_handler()
3520 tmp = h->nr_overcommit_huge_pages; in hugetlb_overcommit_handler()
3523 return -EINVAL; in hugetlb_overcommit_handler()
3532 h->nr_overcommit_huge_pages = tmp; in hugetlb_overcommit_handler()
3550 unsigned long count = h->nr_huge_pages; in hugetlb_report_meminfo()
3562 h->free_huge_pages, in hugetlb_report_meminfo()
3563 h->resv_huge_pages, in hugetlb_report_meminfo()
3564 h->surplus_huge_pages, in hugetlb_report_meminfo()
3571 int hugetlb_report_node_meminfo(char *buf, int len, int nid) in hugetlb_report_node_meminfo()
3582 nid, h->nr_huge_pages_node[nid], in hugetlb_report_node_meminfo()
3583 nid, h->free_huge_pages_node[nid], in hugetlb_report_node_meminfo()
3584 nid, h->surplus_huge_pages_node[nid]); in hugetlb_report_node_meminfo()
3590 int nid; in hugetlb_show_meminfo()
3599 h->nr_huge_pages_node[nid], in hugetlb_show_meminfo()
3600 h->free_huge_pages_node[nid], in hugetlb_show_meminfo()
3601 h->surplus_huge_pages_node[nid], in hugetlb_show_meminfo()
3602 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); in hugetlb_show_meminfo()
3608 atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10)); in hugetlb_report_usage()
3618 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h); in hugetlb_total_pages()
3622 static int hugetlb_acct_memory(struct hstate *h, long delta) in hugetlb_acct_memory()
3624 int ret = -ENOMEM; in hugetlb_acct_memory()
3662 return_unused_surplus_pages(h, (unsigned long) -delta); in hugetlb_acct_memory()
3683 kref_get(&resv->refs); in hugetlb_vm_op_open()
3698 start = vma_hugecache_offset(h, vma, vma->vm_start); in hugetlb_vm_op_close()
3699 end = vma_hugecache_offset(h, vma, vma->vm_end); in hugetlb_vm_op_close()
3701 reserve = (end - start) - region_count(resv, start, end); in hugetlb_vm_op_close()
3709 hugetlb_acct_memory(h, -gbl_reserve); in hugetlb_vm_op_close()
3712 kref_put(&resv->refs, resv_map_release); in hugetlb_vm_op_close()
3715 static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr) in hugetlb_vm_op_split()
3718 return -EINVAL; in hugetlb_vm_op_split()
3731 * handle_mm_fault() to try to instantiate regular-sized pages in the
3757 int writable) in make_huge_pte()
3763 vma->vm_page_prot))); in make_huge_pte()
3766 vma->vm_page_prot)); in make_huge_pte()
3811 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, in copy_hugetlb_page_range()
3817 int cow; in copy_hugetlb_page_range()
3820 struct address_space *mapping = vma->vm_file->f_mapping; in copy_hugetlb_page_range()
3822 int ret = 0; in copy_hugetlb_page_range()
3824 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; in copy_hugetlb_page_range()
3828 vma->vm_start, in copy_hugetlb_page_range()
3829 vma->vm_end); in copy_hugetlb_page_range()
3841 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) { in copy_hugetlb_page_range()
3848 ret = -ENOMEM; in copy_hugetlb_page_range()
3926 struct mm_struct *mm = vma->vm_mm; in __unmap_hugepage_range()
4056 vma->vm_flags &= ~VM_MAYSHARE; in __unmap_hugepage_range_final()
4076 mm = vma->vm_mm; in unmap_hugepage_range()
4102 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + in unmap_ref_private()
4103 vma->vm_pgoff; in unmap_ref_private()
4104 mapping = vma->vm_file->f_mapping; in unmap_ref_private()
4112 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { in unmap_ref_private()
4122 if (iter_vma->vm_flags & VM_MAYSHARE) in unmap_ref_private()
4128 * areas. This is because a future no-page fault on this VMA in unmap_ref_private()
4152 int outside_reserve = 0; in hugetlb_cow()
4161 /* If no-one else is actually using this page, avoid the copy in hugetlb_cow()
4200 struct address_space *mapping = vma->vm_file->f_mapping; in hugetlb_cow()
4230 * race occurs while re-acquiring page table in hugetlb_cow()
4296 mapping = vma->vm_file->f_mapping; in hugetlbfs_pagecache_page()
4313 mapping = vma->vm_file->f_mapping; in hugetlbfs_pagecache_present()
4322 int huge_add_to_page_cache(struct page *page, struct address_space *mapping, in huge_add_to_page_cache()
4325 struct inode *inode = mapping->host; in huge_add_to_page_cache()
4327 int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL); in huge_add_to_page_cache()
4335 * by non-hugetlbfs specific code paths. in huge_add_to_page_cache()
4339 spin_lock(&inode->i_lock); in huge_add_to_page_cache()
4340 inode->i_blocks += blocks_per_huge_page(h); in huge_add_to_page_cache()
4341 spin_unlock(&inode->i_lock); in huge_add_to_page_cache()
4348 unsigned long address, pte_t *ptep, unsigned int flags) in hugetlb_no_page()
4352 int anon_rmap = 0; in hugetlb_no_page()
4368 current->pid); in hugetlb_no_page()
4377 size = i_size_read(mapping->host) >> huge_page_shift(h); in hugetlb_no_page()
4440 if (vma->vm_flags & VM_MAYSHARE) { in hugetlb_no_page()
4441 int err = huge_add_to_page_cache(page, mapping, idx); in hugetlb_no_page()
4444 if (err == -EEXIST) in hugetlb_no_page()
4475 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { in hugetlb_no_page()
4494 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) in hugetlb_no_page()
4495 && (vma->vm_flags & VM_SHARED))); in hugetlb_no_page()
4499 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { in hugetlb_no_page()
4540 return hash & (num_fault_mutexes - 1); in hugetlb_fault_mutex_hash()
4554 unsigned long address, unsigned int flags) in hugetlb_fault()
4565 int need_wait_lock = 0; in hugetlb_fault()
4595 mapping = vma->vm_file->f_mapping; in hugetlb_fault()
4648 if (!(vma->vm_flags & VM_MAYSHARE)) in hugetlb_fault()
4715 int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, in hugetlb_mcopy_atomic_pte()
4725 int vm_shared = dst_vma->vm_flags & VM_SHARED; in hugetlb_mcopy_atomic_pte()
4729 int ret; in hugetlb_mcopy_atomic_pte()
4734 * a non-missing case. Return -EEXIST. in hugetlb_mcopy_atomic_pte()
4738 ret = -EEXIST; in hugetlb_mcopy_atomic_pte()
4744 ret = -ENOMEM; in hugetlb_mcopy_atomic_pte()
4754 ret = -ENOENT; in hugetlb_mcopy_atomic_pte()
4771 mapping = dst_vma->vm_file->f_mapping; in hugetlb_mcopy_atomic_pte()
4778 size = i_size_read(mapping->host) >> huge_page_shift(h); in hugetlb_mcopy_atomic_pte()
4779 ret = -EFAULT; in hugetlb_mcopy_atomic_pte()
4806 size = i_size_read(mapping->host) >> huge_page_shift(h); in hugetlb_mcopy_atomic_pte()
4807 ret = -EFAULT; in hugetlb_mcopy_atomic_pte()
4811 ret = -EEXIST; in hugetlb_mcopy_atomic_pte()
4822 _dst_pte = make_huge_pte(dst_vma, page, dst_vma->vm_flags & VM_WRITE); in hugetlb_mcopy_atomic_pte()
4823 if (dst_vma->vm_flags & VM_WRITE) in hugetlb_mcopy_atomic_pte()
4830 dst_vma->vm_flags & VM_WRITE); in hugetlb_mcopy_atomic_pte()
4833 /* No need to invalidate - it was non-present before */ in hugetlb_mcopy_atomic_pte()
4855 long i, unsigned int flags, int *locked) in follow_hugetlb_page()
4861 int err = -EFAULT; in follow_hugetlb_page()
4863 while (vaddr < vma->vm_end && remainder) { in follow_hugetlb_page()
4866 int absent; in follow_hugetlb_page()
4920 unsigned int fault_flags = 0; in follow_hugetlb_page()
4935 * FAULT_FLAG_TRIED can co-exist in follow_hugetlb_page()
4972 (vaddr + huge_page_size(h) < vma->vm_end) && in follow_hugetlb_page()
4975 remainder -= pages_per_huge_page(h); in follow_hugetlb_page()
4997 err = -ENOMEM; in follow_hugetlb_page()
5007 --remainder; in follow_hugetlb_page()
5009 if (vaddr < vma->vm_end && remainder && in follow_hugetlb_page()
5041 struct mm_struct *mm = vma->vm_mm; in hugetlb_change_protection()
5063 i_mmap_lock_write(vma->vm_file->f_mapping); in hugetlb_change_protection()
5124 i_mmap_unlock_write(vma->vm_file->f_mapping); in hugetlb_change_protection()
5127 return pages << h->order; in hugetlb_change_protection()
5130 int hugetlb_reserve_pages(struct inode *inode, in hugetlb_reserve_pages()
5135 long ret, chg, add = -1; in hugetlb_reserve_pages() local
5145 return -EINVAL; in hugetlb_reserve_pages()
5159 * to reserve the full area even if read-only as mprotect() may be in hugetlb_reserve_pages()
5160 * called to make the mapping read-write. Assume !vma is a shm mapping in hugetlb_reserve_pages()
5162 if (!vma || vma->vm_flags & VM_MAYSHARE) { in hugetlb_reserve_pages()
5170 chg = region_chg(resv_map, from, to, ®ions_needed); in hugetlb_reserve_pages()
5176 return -ENOMEM; in hugetlb_reserve_pages()
5178 chg = to - from; in hugetlb_reserve_pages()
5184 if (chg < 0) { in hugetlb_reserve_pages()
5185 ret = chg; in hugetlb_reserve_pages()
5190 hstate_index(h), chg * pages_per_huge_page(h), &h_cg); in hugetlb_reserve_pages()
5193 ret = -ENOMEM; in hugetlb_reserve_pages()
5197 if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) { in hugetlb_reserve_pages()
5209 gbl_reserve = hugepage_subpool_get_pages(spool, chg); in hugetlb_reserve_pages()
5211 ret = -ENOSPC; in hugetlb_reserve_pages()
5229 * the reservation was consumed. Private mappings are per-VMA and in hugetlb_reserve_pages()
5235 if (!vma || vma->vm_flags & VM_MAYSHARE) { in hugetlb_reserve_pages()
5239 hugetlb_acct_memory(h, -gbl_reserve); in hugetlb_reserve_pages()
5242 } else if (unlikely(chg > add)) { in hugetlb_reserve_pages()
5254 * reference to h_cg->css. See comment below for detail. in hugetlb_reserve_pages()
5258 (chg - add) * pages_per_huge_page(h), h_cg); in hugetlb_reserve_pages()
5261 chg - add); in hugetlb_reserve_pages()
5262 hugetlb_acct_memory(h, -rsv_adjust); in hugetlb_reserve_pages()
5266 * h_cg->css. So we should release the reference held in hugetlb_reserve_pages()
5275 /* put back original number of pages, chg */ in hugetlb_reserve_pages()
5276 (void)hugepage_subpool_put_pages(spool, chg); in hugetlb_reserve_pages()
5279 chg * pages_per_huge_page(h), h_cg); in hugetlb_reserve_pages()
5281 if (!vma || vma->vm_flags & VM_MAYSHARE) in hugetlb_reserve_pages()
5285 if (chg >= 0 && add < 0) in hugetlb_reserve_pages()
5288 kref_put(&resv_map->refs, resv_map_release); in hugetlb_reserve_pages()
5297 long chg = 0; in hugetlb_unreserve_pages() local
5306 chg = region_del(resv_map, start, end); in hugetlb_unreserve_pages()
5312 if (chg < 0) in hugetlb_unreserve_pages()
5313 return chg; in hugetlb_unreserve_pages()
5316 spin_lock(&inode->i_lock); in hugetlb_unreserve_pages()
5317 inode->i_blocks -= (blocks_per_huge_page(h) * freed); in hugetlb_unreserve_pages()
5318 spin_unlock(&inode->i_lock); in hugetlb_unreserve_pages()
5324 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed)); in hugetlb_unreserve_pages()
5325 hugetlb_acct_memory(h, -gbl_reserve); in hugetlb_unreserve_pages()
5335 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + in page_table_shareable()
5336 svma->vm_start; in page_table_shareable()
5341 unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; in page_table_shareable()
5342 unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK; in page_table_shareable()
5350 sbase < svma->vm_start || svma->vm_end < s_end) in page_table_shareable()
5364 if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end)) in vma_shareable()
5377 unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE), in adjust_range_if_pmd_sharing_possible()
5378 v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE); in adjust_range_if_pmd_sharing_possible()
5384 if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) || in adjust_range_if_pmd_sharing_possible()
5416 struct address_space *mapping = vma->vm_file->f_mapping; in huge_pmd_share()
5417 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + in huge_pmd_share()
5418 vma->vm_pgoff; in huge_pmd_share()
5429 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { in huge_pmd_share()
5435 spte = huge_pte_offset(svma->vm_mm, saddr, in huge_pmd_share()
5473 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, in huge_pmd_unshare()
5480 i_mmap_assert_write_locked(vma->vm_file->f_mapping); in huge_pmd_unshare()
5495 *addr |= PUD_SIZE - PMD_SIZE; in huge_pmd_unshare()
5505 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, in huge_pmd_unshare()
5549 * huge_pte_offset() - Walk the page table to resolve the hugepage
5574 /* must be pud huge, non-present or none */ in huge_pte_offset()
5581 /* must be pmd huge, non-present or none */ in huge_pte_offset()
5593 int write) in follow_huge_addr()
5595 return ERR_PTR(-EINVAL); in follow_huge_addr()
5600 unsigned long address, hugepd_t hpd, int flags, int pdshift) in follow_huge_pd()
5607 follow_huge_pmd_pte(struct vm_area_struct *vma, unsigned long address, int flags) in follow_huge_pmd_pte()
5610 struct mm_struct *mm = vma->vm_mm; in follow_huge_pmd_pte()
5660 pud_t *pud, int flags) in follow_huge_pud()
5669 follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags) in follow_huge_pgd()
5677 int isolate_hugetlb(struct page *page, struct list_head *list) in isolate_hugetlb()
5679 int ret = 0; in isolate_hugetlb()
5684 ret = -EBUSY; in isolate_hugetlb()
5688 list_move_tail(&page->lru, list); in isolate_hugetlb()
5699 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); in putback_active_hugepage()
5704 void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason) in move_hugetlb_state()
5717 * Also note that we have to transfer the per-node surplus state in move_hugetlb_state()
5719 * the per-node's. in move_hugetlb_state()
5722 int old_nid = page_to_nid(oldpage); in move_hugetlb_state()
5723 int new_nid = page_to_nid(newpage); in move_hugetlb_state()
5729 if (h->surplus_huge_pages_node[old_nid]) { in move_hugetlb_state()
5730 h->surplus_huge_pages_node[old_nid]--; in move_hugetlb_state()
5731 h->surplus_huge_pages_node[new_nid]++; in move_hugetlb_state()
5740 static int __init cmdline_parse_hugetlb_cma(char *p) in cmdline_parse_hugetlb_cma()
5748 void __init hugetlb_cma_reserve(int order) in hugetlb_cma_reserve()
5751 int nid; in hugetlb_cma_reserve()
5774 int res; in hugetlb_cma_reserve()
5777 size = min(per_node, hugetlb_cma_size - reserved); in hugetlb_cma_reserve()