Lines Matching +full:trim +full:- +full:hs +full:- +full:current
1 // SPDX-License-Identifier: GPL-2.0-only
60 return cma_pages_valid(hugetlb_cma[folio_nid(folio)], &folio->page, in hugetlb_cma_folio()
104 if (spool->count) in subpool_is_free()
106 if (spool->max_hpages != -1) in subpool_is_free()
107 return spool->used_hpages == 0; in subpool_is_free()
108 if (spool->min_hpages != -1) in subpool_is_free()
109 return spool->rsv_hpages == spool->min_hpages; in subpool_is_free()
117 spin_unlock_irqrestore(&spool->lock, irq_flags); in unlock_or_release_subpool()
123 if (spool->min_hpages != -1) in unlock_or_release_subpool()
124 hugetlb_acct_memory(spool->hstate, in unlock_or_release_subpool()
125 -spool->min_hpages); in unlock_or_release_subpool()
139 spin_lock_init(&spool->lock); in hugepage_new_subpool()
140 spool->count = 1; in hugepage_new_subpool()
141 spool->max_hpages = max_hpages; in hugepage_new_subpool()
142 spool->hstate = h; in hugepage_new_subpool()
143 spool->min_hpages = min_hpages; in hugepage_new_subpool()
145 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) { in hugepage_new_subpool()
149 spool->rsv_hpages = min_hpages; in hugepage_new_subpool()
158 spin_lock_irqsave(&spool->lock, flags); in hugepage_put_subpool()
159 BUG_ON(!spool->count); in hugepage_put_subpool()
160 spool->count--; in hugepage_put_subpool()
166 * Return -ENOMEM if there are not enough resources to satisfy the
180 spin_lock_irq(&spool->lock); in hugepage_subpool_get_pages()
182 if (spool->max_hpages != -1) { /* maximum size accounting */ in hugepage_subpool_get_pages()
183 if ((spool->used_hpages + delta) <= spool->max_hpages) in hugepage_subpool_get_pages()
184 spool->used_hpages += delta; in hugepage_subpool_get_pages()
186 ret = -ENOMEM; in hugepage_subpool_get_pages()
192 if (spool->min_hpages != -1 && spool->rsv_hpages) { in hugepage_subpool_get_pages()
193 if (delta > spool->rsv_hpages) { in hugepage_subpool_get_pages()
198 ret = delta - spool->rsv_hpages; in hugepage_subpool_get_pages()
199 spool->rsv_hpages = 0; in hugepage_subpool_get_pages()
202 spool->rsv_hpages -= delta; in hugepage_subpool_get_pages()
207 spin_unlock_irq(&spool->lock); in hugepage_subpool_get_pages()
226 spin_lock_irqsave(&spool->lock, flags); in hugepage_subpool_put_pages()
228 if (spool->max_hpages != -1) /* maximum size accounting */ in hugepage_subpool_put_pages()
229 spool->used_hpages -= delta; in hugepage_subpool_put_pages()
232 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) { in hugepage_subpool_put_pages()
233 if (spool->rsv_hpages + delta <= spool->min_hpages) in hugepage_subpool_put_pages()
236 ret = spool->rsv_hpages + delta - spool->min_hpages; in hugepage_subpool_put_pages()
238 spool->rsv_hpages += delta; in hugepage_subpool_put_pages()
239 if (spool->rsv_hpages > spool->min_hpages) in hugepage_subpool_put_pages()
240 spool->rsv_hpages = spool->min_hpages; in hugepage_subpool_put_pages()
254 return HUGETLBFS_SB(inode->i_sb)->spool; in subpool_inode()
259 return subpool_inode(file_inode(vma->vm_file)); in subpool_vma()
268 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; in hugetlb_vma_lock_read()
270 down_read(&vma_lock->rw_sema); in hugetlb_vma_lock_read()
274 down_read(&resv_map->rw_sema); in hugetlb_vma_lock_read()
281 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; in hugetlb_vma_unlock_read()
283 up_read(&vma_lock->rw_sema); in hugetlb_vma_unlock_read()
287 up_read(&resv_map->rw_sema); in hugetlb_vma_unlock_read()
294 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; in hugetlb_vma_lock_write()
296 down_write(&vma_lock->rw_sema); in hugetlb_vma_lock_write()
300 down_write(&resv_map->rw_sema); in hugetlb_vma_lock_write()
307 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; in hugetlb_vma_unlock_write()
309 up_write(&vma_lock->rw_sema); in hugetlb_vma_unlock_write()
313 up_write(&resv_map->rw_sema); in hugetlb_vma_unlock_write()
321 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; in hugetlb_vma_trylock_write()
323 return down_write_trylock(&vma_lock->rw_sema); in hugetlb_vma_trylock_write()
327 return down_write_trylock(&resv_map->rw_sema); in hugetlb_vma_trylock_write()
336 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; in hugetlb_vma_assert_locked()
338 lockdep_assert_held(&vma_lock->rw_sema); in hugetlb_vma_assert_locked()
342 lockdep_assert_held(&resv_map->rw_sema); in hugetlb_vma_assert_locked()
356 struct vm_area_struct *vma = vma_lock->vma; in __hugetlb_vma_unlock_write_put()
361 * Semaphore synchronizes access to vma_lock->vma field. in __hugetlb_vma_unlock_write_put()
363 vma_lock->vma = NULL; in __hugetlb_vma_unlock_write_put()
364 vma->vm_private_data = NULL; in __hugetlb_vma_unlock_write_put()
365 up_write(&vma_lock->rw_sema); in __hugetlb_vma_unlock_write_put()
366 kref_put(&vma_lock->refs, hugetlb_vma_lock_release); in __hugetlb_vma_unlock_write_put()
372 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; in __hugetlb_vma_unlock_write_free()
379 up_write(&resv_map->rw_sema); in __hugetlb_vma_unlock_write_free()
391 if (vma->vm_private_data) { in hugetlb_vma_lock_free()
392 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; in hugetlb_vma_lock_free()
394 down_write(&vma_lock->rw_sema); in hugetlb_vma_lock_free()
404 if (!vma || !(vma->vm_flags & VM_MAYSHARE)) in hugetlb_vma_lock_alloc()
407 /* Should never get here with non-NULL vm_private_data */ in hugetlb_vma_lock_alloc()
408 if (vma->vm_private_data) in hugetlb_vma_lock_alloc()
427 kref_init(&vma_lock->refs); in hugetlb_vma_lock_alloc()
428 init_rwsem(&vma_lock->rw_sema); in hugetlb_vma_lock_alloc()
429 vma_lock->vma = vma; in hugetlb_vma_lock_alloc()
430 vma->vm_private_data = vma_lock; in hugetlb_vma_lock_alloc()
441 VM_BUG_ON(resv->region_cache_count <= 0); in get_file_region_entry_from_cache()
443 resv->region_cache_count--; in get_file_region_entry_from_cache()
444 nrg = list_first_entry(&resv->region_cache, struct file_region, link); in get_file_region_entry_from_cache()
445 list_del(&nrg->link); in get_file_region_entry_from_cache()
447 nrg->from = from; in get_file_region_entry_from_cache()
448 nrg->to = to; in get_file_region_entry_from_cache()
457 nrg->reservation_counter = rg->reservation_counter; in copy_hugetlb_cgroup_uncharge_info()
458 nrg->css = rg->css; in copy_hugetlb_cgroup_uncharge_info()
459 if (rg->css) in copy_hugetlb_cgroup_uncharge_info()
460 css_get(rg->css); in copy_hugetlb_cgroup_uncharge_info()
472 nrg->reservation_counter = in record_hugetlb_cgroup_uncharge_info()
473 &h_cg->rsvd_hugepage[hstate_index(h)]; in record_hugetlb_cgroup_uncharge_info()
474 nrg->css = &h_cg->css; in record_hugetlb_cgroup_uncharge_info()
476 * The caller will hold exactly one h_cg->css reference for the in record_hugetlb_cgroup_uncharge_info()
481 * exactly one h_cg->css reference, we should do css_get for in record_hugetlb_cgroup_uncharge_info()
485 css_get(&h_cg->css); in record_hugetlb_cgroup_uncharge_info()
486 if (!resv->pages_per_hpage) in record_hugetlb_cgroup_uncharge_info()
487 resv->pages_per_hpage = pages_per_huge_page(h); in record_hugetlb_cgroup_uncharge_info()
491 VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h)); in record_hugetlb_cgroup_uncharge_info()
493 nrg->reservation_counter = NULL; in record_hugetlb_cgroup_uncharge_info()
494 nrg->css = NULL; in record_hugetlb_cgroup_uncharge_info()
502 if (rg->css) in put_uncharge_info()
503 css_put(rg->css); in put_uncharge_info()
511 return rg->reservation_counter == org->reservation_counter && in has_same_uncharge_info()
512 rg->css == org->css; in has_same_uncharge_info()
524 if (&prg->link != &resv->regions && prg->to == rg->from && in coalesce_file_region()
526 prg->to = rg->to; in coalesce_file_region()
528 list_del(&rg->link); in coalesce_file_region()
536 if (&nrg->link != &resv->regions && nrg->from == rg->to && in coalesce_file_region()
538 nrg->from = rg->from; in coalesce_file_region()
540 list_del(&rg->link); in coalesce_file_region()
556 list_add(&nrg->link, rg); in hugetlb_resv_map_add()
561 return to - from; in hugetlb_resv_map_add()
565 * Must be called with resv->lock held.
577 struct list_head *head = &resv->regions; in add_reservation_in_range()
586 * [last_accounted_offset, iter->from), at every iteration, with some in add_reservation_in_range()
591 if (iter->from < f) { in add_reservation_in_range()
595 if (iter->to > last_accounted_offset) in add_reservation_in_range()
596 last_accounted_offset = iter->to; in add_reservation_in_range()
603 if (iter->from >= t) { in add_reservation_in_range()
604 rg = iter->link.prev; in add_reservation_in_range()
608 /* Add an entry for last_accounted_offset -> iter->from, and in add_reservation_in_range()
611 if (iter->from > last_accounted_offset) in add_reservation_in_range()
612 add += hugetlb_resv_map_add(resv, iter->link.prev, in add_reservation_in_range()
614 iter->from, h, h_cg, in add_reservation_in_range()
617 last_accounted_offset = iter->to; in add_reservation_in_range()
624 rg = head->prev; in add_reservation_in_range()
632 /* Must be called with resv->lock acquired. Will drop lock to allocate entries.
636 __must_hold(&resv->lock) in allocate_file_region_entries()
653 while (resv->region_cache_count < in allocate_file_region_entries()
654 (resv->adds_in_progress + regions_needed)) { in allocate_file_region_entries()
655 to_allocate = resv->adds_in_progress + regions_needed - in allocate_file_region_entries()
656 resv->region_cache_count; in allocate_file_region_entries()
662 VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress); in allocate_file_region_entries()
664 spin_unlock(&resv->lock); in allocate_file_region_entries()
669 list_add(&trg->link, &allocated_regions); in allocate_file_region_entries()
672 spin_lock(&resv->lock); in allocate_file_region_entries()
674 list_splice(&allocated_regions, &resv->region_cache); in allocate_file_region_entries()
675 resv->region_cache_count += to_allocate; in allocate_file_region_entries()
682 list_del(&rg->link); in allocate_file_region_entries()
685 return -ENOMEM; in allocate_file_region_entries()
700 * this operation and we were not able to allocate, it returns -ENOMEM.
711 spin_lock(&resv->lock); in region_add()
728 resv->region_cache_count < in region_add()
729 resv->adds_in_progress + in region_add()
730 (actual_regions_needed - in_regions_needed)) { in region_add()
734 VM_BUG_ON(t - f <= 1); in region_add()
737 resv, actual_regions_needed - in_regions_needed)) { in region_add()
738 return -ENOMEM; in region_add()
746 resv->adds_in_progress -= in_regions_needed; in region_add()
748 spin_unlock(&resv->lock); in region_add()
764 * resv->adds_in_progress. This value needs to be provided to a follow up call
769 * zero. -ENOMEM is returned if a new file_region structure or cache entry
777 spin_lock(&resv->lock); in region_chg()
787 return -ENOMEM; in region_chg()
789 resv->adds_in_progress += *out_regions_needed; in region_chg()
791 spin_unlock(&resv->lock); in region_chg()
811 spin_lock(&resv->lock); in region_abort()
812 VM_BUG_ON(!resv->region_cache_count); in region_abort()
813 resv->adds_in_progress -= regions_needed; in region_abort()
814 spin_unlock(&resv->lock); in region_abort()
821 * and either trim, delete or split the existing regions.
826 * be allocated. If the allocation fails, -ENOMEM will be returned.
828 * a region and possibly return -ENOMEM. Callers specifying
829 * t == LONG_MAX do not need to check for -ENOMEM error.
833 struct list_head *head = &resv->regions; in region_del()
839 spin_lock(&resv->lock); in region_del()
848 if (rg->to <= f && (rg->to != rg->from || rg->to != f)) in region_del()
851 if (rg->from >= t) in region_del()
854 if (f > rg->from && t < rg->to) { /* Must split region */ in region_del()
860 resv->region_cache_count > resv->adds_in_progress) { in region_del()
861 nrg = list_first_entry(&resv->region_cache, in region_del()
864 list_del(&nrg->link); in region_del()
865 resv->region_cache_count--; in region_del()
869 spin_unlock(&resv->lock); in region_del()
872 return -ENOMEM; in region_del()
876 del += t - f; in region_del()
878 resv, rg, t - f, false); in region_del()
881 nrg->from = t; in region_del()
882 nrg->to = rg->to; in region_del()
886 INIT_LIST_HEAD(&nrg->link); in region_del()
889 rg->to = f; in region_del()
891 list_add(&nrg->link, &rg->link); in region_del()
896 if (f <= rg->from && t >= rg->to) { /* Remove entire region */ in region_del()
897 del += rg->to - rg->from; in region_del()
899 rg->to - rg->from, true); in region_del()
900 list_del(&rg->link); in region_del()
905 if (f <= rg->from) { /* Trim beginning of region */ in region_del()
907 t - rg->from, false); in region_del()
909 del += t - rg->from; in region_del()
910 rg->from = t; in region_del()
911 } else { /* Trim end of region */ in region_del()
913 rg->to - f, false); in region_del()
915 del += rg->to - f; in region_del()
916 rg->to = f; in region_del()
920 spin_unlock(&resv->lock); in region_del()
960 struct list_head *head = &resv->regions; in region_count()
964 spin_lock(&resv->lock); in region_count()
970 if (rg->to <= f) in region_count()
972 if (rg->from >= t) in region_count()
975 seg_from = max(rg->from, f); in region_count()
976 seg_to = min(rg->to, t); in region_count()
978 chg += seg_to - seg_from; in region_count()
980 spin_unlock(&resv->lock); in region_count()
992 return ((address - vma->vm_start) >> huge_page_shift(h)) + in vma_hugecache_offset()
993 (vma->vm_pgoff >> huge_page_order(h)); in vma_hugecache_offset()
1004 * vma_kernel_pagesize - Page size granularity for this VMA.
1014 if (vma->vm_ops && vma->vm_ops->pagesize) in vma_kernel_pagesize()
1015 return vma->vm_ops->pagesize(vma); in vma_kernel_pagesize()
1023 * architectures where it differs, an architecture-specific 'strong'
1061 return (unsigned long)vma->vm_private_data; in get_vma_private_data()
1067 vma->vm_private_data = (void *)value; in set_vma_private_data()
1077 resv_map->reservation_counter = NULL; in resv_map_set_hugetlb_cgroup_uncharge_info()
1078 resv_map->pages_per_hpage = 0; in resv_map_set_hugetlb_cgroup_uncharge_info()
1079 resv_map->css = NULL; in resv_map_set_hugetlb_cgroup_uncharge_info()
1081 resv_map->reservation_counter = in resv_map_set_hugetlb_cgroup_uncharge_info()
1082 &h_cg->rsvd_hugepage[hstate_index(h)]; in resv_map_set_hugetlb_cgroup_uncharge_info()
1083 resv_map->pages_per_hpage = pages_per_huge_page(h); in resv_map_set_hugetlb_cgroup_uncharge_info()
1084 resv_map->css = &h_cg->css; in resv_map_set_hugetlb_cgroup_uncharge_info()
1100 kref_init(&resv_map->refs); in resv_map_alloc()
1101 spin_lock_init(&resv_map->lock); in resv_map_alloc()
1102 INIT_LIST_HEAD(&resv_map->regions); in resv_map_alloc()
1103 init_rwsem(&resv_map->rw_sema); in resv_map_alloc()
1105 resv_map->adds_in_progress = 0; in resv_map_alloc()
1109 * re-initialized to the proper values, to indicate that hugetlb cgroup in resv_map_alloc()
1110 * reservations are to be un-charged from here. in resv_map_alloc()
1114 INIT_LIST_HEAD(&resv_map->region_cache); in resv_map_alloc()
1115 list_add(&rg->link, &resv_map->region_cache); in resv_map_alloc()
1116 resv_map->region_cache_count = 1; in resv_map_alloc()
1124 struct list_head *head = &resv_map->region_cache; in resv_map_release()
1132 list_del(&rg->link); in resv_map_release()
1136 VM_BUG_ON(resv_map->adds_in_progress); in resv_map_release()
1148 * The VERY common case is inode->mapping == &inode->i_data but, in inode_resv_map()
1151 return (struct resv_map *)(&inode->i_data)->private_data; in inode_resv_map()
1157 if (vma->vm_flags & VM_MAYSHARE) { in vma_resv_map()
1158 struct address_space *mapping = vma->vm_file->f_mapping; in vma_resv_map()
1159 struct inode *inode = mapping->host; in vma_resv_map()
1172 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); in set_vma_resv_map()
1180 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); in set_vma_resv_flags()
1194 return !(vma->vm_flags & VM_MAYSHARE) && in __vma_private_lock()
1204 * - For shared mappings this is a per-vma semaphore that may be in hugetlb_dup_vma_private()
1210 * - For MAP_PRIVATE mappings, this is the reserve map which does in hugetlb_dup_vma_private()
1212 * not guaranteed to succeed, even if read-only. in hugetlb_dup_vma_private()
1214 if (vma->vm_flags & VM_MAYSHARE) { in hugetlb_dup_vma_private()
1215 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; in hugetlb_dup_vma_private()
1217 if (vma_lock && vma_lock->vma != vma) in hugetlb_dup_vma_private()
1218 vma->vm_private_data = NULL; in hugetlb_dup_vma_private()
1220 vma->vm_private_data = NULL; in hugetlb_dup_vma_private()
1225 * Called with mm->mmap_lock writer semaphore held.
1248 kref_put(&reservations->refs, resv_map_release); in clear_vma_resv_huge_pages()
1257 if (vma->vm_flags & VM_NORESERVE) { in vma_has_reserves()
1265 * properly, so add work-around here. in vma_has_reserves()
1267 if (vma->vm_flags & VM_MAYSHARE && chg == 0) in vma_has_reserves()
1274 if (vma->vm_flags & VM_MAYSHARE) { in vma_has_reserves()
1298 * Very Subtle - The value of chg comes from a previous in vma_has_reserves()
1324 list_move(&folio->lru, &h->hugepage_freelists[nid]); in enqueue_hugetlb_folio()
1325 h->free_huge_pages++; in enqueue_hugetlb_folio()
1326 h->free_huge_pages_node[nid]++; in enqueue_hugetlb_folio()
1334 bool pin = !!(current->flags & PF_MEMALLOC_PIN); in dequeue_hugetlb_folio_node_exact()
1337 list_for_each_entry(folio, &h->hugepage_freelists[nid], lru) { in dequeue_hugetlb_folio_node_exact()
1344 list_move(&folio->lru, &h->hugepage_activelist); in dequeue_hugetlb_folio_node_exact()
1347 h->free_huge_pages--; in dequeue_hugetlb_folio_node_exact()
1348 h->free_huge_pages_node[nid]--; in dequeue_hugetlb_folio_node_exact()
1393 return h->free_huge_pages - h->resv_huge_pages; in available_huge_pages()
1436 h->resv_huge_pages--; in dequeue_hugetlb_folio_vma()
1449 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
1481 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed); in hstate_next_node_to_alloc()
1482 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed); in hstate_next_node_to_alloc()
1488 * helper for remove_pool_huge_page() - return the previously saved
1499 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed); in hstate_next_node_to_free()
1500 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed); in hstate_next_node_to_free()
1505 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \ argument
1508 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \
1509 nr_nodes--)
1511 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \ argument
1514 ((node = hstate_next_node_to_free(hs, mask)) || 1); \
1515 nr_nodes--)
1517 /* used to demote non-gigantic_huge pages as well */
1525 atomic_set(&folio->_entire_mapcount, 0); in __destroy_compound_gigantic_folio()
1526 atomic_set(&folio->_nr_pages_mapped, 0); in __destroy_compound_gigantic_folio()
1527 atomic_set(&folio->_pincount, 0); in __destroy_compound_gigantic_folio()
1531 p->flags &= ~PAGE_FLAGS_CHECK_AT_FREE; in __destroy_compound_gigantic_folio()
1532 p->mapping = NULL; in __destroy_compound_gigantic_folio()
1563 if (cma_release(hugetlb_cma[nid], &folio->page, 1 << order)) in free_gigantic_folio()
1659 list_del(&folio->lru); in __remove_hugetlb_folio()
1662 h->free_huge_pages--; in __remove_hugetlb_folio()
1663 h->free_huge_pages_node[nid]--; in __remove_hugetlb_folio()
1666 h->surplus_huge_pages--; in __remove_hugetlb_folio()
1667 h->surplus_huge_pages_node[nid]--; in __remove_hugetlb_folio()
1685 h->nr_huge_pages--; in __remove_hugetlb_folio()
1686 h->nr_huge_pages_node[nid]--; in __remove_hugetlb_folio()
1711 INIT_LIST_HEAD(&folio->lru); in add_hugetlb_folio()
1712 h->nr_huge_pages++; in add_hugetlb_folio()
1713 h->nr_huge_pages_node[nid]++; in add_hugetlb_folio()
1716 h->surplus_huge_pages++; in add_hugetlb_folio()
1717 h->surplus_huge_pages_node[nid]++; in add_hugetlb_folio()
1743 arch_clear_hugepage_flags(&folio->page); in add_hugetlb_folio()
1760 if (hugetlb_vmemmap_restore(h, &folio->page)) { in __update_and_free_hugetlb_folio()
1790 * Non-gigantic pages demoted from CMA allocated gigantic pages in __update_and_free_hugetlb_folio()
1798 INIT_LIST_HEAD(&folio->_deferred_list); in __update_and_free_hugetlb_folio()
1799 __free_pages(&folio->page, huge_page_order(h)); in __update_and_free_hugetlb_folio()
1810 * freed and frees them one-by-one. As the page->mapping pointer is going
1828 node = node->next; in free_hpage_workfn()
1829 page->mapping = NULL; in free_hpage_workfn()
1866 if (llist_add((struct llist_node *)&folio->mapping, &hpage_freelist)) in update_and_free_hugetlb_folio()
1910 __ClearPageAnonExclusive(&folio->page); in free_huge_folio()
1911 folio->mapping = NULL; in free_huge_folio()
1941 h->resv_huge_pages++; in free_huge_folio()
1947 } else if (h->surplus_huge_pages_node[nid]) { in free_huge_folio()
1953 arch_clear_hugepage_flags(&folio->page); in free_huge_folio()
1965 h->nr_huge_pages++; in __prep_account_new_huge_page()
1966 h->nr_huge_pages_node[nid]++; in __prep_account_new_huge_page()
1971 hugetlb_vmemmap_optimize(h, &folio->page); in __prep_new_hugetlb_folio()
1972 INIT_LIST_HEAD(&folio->lru); in __prep_new_hugetlb_folio()
2000 * boot, it's safer to be consistent with the not-gigantic in __prep_compound_gigantic_folio()
2038 set_compound_head(p, &folio->page); in __prep_compound_gigantic_folio()
2043 atomic_set(&folio->_entire_mapcount, -1); in __prep_compound_gigantic_folio()
2044 atomic_set(&folio->_nr_pages_mapped, 0); in __prep_compound_gigantic_folio()
2045 atomic_set(&folio->_pincount, 0); in __prep_compound_gigantic_folio()
2103 compound_idx = page_to_pfn(page) - page_to_pfn(page_head); in hugetlb_basepage_index()
2105 compound_idx = page - page_head; in hugetlb_basepage_index()
2199 * Free pages and try again - ONCE! in alloc_fresh_hugetlb_folio()
2258 if ((!acct_surplus || h->surplus_huge_pages_node[node]) && in remove_pool_huge_page()
2259 !list_empty(&h->hugepage_freelists[node])) { in remove_pool_huge_page()
2260 page = list_entry(h->hugepage_freelists[node].next, in remove_pool_huge_page()
2273 * nothing for in-use hugepages and non-hugepages.
2276 * -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages
2280 * -EBUSY: failed to dissolved free hugepages or the hugepage is in-use
2287 int rc = -EBUSY; in dissolve_free_huge_page()
2315 * Theoretically, we should return -EBUSY when we in dissolve_free_huge_page()
2326 h->max_huge_pages--; in dissolve_free_huge_page()
2337 rc = hugetlb_vmemmap_restore(h, &folio->page); in dissolve_free_huge_page()
2343 h->max_huge_pages++; in dissolve_free_huge_page()
2399 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) in alloc_surplus_hugetlb_folio()
2415 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { in alloc_surplus_hugetlb_folio()
2422 h->surplus_huge_pages++; in alloc_surplus_hugetlb_folio()
2423 h->surplus_huge_pages_node[folio_nid(folio)]++; in alloc_surplus_hugetlb_folio()
2525 struct mempolicy *mpol = get_task_policy(current); in policy_mbind_nodemask()
2531 if (mpol->mode == MPOL_BIND && in policy_mbind_nodemask()
2533 cpuset_nodemask_valid_mems_allowed(&mpol->nodes))) in policy_mbind_nodemask()
2534 return &mpol->nodes; in policy_mbind_nodemask()
2556 needed = (h->resv_huge_pages + delta) - h->free_huge_pages; in gather_surplus_pages()
2558 h->resv_huge_pages += delta; in gather_surplus_pages()
2564 ret = -ENOMEM; in gather_surplus_pages()
2581 list_add(&folio->lru, &surplus_list); in gather_surplus_pages()
2591 needed = (h->resv_huge_pages + delta) - in gather_surplus_pages()
2592 (h->free_huge_pages + allocated); in gather_surplus_pages()
2612 h->resv_huge_pages += delta; in gather_surplus_pages()
2617 if ((--needed) < 0) in gather_surplus_pages()
2653 h->resv_huge_pages -= unused_resv_pages; in return_unused_surplus_pages()
2660 * by pre-allocated pages. Only free surplus pages. in return_unused_surplus_pages()
2662 nr_pages = min(unused_resv_pages, h->surplus_huge_pages); in return_unused_surplus_pages()
2670 * on-line nodes with memory and will handle the hstate accounting. in return_unused_surplus_pages()
2672 while (nr_pages--) { in return_unused_surplus_pages()
2677 list_add(&page->lru, &page_list); in return_unused_surplus_pages()
2756 if (vma->vm_flags & VM_MAYSHARE) { in __vma_reservation_common()
2766 if (vma->vm_flags & VM_MAYSHARE) { in __vma_reservation_common()
2779 if (vma->vm_flags & VM_MAYSHARE || mode == VMA_DEL_RESV) in __vma_reservation_common()
2788 * Subtle - The reserve map for private mappings has the in __vma_reservation_common()
2908 if (!(vma->vm_flags & VM_MAYSHARE)) in restore_reserve_on_error()
2927 * alloc_and_dissolve_hugetlb_folio - Allocate a new folio and dissolve
2951 return -ENOMEM; in alloc_and_dissolve_hugetlb_folio()
2966 * Fail with -EBUSY if not possible. in alloc_and_dissolve_hugetlb_folio()
2970 ret = isolated ? 0 : -EBUSY; in alloc_and_dissolve_hugetlb_folio()
3021 int ret = -EBUSY; in isolate_or_dissolve_huge_page()
3039 * alloc_contig_range and them. Return -ENOMEM as this has the effect in isolate_or_dissolve_huge_page()
3043 return -ENOMEM; in isolate_or_dissolve_huge_page()
3073 return ERR_PTR(-ENOMEM); in alloc_hugetlb_folio()
3086 return ERR_PTR(-ENOSPC); in alloc_hugetlb_folio()
3130 h->resv_huge_pages--; in alloc_hugetlb_folio()
3132 list_add(&folio->lru, &h->hugepage_activelist); in alloc_hugetlb_folio()
3164 hugetlb_acct_memory(h, -rsv_adjust); in alloc_hugetlb_folio()
3184 return ERR_PTR(-ENOSPC); in alloc_hugetlb_folio()
3219 INIT_LIST_HEAD(&m->list); in __alloc_bootmem_huge_page()
3220 list_add(&m->list, &huge_boot_pages); in __alloc_bootmem_huge_page()
3221 m->hstate = h; in __alloc_bootmem_huge_page()
3236 struct hstate *h = m->hstate; in gather_bootmem_prealloc()
3252 * other side-effects, like CommitLimit going negative. in gather_bootmem_prealloc()
3263 for (i = 0; i < h->max_huge_pages_node[nid]; ++i) { in hugetlb_hstate_alloc_pages_onenode()
3279 if (i == h->max_huge_pages_node[nid]) in hugetlb_hstate_alloc_pages_onenode()
3284 h->max_huge_pages_node[nid], buf, nid, i); in hugetlb_hstate_alloc_pages_onenode()
3285 h->max_huge_pages -= (h->max_huge_pages_node[nid] - i); in hugetlb_hstate_alloc_pages_onenode()
3286 h->max_huge_pages_node[nid] = i; in hugetlb_hstate_alloc_pages_onenode()
3303 if (h->max_huge_pages_node[i] > 0) { in hugetlb_hstate_alloc_pages()
3315 * Bit mask controlling how hard we retry per-node allocations. in hugetlb_hstate_alloc_pages()
3327 /* bit mask controlling how hard we retry per-node allocations */ in hugetlb_hstate_alloc_pages()
3331 for (i = 0; i < h->max_huge_pages; ++i) { in hugetlb_hstate_alloc_pages()
3341 if (i < h->max_huge_pages) { in hugetlb_hstate_alloc_pages()
3346 h->max_huge_pages, buf, i); in hugetlb_hstate_alloc_pages()
3347 h->max_huge_pages = i; in hugetlb_hstate_alloc_pages()
3363 * h->demote_order is initially 0. in hugetlb_init_hstates()
3364 * - We can not demote gigantic pages if runtime freeing in hugetlb_init_hstates()
3366 * - If CMA allocation is possible, we can not demote in hugetlb_init_hstates()
3371 if (hugetlb_cma_size && h->order <= HUGETLB_PAGE_ORDER) in hugetlb_init_hstates()
3376 if (h2->order < h->order && in hugetlb_init_hstates()
3377 h2->order > h->demote_order) in hugetlb_init_hstates()
3378 h->demote_order = h2->order; in hugetlb_init_hstates()
3391 pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n", in report_hugepages()
3392 buf, h->free_huge_pages); in report_hugepages()
3414 struct list_head *freel = &h->hugepage_freelists[i]; in try_to_free_low()
3416 if (count >= h->nr_huge_pages) in try_to_free_low()
3421 list_add(&page->lru, &page_list); in try_to_free_low()
3438 * Increment or decrement surplus_huge_pages. Keep node-specific counters
3439 * balanced by operating on them in a round-robin fashion.
3448 VM_BUG_ON(delta != -1 && delta != 1); in adjust_pool_surplus()
3452 if (h->surplus_huge_pages_node[node]) in adjust_pool_surplus()
3457 if (h->surplus_huge_pages_node[node] < in adjust_pool_surplus()
3458 h->nr_huge_pages_node[node]) in adjust_pool_surplus()
3465 h->surplus_huge_pages += delta; in adjust_pool_surplus()
3466 h->surplus_huge_pages_node[node] += delta; in adjust_pool_surplus()
3470 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
3480 * Bit mask controlling how hard we retry per-node allocations. in set_max_huge_pages()
3487 return -ENOMEM; in set_max_huge_pages()
3493 mutex_lock(&h->resize_lock); in set_max_huge_pages()
3506 count += h->nr_huge_pages - h->nr_huge_pages_node[nid]; in set_max_huge_pages()
3527 mutex_unlock(&h->resize_lock); in set_max_huge_pages()
3529 return -EINVAL; in set_max_huge_pages()
3545 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) { in set_max_huge_pages()
3546 if (!adjust_pool_surplus(h, nodes_allowed, -1)) in set_max_huge_pages()
3567 /* Bail for signals. Probably ctrl-c from user */ in set_max_huge_pages()
3568 if (signal_pending(current)) in set_max_huge_pages()
3587 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages; in set_max_huge_pages()
3599 list_add(&page->lru, &page_list); in set_max_huge_pages()
3612 h->max_huge_pages = persistent_huge_pages(h); in set_max_huge_pages()
3614 mutex_unlock(&h->resize_lock); in set_max_huge_pages()
3629 target_hstate = size_to_hstate(PAGE_SIZE << h->demote_order); in demote_free_hugetlb_folio()
3634 rc = hugetlb_vmemmap_restore(h, &folio->page); in demote_free_hugetlb_folio()
3654 * Note that we already hold h->resize_lock. To prevent deadlock, in demote_free_hugetlb_folio()
3657 mutex_lock(&target_hstate->resize_lock); in demote_free_hugetlb_folio()
3664 target_hstate->order); in demote_free_hugetlb_folio()
3666 prep_compound_page(subpage, target_hstate->order); in demote_free_hugetlb_folio()
3671 mutex_unlock(&target_hstate->resize_lock); in demote_free_hugetlb_folio()
3679 h->max_huge_pages--; in demote_free_hugetlb_folio()
3680 target_hstate->max_huge_pages += in demote_free_hugetlb_folio()
3695 if (!h->demote_order) { in demote_pool_huge_page()
3697 return -EINVAL; /* internal error */ in demote_pool_huge_page()
3701 list_for_each_entry(folio, &h->hugepage_freelists[node], lru) { in demote_pool_huge_page()
3710 * Return -EBUSY so that caller will not retry. in demote_pool_huge_page()
3712 return -EBUSY; in demote_pool_huge_page()
3752 nr_huge_pages = h->nr_huge_pages; in nr_hugepages_show_common()
3754 nr_huge_pages = h->nr_huge_pages_node[nid]; in nr_hugepages_show_common()
3767 return -EINVAL; in __nr_hugepages_store_common()
3825 * hstate attribute for optionally mempolicy-based constraint on persistent
3848 return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages); in nr_overcommit_hugepages_show()
3859 return -EINVAL; in nr_overcommit_hugepages_store()
3866 h->nr_overcommit_huge_pages = input; in nr_overcommit_hugepages_store()
3882 free_huge_pages = h->free_huge_pages; in free_hugepages_show()
3884 free_huge_pages = h->free_huge_pages_node[nid]; in free_hugepages_show()
3894 return sysfs_emit(buf, "%lu\n", h->resv_huge_pages); in resv_hugepages_show()
3907 surplus_huge_pages = h->surplus_huge_pages; in surplus_hugepages_show()
3909 surplus_huge_pages = h->surplus_huge_pages_node[nid]; in surplus_hugepages_show()
3938 mutex_lock(&h->resize_lock); in demote_store()
3947 nr_available = h->free_huge_pages_node[nid]; in demote_store()
3949 nr_available = h->free_huge_pages; in demote_store()
3950 nr_available -= h->resv_huge_pages; in demote_store()
3958 nr_demote--; in demote_store()
3962 mutex_unlock(&h->resize_lock); in demote_store()
3974 unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K; in demote_size_show()
3991 return -EINVAL; in demote_size_store()
3992 demote_order = demote_hstate->order; in demote_size_store()
3994 return -EINVAL; in demote_size_store()
3998 if (demote_order >= h->order) in demote_size_store()
3999 return -EINVAL; in demote_size_store()
4002 mutex_lock(&h->resize_lock); in demote_size_store()
4003 h->demote_order = demote_order; in demote_size_store()
4004 mutex_unlock(&h->resize_lock); in demote_size_store()
4043 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); in hugetlb_sysfs_add_hstate()
4045 return -ENOMEM; in hugetlb_sysfs_add_hstate()
4054 if (h->demote_order) { in hugetlb_sysfs_add_hstate()
4058 pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name); in hugetlb_sysfs_add_hstate()
4073 * node_hstate/s - associate per node hstate attributes, via their kobjects,
4100 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
4101 * Returns node id via non-NULL nidp.
4111 if (nhs->hstate_kobjs[i] == kobj) { in kobj_to_node_hstate()
4124 * No-op if no hstate attributes attached.
4129 struct node_hstate *nhs = &node_hstates[node->dev.id]; in hugetlb_unregister_node()
4131 if (!nhs->hugepages_kobj) in hugetlb_unregister_node()
4136 struct kobject *hstate_kobj = nhs->hstate_kobjs[idx]; in hugetlb_unregister_node()
4140 if (h->demote_order) in hugetlb_unregister_node()
4144 nhs->hstate_kobjs[idx] = NULL; in hugetlb_unregister_node()
4147 kobject_put(nhs->hugepages_kobj); in hugetlb_unregister_node()
4148 nhs->hugepages_kobj = NULL; in hugetlb_unregister_node()
4154 * No-op if attributes already registered.
4159 struct node_hstate *nhs = &node_hstates[node->dev.id]; in hugetlb_register_node()
4165 if (nhs->hugepages_kobj) in hugetlb_register_node()
4168 nhs->hugepages_kobj = kobject_create_and_add("hugepages", in hugetlb_register_node()
4169 &node->dev.kobj); in hugetlb_register_node()
4170 if (!nhs->hugepages_kobj) in hugetlb_register_node()
4174 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, in hugetlb_register_node()
4175 nhs->hstate_kobjs, in hugetlb_register_node()
4179 h->name, node->dev.id); in hugetlb_register_node()
4188 * devices of nodes that have memory. All on-line nodes should have
4204 *nidp = -1; in kobj_to_node_hstate()
4233 pr_err("HugeTLB: Unable to add hstate %s", h->name); in hugetlb_sysfs_init()
4257 pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n"); in hugetlb_init()
4338 __mutex_init(&h->resize_lock, "resize mutex", &h->resize_key); in hugetlb_add_hstate()
4339 h->order = order; in hugetlb_add_hstate()
4340 h->mask = ~(huge_page_size(h) - 1); in hugetlb_add_hstate()
4342 INIT_LIST_HEAD(&h->hugepage_freelists[i]); in hugetlb_add_hstate()
4343 INIT_LIST_HEAD(&h->hugepage_activelist); in hugetlb_add_hstate()
4344 h->next_nid_to_alloc = first_memory_node; in hugetlb_add_hstate()
4345 h->next_nid_to_free = first_memory_node; in hugetlb_add_hstate()
4346 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", in hugetlb_add_hstate()
4364 parsed_hstate->max_huge_pages = 0; in hugepages_clear_pages_in_node()
4365 memset(parsed_hstate->max_huge_pages_node, 0, in hugepages_clear_pages_in_node()
4366 sizeof(parsed_hstate->max_huge_pages_node)); in hugepages_clear_pages_in_node()
4401 mhp = &parsed_hstate->max_huge_pages; in hugepages_setup()
4428 parsed_hstate->max_huge_pages_node[node] = tmp; in hugepages_setup()
4507 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); in hugepagesz_setup()
4535 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); in default_hugepagesz_setup()
4566 unsigned int *array = h->free_huge_pages_node; in allowed_mems_nr()
4600 unsigned long tmp = h->max_huge_pages; in hugetlb_sysctl_handler_common()
4604 return -EOPNOTSUPP; in hugetlb_sysctl_handler_common()
4643 return -EOPNOTSUPP; in hugetlb_overcommit_handler()
4645 tmp = h->nr_overcommit_huge_pages; in hugetlb_overcommit_handler()
4648 return -EINVAL; in hugetlb_overcommit_handler()
4657 h->nr_overcommit_huge_pages = tmp; in hugetlb_overcommit_handler()
4713 unsigned long count = h->nr_huge_pages; in hugetlb_report_meminfo()
4725 h->free_huge_pages, in hugetlb_report_meminfo()
4726 h->resv_huge_pages, in hugetlb_report_meminfo()
4727 h->surplus_huge_pages, in hugetlb_report_meminfo()
4745 nid, h->nr_huge_pages_node[nid], in hugetlb_report_node_meminfo()
4746 nid, h->free_huge_pages_node[nid], in hugetlb_report_node_meminfo()
4747 nid, h->surplus_huge_pages_node[nid]); in hugetlb_report_node_meminfo()
4760 h->nr_huge_pages_node[nid], in hugetlb_show_meminfo_node()
4761 h->free_huge_pages_node[nid], in hugetlb_show_meminfo_node()
4762 h->surplus_huge_pages_node[nid], in hugetlb_show_meminfo_node()
4769 K(atomic_long_read(&mm->hugetlb_usage))); in hugetlb_report_usage()
4779 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h); in hugetlb_total_pages()
4785 int ret = -ENOMEM; in hugetlb_acct_memory()
4796 * current cpuset. Application can still potentially OOM'ed by kernel in hugetlb_acct_memory()
4804 * we fall back to check against current free page availability as in hugetlb_acct_memory()
4811 * the memory policy of the current task. Similar to the description in hugetlb_acct_memory()
4826 return_unused_surplus_pages(h, (unsigned long) -delta); in hugetlb_acct_memory()
4848 kref_get(&resv->refs); in hugetlb_vm_op_open()
4857 if (vma->vm_flags & VM_MAYSHARE) { in hugetlb_vm_op_open()
4858 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; in hugetlb_vm_op_open()
4861 if (vma_lock->vma != vma) { in hugetlb_vm_op_open()
4862 vma->vm_private_data = NULL; in hugetlb_vm_op_open()
4885 start = vma_hugecache_offset(h, vma, vma->vm_start); in hugetlb_vm_op_close()
4886 end = vma_hugecache_offset(h, vma, vma->vm_end); in hugetlb_vm_op_close()
4888 reserve = (end - start) - region_count(resv, start, end); in hugetlb_vm_op_close()
4896 hugetlb_acct_memory(h, -gbl_reserve); in hugetlb_vm_op_close()
4899 kref_put(&resv->refs, resv_map_release); in hugetlb_vm_op_close()
4905 return -EINVAL; in hugetlb_vm_op_split()
4912 * PMD sharing is only possible for PUD_SIZE-aligned address ranges in hugetlb_split()
4916 * MM, VMA and rmap all write-locked to prevent concurrent page table in hugetlb_split()
4920 i_mmap_assert_write_locked(vma->vm_file->f_mapping); in hugetlb_split()
4926 if (floor >= vma->vm_start && ceil <= vma->vm_end) { in hugetlb_split()
4949 * handle_mm_fault() to try to instantiate regular-sized pages in the
4982 vma->vm_page_prot))); in make_huge_pte()
4985 vma->vm_page_prot)); in make_huge_pte()
4988 entry = arch_make_huge_pte(entry, shift, vma->vm_flags); in make_huge_pte()
5033 pte_t newpte = make_huge_pte(vma, &new_folio->page, 1); in hugetlb_install_folio()
5039 set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz); in hugetlb_install_folio()
5040 hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm); in hugetlb_install_folio()
5051 bool cow = is_cow_mapping(src_vma->vm_flags); in copy_hugetlb_page_range()
5061 src_vma->vm_start, in copy_hugetlb_page_range()
5062 src_vma->vm_end); in copy_hugetlb_page_range()
5065 raw_write_seqcount_begin(&src->write_protect_seq); in copy_hugetlb_page_range()
5077 for (addr = src_vma->vm_start; addr < src_vma->vm_end; addr += sz) { in copy_hugetlb_page_range()
5086 ret = -ENOMEM; in copy_hugetlb_page_range()
5154 * When pre-allocating the page or copying data, we in copy_hugetlb_page_range()
5159 page_dup_file_rmap(&pte_folio->page, true); in copy_hugetlb_page_range()
5160 } else if (page_try_dup_anon_rmap(&pte_folio->page, in copy_hugetlb_page_range()
5225 raw_write_seqcount_end(&src->write_protect_seq); in copy_hugetlb_page_range()
5239 struct mm_struct *mm = vma->vm_mm; in move_huge_pte()
5267 struct address_space *mapping = vma->vm_file->f_mapping; in move_hugetlb_page_tables()
5269 struct mm_struct *mm = vma->vm_mm; in move_hugetlb_page_tables()
5317 flush_hugetlb_tlb_range(vma, old_end - len, old_end); in move_hugetlb_page_tables()
5322 return len + old_addr - old_end; in move_hugetlb_page_tables()
5329 struct mm_struct *mm = vma->vm_mm; in __unmap_hugepage_range()
5381 * If the pte was wr-protected by uffd-wp in any of the in __unmap_hugepage_range()
5383 * drop the uffd-wp bit in this zap, then replace the in __unmap_hugepage_range()
5420 /* Leave a uffd-wp pte marker if needed */ in __unmap_hugepage_range()
5459 if (!vma->vm_file) /* hugetlbfs_file_mmap error */ in __hugetlb_zap_begin()
5464 if (vma->vm_file) in __hugetlb_zap_begin()
5465 i_mmap_lock_write(vma->vm_file->f_mapping); in __hugetlb_zap_begin()
5471 zap_flags_t zap_flags = details ? details->zap_flags : 0; in __hugetlb_zap_end()
5473 if (!vma->vm_file) /* hugetlbfs_file_mmap error */ in __hugetlb_zap_end()
5491 if (vma->vm_file) in __hugetlb_zap_end()
5492 i_mmap_unlock_write(vma->vm_file->f_mapping); in __hugetlb_zap_end()
5502 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, in unmap_hugepage_range()
5506 tlb_gather_mmu(&tlb, vma->vm_mm); in unmap_hugepage_range()
5533 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + in unmap_ref_private()
5534 vma->vm_pgoff; in unmap_ref_private()
5535 mapping = vma->vm_file->f_mapping; in unmap_ref_private()
5543 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { in unmap_ref_private()
5544 /* Do not unmap the current VMA */ in unmap_ref_private()
5553 if (iter_vma->vm_flags & VM_MAYSHARE) in unmap_ref_private()
5559 * areas. This is because a future no-page fault on this VMA in unmap_ref_private()
5591 * Never handle CoW for uffd-wp protected pages. It should be only in hugetlb_wp()
5592 * handled when the uffd-wp protection is removed. in hugetlb_wp()
5596 * uffd-wp bit first. in hugetlb_wp()
5602 * hugetlb does not support FOLL_FORCE-style write faults that keep the in hugetlb_wp()
5605 if (WARN_ON_ONCE(!unshare && !(vma->vm_flags & VM_WRITE))) in hugetlb_wp()
5609 if (vma->vm_flags & VM_MAYSHARE) { in hugetlb_wp()
5620 * If no-one else is actually using this page, we're the exclusive in hugetlb_wp()
5624 if (!PageAnonExclusive(&old_folio->page)) in hugetlb_wp()
5625 page_move_anon_rmap(&old_folio->page, vma); in hugetlb_wp()
5633 PageAnonExclusive(&old_folio->page), &old_folio->page); in hugetlb_wp()
5666 struct address_space *mapping = vma->vm_file->f_mapping; in hugetlb_wp()
5685 unmap_ref_private(mm, vma, &old_folio->page, haddr); in hugetlb_wp()
5695 * race occurs while re-acquiring page table in hugetlb_wp()
5732 pte_t newpte = make_huge_pte(vma, &new_folio->page, !unshare); in hugetlb_wp()
5736 page_remove_rmap(&old_folio->page, vma, true); in hugetlb_wp()
5770 struct address_space *mapping = vma->vm_file->f_mapping; in hugetlbfs_pagecache_present()
5784 struct inode *inode = mapping->host; in hugetlb_add_to_page_cache()
5799 * by non-hugetlbfs specific code paths. in hugetlb_add_to_page_cache()
5803 spin_lock(&inode->i_lock); in hugetlb_add_to_page_cache()
5804 inode->i_blocks += blocks_per_huge_page(h); in hugetlb_add_to_page_cache()
5805 spin_unlock(&inode->i_lock); in hugetlb_add_to_page_cache()
5886 current->pid); in hugetlb_no_page()
5897 size = i_size_read(mapping->host) >> huge_page_shift(h); in hugetlb_no_page()
5904 * without pgtable lock, we need to re-test under in hugetlb_no_page()
5907 * either changed or during-changing ptes and retry in hugetlb_no_page()
5949 clear_huge_page(&folio->page, address, pages_per_huge_page(h)); in hugetlb_no_page()
5953 if (vma->vm_flags & VM_MAYSHARE) { in hugetlb_no_page()
5957 * err can't be -EEXIST which implies someone in hugetlb_no_page()
6009 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { in hugetlb_no_page()
6027 page_dup_file_rmap(&folio->page, true); in hugetlb_no_page()
6028 new_pte = make_huge_pte(vma, &folio->page, ((vma->vm_flags & VM_WRITE) in hugetlb_no_page()
6029 && (vma->vm_flags & VM_SHARED))); in hugetlb_no_page()
6031 * If this pte was previously wr-protected, keep it wr-protected even in hugetlb_no_page()
6039 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { in hugetlb_no_page()
6082 return hash & (num_fault_mutexes - 1); in hugetlb_fault_mutex_hash()
6121 mapping = vma->vm_file->f_mapping; in hugetlb_fault()
6196 !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(entry)) { in hugetlb_fault()
6215 /* Handle userfault-wp first, before trying to lock more pages */ in hugetlb_fault()
6279 * the page is not used after unlocked before returning from the current in hugetlb_fault()
6300 struct mm_struct *dst_mm = dst_vma->vm_mm; in hugetlb_mfill_atomic_pte()
6304 struct address_space *mapping = dst_vma->vm_file->f_mapping; in hugetlb_mfill_atomic_pte()
6307 int vm_shared = dst_vma->vm_flags & VM_SHARED; in hugetlb_mfill_atomic_pte()
6310 int ret = -ENOMEM; in hugetlb_mfill_atomic_pte()
6321 return -EEXIST; in hugetlb_mfill_atomic_pte()
6328 /* No need to invalidate - it was non-present before */ in hugetlb_mfill_atomic_pte()
6336 ret = -EFAULT; in hugetlb_mfill_atomic_pte()
6343 * a non-missing case. Return -EEXIST. in hugetlb_mfill_atomic_pte()
6347 ret = -EEXIST; in hugetlb_mfill_atomic_pte()
6353 ret = -ENOMEM; in hugetlb_mfill_atomic_pte()
6362 ret = -ENOENT; in hugetlb_mfill_atomic_pte()
6374 ret = -ENOMEM; in hugetlb_mfill_atomic_pte()
6388 ret = -EEXIST; in hugetlb_mfill_atomic_pte()
6396 ret = -ENOMEM; in hugetlb_mfill_atomic_pte()
6418 size = i_size_read(mapping->host) >> huge_page_shift(h); in hugetlb_mfill_atomic_pte()
6419 ret = -EFAULT; in hugetlb_mfill_atomic_pte()
6437 ret = -EIO; in hugetlb_mfill_atomic_pte()
6443 * registered, we firstly wr-protect a none pte which has no page cache in hugetlb_mfill_atomic_pte()
6446 ret = -EEXIST; in hugetlb_mfill_atomic_pte()
6451 page_dup_file_rmap(&folio->page, true); in hugetlb_mfill_atomic_pte()
6456 * For either: (1) CONTINUE on a non-shared VMA, or (2) UFFDIO_COPY in hugetlb_mfill_atomic_pte()
6462 writable = dst_vma->vm_flags & VM_WRITE; in hugetlb_mfill_atomic_pte()
6464 _dst_pte = make_huge_pte(dst_vma, &folio->page, writable); in hugetlb_mfill_atomic_pte()
6481 /* No need to invalidate - it was non-present before */ in hugetlb_mfill_atomic_pte()
6509 struct mm_struct *mm = vma->vm_mm; in hugetlb_follow_page_mask()
6534 page = ERR_PTR(-EMLINK); in hugetlb_follow_page_mask()
6542 * Note that page may be a sub-page, and with vmemmap in hugetlb_follow_page_mask()
6557 *page_mask = (1U << huge_page_order(h)) - 1; in hugetlb_follow_page_mask()
6570 page = ERR_PTR(-EFAULT); in hugetlb_follow_page_mask()
6579 struct mm_struct *mm = vma->vm_mm; in hugetlb_change_protection()
6605 i_mmap_lock_write(vma->vm_file->f_mapping); in hugetlb_change_protection()
6616 * Userfaultfd wr-protect requires pgtable in hugetlb_change_protection()
6617 * pre-allocations to install pte markers. in hugetlb_change_protection()
6621 pages = -ENOMEM; in hugetlb_change_protection()
6628 * When uffd-wp is enabled on the vma, unshare in hugetlb_change_protection()
6672 /* Safe to modify directly (non-present->none). */ in hugetlb_change_protection()
6680 pte = arch_make_huge_pte(pte, shift, vma->vm_flags); in hugetlb_change_protection()
6690 /* Safe to modify directly (none->non-present). */ in hugetlb_change_protection()
6715 i_mmap_unlock_write(vma->vm_file->f_mapping); in hugetlb_change_protection()
6719 return pages > 0 ? (pages << h->order) : pages; in hugetlb_change_protection()
6728 long chg = -1, add = -1; in hugetlb_reserve_pages()
6758 * to reserve the full area even if read-only as mprotect() may be in hugetlb_reserve_pages()
6759 * called to make the mapping read-write. Assume !vma is a shm mapping in hugetlb_reserve_pages()
6761 if (!vma || vma->vm_flags & VM_MAYSHARE) { in hugetlb_reserve_pages()
6776 chg = to - from; in hugetlb_reserve_pages()
6789 if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) { in hugetlb_reserve_pages()
6817 * the reservation was consumed. Private mappings are per-VMA and in hugetlb_reserve_pages()
6823 if (!vma || vma->vm_flags & VM_MAYSHARE) { in hugetlb_reserve_pages()
6827 hugetlb_acct_memory(h, -gbl_reserve); in hugetlb_reserve_pages()
6841 * reference to h_cg->css. See comment below for detail. in hugetlb_reserve_pages()
6845 (chg - add) * pages_per_huge_page(h), h_cg); in hugetlb_reserve_pages()
6848 chg - add); in hugetlb_reserve_pages()
6849 hugetlb_acct_memory(h, -rsv_adjust); in hugetlb_reserve_pages()
6853 * h_cg->css. So we should release the reference held in hugetlb_reserve_pages()
6870 if (!vma || vma->vm_flags & VM_MAYSHARE) in hugetlb_reserve_pages()
6877 kref_put(&resv_map->refs, resv_map_release); in hugetlb_reserve_pages()
6907 spin_lock(&inode->i_lock); in hugetlb_unreserve_pages()
6908 inode->i_blocks -= (blocks_per_huge_page(h) * freed); in hugetlb_unreserve_pages()
6909 spin_unlock(&inode->i_lock); in hugetlb_unreserve_pages()
6915 * Note that !resv_map implies freed == 0. So (chg - freed) in hugetlb_unreserve_pages()
6918 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed)); in hugetlb_unreserve_pages()
6919 hugetlb_acct_memory(h, -gbl_reserve); in hugetlb_unreserve_pages()
6929 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + in page_table_shareable()
6930 svma->vm_start; in page_table_shareable()
6935 unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED_MASK; in page_table_shareable()
6936 unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED_MASK; in page_table_shareable()
6947 !svma->vm_private_data) in page_table_shareable()
6965 if (!(vma->vm_flags & VM_MAYSHARE)) in want_pmd_share()
6967 if (!vma->vm_private_data) /* vma lock required for sharing */ in want_pmd_share()
6982 unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE), in adjust_range_if_pmd_sharing_possible()
6983 v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE); in adjust_range_if_pmd_sharing_possible()
6989 if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) || in adjust_range_if_pmd_sharing_possible()
7006 * pud has to be populated inside the same i_mmap_rwsem section - otherwise
7013 struct address_space *mapping = vma->vm_file->f_mapping; in huge_pmd_share()
7014 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + in huge_pmd_share()
7015 vma->vm_pgoff; in huge_pmd_share()
7022 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { in huge_pmd_share()
7040 spin_lock(&mm->page_table_lock); in huge_pmd_share()
7048 spin_unlock(&mm->page_table_lock); in huge_pmd_share()
7071 i_mmap_assert_write_locked(vma->vm_file->f_mapping); in huge_pmd_unshare()
7081 * using this page table as a normal, non-hugetlb page table. in huge_pmd_unshare()
7152 * huge_pte_offset() - Walk the page table to resolve the hugepage
7177 /* must be pud huge, non-present or none */ in huge_pte_offset()
7184 /* must be pmd huge, non-present or none */ in huge_pte_offset()
7190 * page in a page table page mapping size. Used to skip non-present
7200 return P4D_SIZE - PUD_SIZE; in hugetlb_mask_last_page()
7202 return PUD_SIZE - PMD_SIZE; in hugetlb_mask_last_page()
7214 return PUD_SIZE - PMD_SIZE; in hugetlb_mask_last_page()
7237 list_move_tail(&folio->lru, list); in isolate_hugetlb()
7256 ret = -EBUSY; in get_hwpoison_hugetlb_folio()
7277 list_move_tail(&folio->lru, &(folio_hstate(folio))->hugepage_activelist); in folio_putback_active_hugetlb()
7287 set_page_owner_migrate_reason(&new_folio->page, reason); in move_hugetlb_state()
7295 * Also note that we have to transfer the per-node surplus state in move_hugetlb_state()
7297 * the per-node's. in move_hugetlb_state()
7308 * There is no need to transfer the per-node surplus state in move_hugetlb_state()
7314 if (h->surplus_huge_pages_node[old_nid]) { in move_hugetlb_state()
7315 h->surplus_huge_pages_node[old_nid]--; in move_hugetlb_state()
7316 h->surplus_huge_pages_node[new_nid]++; in move_hugetlb_state()
7335 struct mm_struct *mm = vma->vm_mm; in hugetlb_unshare_pmds()
7341 if (!(vma->vm_flags & VM_MAYSHARE)) in hugetlb_unshare_pmds()
7357 i_mmap_lock_write(vma->vm_file->f_mapping); in hugetlb_unshare_pmds()
7359 i_mmap_assert_write_locked(vma->vm_file->f_mapping); in hugetlb_unshare_pmds()
7371 i_mmap_unlock_write(vma->vm_file->f_mapping); in hugetlb_unshare_pmds()
7387 hugetlb_unshare_pmds(vma, ALIGN(vma->vm_start, PUD_SIZE), in hugetlb_unshare_all_pmds()
7388 ALIGN_DOWN(vma->vm_end, PUD_SIZE), in hugetlb_unshare_all_pmds()
7451 hugetlb_cma_size -= hugetlb_cma_size_in_node[nid]; in hugetlb_cma_reserve()
7459 hugetlb_cma_size -= hugetlb_cma_size_in_node[nid]; in hugetlb_cma_reserve()
7498 size = min(per_node, hugetlb_cma_size - reserved); in hugetlb_cma_reserve()
7540 pr_warn("hugetlb_cma: the option isn't supported by current arch\n"); in hugetlb_cma_check()