855 	int nid = page_to_nid(page);  in enqueue_huge_page()  local
856 	list_move(&page->lru, &h->hugepage_freelists[nid]);  in enqueue_huge_page()
858 	h->free_huge_pages_node[nid]++;  in enqueue_huge_page()
862 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)  in dequeue_huge_page_node()  argument
866 	list_for_each_entry(page, &h->hugepage_freelists[nid], lru)  in dequeue_huge_page_node()
873 	if (&h->hugepage_freelists[nid] == &page->lru)  in dequeue_huge_page_node()
879 	h->free_huge_pages_node[nid]--;  in dequeue_huge_page_node()
956 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)  in next_node_allowed()  argument
958 	nid = next_node(nid, *nodes_allowed);  in next_node_allowed()
959 	if (nid == MAX_NUMNODES)  in next_node_allowed()
960 		nid = first_node(*nodes_allowed);  in next_node_allowed()
961 	VM_BUG_ON(nid >= MAX_NUMNODES);  in next_node_allowed()
963 	return nid;  in next_node_allowed()
966 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)  in get_valid_node_allowed()  argument
968 	if (!node_isset(nid, *nodes_allowed))  in get_valid_node_allowed()
969 		nid = next_node_allowed(nid, nodes_allowed);  in get_valid_node_allowed()
970 	return nid;  in get_valid_node_allowed()
982 	int nid;  in hstate_next_node_to_alloc()  local
986 	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);  in hstate_next_node_to_alloc()
987 	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);  in hstate_next_node_to_alloc()
989 	return nid;  in hstate_next_node_to_alloc()
1000 	int nid;  in hstate_next_node_to_free()  local
1004 	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);  in hstate_next_node_to_free()
1005 	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);  in hstate_next_node_to_free()
1007 	return nid;  in hstate_next_node_to_free()
1083 static struct page *alloc_gigantic_page(int nid, unsigned int order)  in alloc_gigantic_page()  argument
1089 	z = NODE_DATA(nid)->node_zones;  in alloc_gigantic_page()
1090 	for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {  in alloc_gigantic_page()
1118 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
1121 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)  in alloc_fresh_gigantic_page_node()  argument
1125 	page = alloc_gigantic_page(nid, huge_page_order(h));  in alloc_fresh_gigantic_page_node()
1128 		prep_new_huge_page(h, page, nid);  in alloc_fresh_gigantic_page_node()
1229 	int nid = page_to_nid(page);  in free_huge_page()  local
1267 	if (h->surplus_huge_pages_node[nid]) {  in free_huge_page()
1272 		h->surplus_huge_pages_node[nid]--;  in free_huge_page()
1280 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)  in prep_new_huge_page()  argument
1287 	h->nr_huge_pages_node[nid]++;  in prep_new_huge_page()
1366 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)  in alloc_fresh_huge_page_node()  argument
1370 	page = __alloc_pages_node(nid,  in alloc_fresh_huge_page_node()
1375 		prep_new_huge_page(h, page, nid);  in alloc_fresh_huge_page_node()
1452 		int nid = page_to_nid(head);  in dissolve_free_huge_page()  local
1475 		h->free_huge_pages_node[nid]--;  in dissolve_free_huge_page()
1508 		struct vm_area_struct *vma, unsigned long addr, int nid)  in __hugetlb_alloc_buddy_huge_page()  argument
1529 		if (nid != NUMA_NO_NODE)  in __hugetlb_alloc_buddy_huge_page()
1535 		return alloc_pages_node(nid, gfp, order);  in __hugetlb_alloc_buddy_huge_page()
1573 		struct vm_area_struct *vma, unsigned long addr, int nid)  in __alloc_buddy_huge_page()  argument
1588 		VM_WARN_ON_ONCE(nid != NUMA_NO_NODE);  in __alloc_buddy_huge_page()
1623 	page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid);  in __alloc_buddy_huge_page()
1653 struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid)  in __alloc_buddy_huge_page_no_mpol()  argument
1657 	return __alloc_buddy_huge_page(h, NULL, addr, nid);  in __alloc_buddy_huge_page_no_mpol()
1675 struct page *alloc_huge_page_node(struct hstate *h, int nid)  in alloc_huge_page_node()  argument
1681 		page = dequeue_huge_page_node(h, nid);  in alloc_huge_page_node()
1685 		page = __alloc_buddy_huge_page_no_mpol(h, nid);  in alloc_huge_page_node()
2326 	int nid;  in nr_hugepages_show_common()  local
2328 	h = kobj_to_hstate(kobj, &nid);  in nr_hugepages_show_common()
2329 	if (nid == NUMA_NO_NODE)  in nr_hugepages_show_common()
2332 		nr_huge_pages = h->nr_huge_pages_node[nid];  in nr_hugepages_show_common()
2338 					   struct hstate *h, int nid,  in __nr_hugepages_store_common()  argument
2349 	if (nid == NUMA_NO_NODE) {  in __nr_hugepages_store_common()
2363 		count += h->nr_huge_pages - h->nr_huge_pages_node[nid];  in __nr_hugepages_store_common()
2364 		init_nodemask_of_node(nodes_allowed, nid);  in __nr_hugepages_store_common()
2385 	int nid;  in nr_hugepages_store_common()  local
2392 	h = kobj_to_hstate(kobj, &nid);  in nr_hugepages_store_common()
2393 	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);  in nr_hugepages_store_common()
2464 	int nid;  in free_hugepages_show()  local
2466 	h = kobj_to_hstate(kobj, &nid);  in free_hugepages_show()
2467 	if (nid == NUMA_NO_NODE)  in free_hugepages_show()
2470 		free_huge_pages = h->free_huge_pages_node[nid];  in free_hugepages_show()
2489 	int nid;  in surplus_hugepages_show()  local
2491 	h = kobj_to_hstate(kobj, &nid);  in surplus_hugepages_show()
2492 	if (nid == NUMA_NO_NODE)  in surplus_hugepages_show()
2495 		surplus_huge_pages = h->surplus_huge_pages_node[nid];  in surplus_hugepages_show()
2589 	int nid;  in kobj_to_node_hstate()  local
2591 	for (nid = 0; nid < nr_node_ids; nid++) {  in kobj_to_node_hstate()
2592 		struct node_hstate *nhs = &node_hstates[nid];  in kobj_to_node_hstate()
2597 					*nidp = nid;  in kobj_to_node_hstate()
2636 	int nid;  in hugetlb_unregister_all_nodes()  local
2646 	for (nid = 0; nid < nr_node_ids; nid++)  in hugetlb_unregister_all_nodes()
2647 		hugetlb_unregister_node(node_devices[nid]);  in hugetlb_unregister_all_nodes()
2688 	int nid;  in hugetlb_register_all_nodes()  local
2690 	for_each_node_state(nid, N_MEMORY) {  in hugetlb_register_all_nodes()
2691 		struct node *node = node_devices[nid];  in hugetlb_register_all_nodes()
2692 		if (node->dev.id == nid)  in hugetlb_register_all_nodes()
2963 int hugetlb_report_node_meminfo(int nid, char *buf)  in hugetlb_report_node_meminfo()  argument
2972 		nid, h->nr_huge_pages_node[nid],  in hugetlb_report_node_meminfo()
2973 		nid, h->free_huge_pages_node[nid],  in hugetlb_report_node_meminfo()
2974 		nid, h->surplus_huge_pages_node[nid]);  in hugetlb_report_node_meminfo()
2980 	int nid;  in hugetlb_show_meminfo()  local
2985 	for_each_node_state(nid, N_MEMORY)  in hugetlb_show_meminfo()
2988 				nid,  in hugetlb_show_meminfo()
2989 				h->nr_huge_pages_node[nid],  in hugetlb_show_meminfo()
2990 				h->free_huge_pages_node[nid],  in hugetlb_show_meminfo()
2991 				h->surplus_huge_pages_node[nid],  in hugetlb_show_meminfo()
4597 	int nid = page_to_nid(hpage);  in dequeue_hwpoisoned_huge_page()  local
4615 		h->free_huge_pages_node[nid]--;  in dequeue_hwpoisoned_huge_page()