15  * interleave     Allocate memory interleaved over a set of nodes,
22  * bind           Only allocate memory on a specific set of nodes,
26  *                the allocation to memory nodes instead
34  * preferred many Try a set of nodes first before normal fallback. This is
185 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
186 	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
202 static int mpol_new_nodemask(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_nodemask()  argument
204 	if (nodes_empty(*nodes))  in mpol_new_nodemask()
206 	pol->nodes = *nodes;  in mpol_new_nodemask()
210 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_preferred()  argument
212 	if (nodes_empty(*nodes))  in mpol_new_preferred()
215 	nodes_clear(pol->nodes);  in mpol_new_preferred()
216 	node_set(first_node(*nodes), pol->nodes);  in mpol_new_preferred()
222  * any, for the new policy.  mpol_new() has already validated the nodes
229 		     const nodemask_t *nodes, struct nodemask_scratch *nsc)  in mpol_set_nodemask()  argument
245 	VM_BUG_ON(!nodes);  in mpol_set_nodemask()
248 		mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);  in mpol_set_nodemask()
250 		nodes_and(nsc->mask2, *nodes, nsc->mask1);  in mpol_set_nodemask()
253 		pol->w.user_nodemask = *nodes;  in mpol_set_nodemask()
263  * initialization. You must invoke mpol_set_nodemask() to set nodes.
266 				  nodemask_t *nodes)  in mpol_new()  argument
270 	pr_debug("setting mode %d flags %d nodes[0] %lx\n",  in mpol_new()
271 		 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE);  in mpol_new()
274 		if (nodes && !nodes_empty(*nodes))  in mpol_new()
278 	VM_BUG_ON(!nodes);  in mpol_new()
286 		if (nodes_empty(*nodes)) {  in mpol_new()
294 		if (!nodes_empty(*nodes) ||  in mpol_new()
298 	} else if (nodes_empty(*nodes))  in mpol_new()
319 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_rebind_default()  argument
323 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_rebind_nodemask()  argument
328 		nodes_and(tmp, pol->w.user_nodemask, *nodes);  in mpol_rebind_nodemask()
330 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);  in mpol_rebind_nodemask()
332 		nodes_remap(tmp, pol->nodes, pol->w.cpuset_mems_allowed,  in mpol_rebind_nodemask()
333 								*nodes);  in mpol_rebind_nodemask()
334 		pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_nodemask()
338 		tmp = *nodes;  in mpol_rebind_nodemask()
340 	pol->nodes = tmp;  in mpol_rebind_nodemask()
344 						const nodemask_t *nodes)  in mpol_rebind_preferred()  argument
346 	pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_preferred()
350  * mpol_rebind_policy - Migrate a policy to a different set of nodes
734  * If pages found in a given range are on a set of nodes (determined by
735  * @nodes and @flags,) it's isolated and queued to the pagelist which is
748 		nodemask_t *nodes, unsigned long flags,  in queue_pages_range()  argument
755 		.nmask = nodes,  in queue_pages_range()
863 			     nodemask_t *nodes)  in do_set_mempolicy()  argument
872 	new = mpol_new(mode, flags, nodes);  in do_set_mempolicy()
879 	ret = mpol_set_nodemask(new, nodes, scratch);  in do_set_mempolicy()
903 static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)  in get_policy_nodemask()  argument
905 	nodes_clear(*nodes);  in get_policy_nodemask()
914 		*nodes = p->nodes;  in get_policy_nodemask()
999 			*policy = next_node_in(current->il_prev, pol->nodes);  in do_get_mempolicy()
1138 	 * This lets us pick a pair of nodes to migrate between, such that  in do_migrate_pages()
1167 			 * However if the number of source nodes is not equal to  in do_migrate_pages()
1168 			 * the number of destination nodes we can not preserve  in do_migrate_pages()
1188 			/* dest not in remaining from nodes? */  in do_migrate_pages()
1311 	pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",  in do_mbind()
1407 static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,  in get_nodes()  argument
1411 	nodes_clear(*nodes);  in get_nodes()
1418 	 * When the user specified more nodes than supported just check  in get_nodes()
1439 	return get_bitmap(nodes_addr(*nodes), nmask, maxnode);  in get_nodes()
1444 			      nodemask_t *nodes)  in copy_nodes_to_user()  argument
1464 					 nodes_addr(*nodes), maxnode);  in copy_nodes_to_user()
1466 	return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;  in copy_nodes_to_user()
1492 	nodemask_t nodes;  in kernel_mbind()  local
1501 	err = get_nodes(&nodes, nmask, maxnode);  in kernel_mbind()
1505 	return do_mbind(start, len, lmode, mode_flags, &nodes, flags);  in kernel_mbind()
1587 	nodemask_t nodes;  in kernel_set_mempolicy()  local
1595 	err = get_nodes(&nodes, nmask, maxnode);  in kernel_set_mempolicy()
1599 	return do_set_mempolicy(lmode, mode_flags, &nodes);  in kernel_set_mempolicy()
1658 	/* Is the user allowed to access the target nodes? */  in kernel_migrate_pages()
1713 	nodemask_t nodes;  in kernel_get_mempolicy()  local
1720 	err = do_get_mempolicy(&pval, &nodes, addr, flags);  in kernel_get_mempolicy()
1729 		err = copy_nodes_to_user(nmask, maxnode, &nodes);  in kernel_get_mempolicy()
1846 	 * if policy->nodes has movable memory only,  in apply_policy_zone()
1849 	 * policy->nodes is intersect with node_states[N_MEMORY].  in apply_policy_zone()
1851 	 * policy->nodes has movable memory only.  in apply_policy_zone()
1853 	if (!nodes_intersects(policy->nodes, node_states[N_HIGH_MEMORY]))  in apply_policy_zone()
1860  * Return a nodemask representing a mempolicy for filtering nodes for
1870 		cpuset_nodemask_valid_mems_allowed(&policy->nodes))  in policy_nodemask()
1871 		return &policy->nodes;  in policy_nodemask()
1874 		return &policy->nodes;  in policy_nodemask()
1889 		nd = first_node(policy->nodes);  in policy_node()
1913 	next = next_node_in(me->il_prev, policy->nodes);  in interleave_nodes()
1937 		return first_node(policy->nodes);  in mempolicy_slab_node()
1955 							&policy->nodes);  in mempolicy_slab_node()
1968  * node in pol->nodes (starting from n=0), wrapping around if n exceeds the
1969  * number of present nodes.
1973 	nodemask_t nodemask = pol->nodes;  in offset_il_node()
1981 	 * Between first_node() and next_node(), pol->nodes could be changed  in offset_il_node()
1982 	 * by other threads. So we put pol->nodes in a local stack.  in offset_il_node()
2050 			*nodemask = &(*mpol)->nodes;  in huge_node()
2085 		*mask = mempolicy->nodes;  in init_nodemask_of_mempolicy()
2107  * memory allocated from all nodes in system.
2123 		ret = nodes_intersects(mempolicy->nodes, *mask);  in mempolicy_in_oom_domain()
2156 	 * preferred nodes but skip the direct reclaim and allow the  in alloc_pages_preferred_many()
2158 	 * nodes in system.  in alloc_pages_preferred_many()
2162 	page = __alloc_pages(preferred_gfp, order, nid, &pol->nodes);  in alloc_pages_preferred_many()
2229 		 * node and don't fall back to other nodes, as the cost of  in vma_alloc_folio()
2236 			hpage_node = first_node(pol->nodes);  in vma_alloc_folio()
2326 	int nodes;  in alloc_pages_bulk_array_interleave()  local
2333 	nodes = nodes_weight(pol->nodes);  in alloc_pages_bulk_array_interleave()
2334 	nr_pages_per_node = nr_pages / nodes;  in alloc_pages_bulk_array_interleave()
2335 	delta = nr_pages - nodes * nr_pages_per_node;  in alloc_pages_bulk_array_interleave()
2337 	for (i = 0; i < nodes; i++) {  in alloc_pages_bulk_array_interleave()
2367 	nr_allocated  = __alloc_pages_bulk(preferred_gfp, nid, &pol->nodes,  in alloc_pages_bulk_array_preferred_many()
2469 		return !!nodes_equal(a->nodes, b->nodes);  in __mpol_equal()
2609 		if (node_isset(curnid, pol->nodes))  in mpol_misplaced()
2611 		polnid = first_node(pol->nodes);  in mpol_misplaced()
2619 		/* Optimize placement among multiple nodes via NUMA balancing */  in mpol_misplaced()
2621 			if (node_isset(thisnid, pol->nodes))  in mpol_misplaced()
2631 		 * If no allowed nodes, use current [!misplaced].  in mpol_misplaced()
2633 		if (node_isset(curnid, pol->nodes))  in mpol_misplaced()
2638 				&pol->nodes);  in mpol_misplaced()
2842 		 npol ? nodes_addr(npol->nodes)[0] : NUMA_NO_NODE);  in mpol_set_shared_policy()
2940 			.nodes = nodemask_of_node(nid),  in numa_policy_init()
2946 	 * enabled across suitably sized nodes (default is >= 16MB), or  in numa_policy_init()
3010 	nodemask_t nodes;  in mpol_parse_str()  local
3021 		if (nodelist_parse(nodelist, nodes))  in mpol_parse_str()
3023 		if (!nodes_subset(nodes, node_states[N_MEMORY]))  in mpol_parse_str()
3026 		nodes_clear(nodes);  in mpol_parse_str()
3036 		 * we use first_node(nodes) to grab a single node, so here  in mpol_parse_str()
3037 		 * nodelist (or nodes) cannot be empty.  in mpol_parse_str()
3045 			if (nodes_empty(nodes))  in mpol_parse_str()
3051 		 * Default to online nodes with memory if no nodelist  in mpol_parse_str()
3054 			nodes = node_states[N_MEMORY];  in mpol_parse_str()
3093 	new = mpol_new(mode, mode_flags, &nodes);  in mpol_parse_str()
3098 	 * Save nodes for mpol_to_str() to show the tmpfs mount options  in mpol_parse_str()
3102 		new->nodes = nodes;  in mpol_parse_str()
3104 		nodes_clear(new->nodes);  in mpol_parse_str()
3105 		node_set(first_node(nodes), new->nodes);  in mpol_parse_str()
3111 	 * Save nodes for contextualization: this will be used to "clone"  in mpol_parse_str()
3114 	new->w.user_nodemask = nodes;  in mpol_parse_str()
3143 	nodemask_t nodes = NODE_MASK_NONE;  in mpol_to_str()  local
3160 		nodes = pol->nodes;  in mpol_to_str()
3182 	if (!nodes_empty(nodes))  in mpol_to_str()
3184 			       nodemask_pr_args(&nodes));  in mpol_to_str()