1276 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv)  in task_faults_idx()  argument
1278 	return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv;  in task_faults_idx()
1281 static inline unsigned long task_faults(struct task_struct *p, int nid)  in task_faults()  argument
1286 	return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] +  in task_faults()
1287 		p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)];  in task_faults()
1290 static inline unsigned long group_faults(struct task_struct *p, int nid)  in group_faults()  argument
1297 	return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] +  in group_faults()
1298 		ng->faults[task_faults_idx(NUMA_MEM, nid, 1)];  in group_faults()
1301 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid)  in group_faults_cpu()  argument
1303 	return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] +  in group_faults_cpu()
1304 		group->faults[task_faults_idx(NUMA_CPU, nid, 1)];  in group_faults_cpu()
1338 static bool numa_is_active_node(int nid, struct numa_group *ng)  in numa_is_active_node()  argument
1340 	return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu;  in numa_is_active_node()
1344 static unsigned long score_nearby_nodes(struct task_struct *p, int nid,  in score_nearby_nodes()  argument
1365 		int dist = node_distance(nid, node);  in score_nearby_nodes()
1371 		if (dist >= max_dist || node == nid)  in score_nearby_nodes()
1415 static inline unsigned long task_weight(struct task_struct *p, int nid,  in task_weight()  argument
1428 	faults = task_faults(p, nid);  in task_weight()
1429 	faults += score_nearby_nodes(p, nid, dist, true);  in task_weight()
1434 static inline unsigned long group_weight(struct task_struct *p, int nid,  in group_weight()  argument
1448 	faults = group_faults(p, nid);  in group_weight()
1449 	faults += score_nearby_nodes(p, nid, dist, false);  in group_weight()
1789 			      struct numa_stats *ns, int nid,  in update_numa_stats()  argument
1798 	for_each_cpu(cpu, cpumask_of_node(nid)) {  in update_numa_stats()
1820 	ns->weight = cpumask_weight(cpumask_of_node(nid));  in update_numa_stats()
2181 	int nid, ret, dist;  in task_numa_migrate()  local
2231 		for_each_node_state(nid, N_CPU) {  in task_numa_migrate()
2232 			if (nid == env.src_nid || nid == p->numa_preferred_nid)  in task_numa_migrate()
2243 			taskimp = task_weight(p, nid, dist) - taskweight;  in task_numa_migrate()
2244 			groupimp = group_weight(p, nid, dist) - groupweight;  in task_numa_migrate()
2249 			env.dst_nid = nid;  in task_numa_migrate()
2265 			nid = env.src_nid;  in task_numa_migrate()
2267 			nid = cpu_to_node(env.best_cpu);  in task_numa_migrate()
2269 		if (nid != p->numa_preferred_nid)  in task_numa_migrate()
2270 			sched_setnuma(p, nid);  in task_numa_migrate()
2327 	int nid, active_nodes = 0;  in numa_group_count_active_nodes()  local
2329 	for_each_node_state(nid, N_CPU) {  in numa_group_count_active_nodes()
2330 		faults = group_faults_cpu(numa_group, nid);  in numa_group_count_active_nodes()
2335 	for_each_node_state(nid, N_CPU) {  in numa_group_count_active_nodes()
2336 		faults = group_faults_cpu(numa_group, nid);  in numa_group_count_active_nodes()
2469 static int preferred_group_nid(struct task_struct *p, int nid)  in preferred_group_nid()  argument
2476 		return nid;  in preferred_group_nid()
2485 		int node, max_node = nid;  in preferred_group_nid()
2541 				nid = a;  in preferred_group_nid()
2549 	return nid;  in preferred_group_nid()
2554 	int seq, nid, max_nid = NUMA_NO_NODE;  in task_numa_placement()  local
2585 	for_each_online_node(nid) {  in task_numa_placement()
2594 			mem_idx = task_faults_idx(NUMA_MEM, nid, priv);  in task_numa_placement()
2595 			membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv);  in task_numa_placement()
2596 			cpu_idx = task_faults_idx(NUMA_CPU, nid, priv);  in task_numa_placement()
2597 			cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv);  in task_numa_placement()
2639 				max_nid = nid;  in task_numa_placement()
2643 			max_nid = nid;  in task_numa_placement()
2652 		for_each_node_state(nid, N_CPU) {  in task_numa_placement()
2653 			distance = node_distance(max_nid, nid);  in task_numa_placement()
2655 				near_nid = nid;  in task_numa_placement()