• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 #ifndef __LINUX_CPUMASK_H
2 #define __LINUX_CPUMASK_H
3 
4 /*
5  * Cpumasks provide a bitmap suitable for representing the
6  * set of CPU's in a system, one bit position per CPU number.  In general,
7  * only nr_cpu_ids (<= NR_CPUS) bits are valid.
8  */
9 #include <linux/kernel.h>
10 #include <linux/threads.h>
11 #include <linux/bitmap.h>
12 #include <linux/bug.h>
13 
14 /* Don't assign or return these: may not be this big! */
15 typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
16 
17 /**
18  * cpumask_bits - get the bits in a cpumask
19  * @maskp: the struct cpumask *
20  *
21  * You should only assume nr_cpu_ids bits of this mask are valid.  This is
22  * a macro so it's const-correct.
23  */
24 #define cpumask_bits(maskp) ((maskp)->bits)
25 
26 /**
27  * cpumask_pr_args - printf args to output a cpumask
28  * @maskp: cpumask to be printed
29  *
30  * Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
31  */
32 #define cpumask_pr_args(maskp)		nr_cpu_ids, cpumask_bits(maskp)
33 
34 #if NR_CPUS == 1
35 #define nr_cpu_ids		1
36 #else
37 extern int nr_cpu_ids;
38 #endif
39 
40 #ifdef CONFIG_CPUMASK_OFFSTACK
41 /* Assuming NR_CPUS is huge, a runtime limit is more efficient.  Also,
42  * not all bits may be allocated. */
43 #define nr_cpumask_bits	nr_cpu_ids
44 #else
45 #define nr_cpumask_bits	NR_CPUS
46 #endif
47 
48 /*
49  * The following particular system cpumasks and operations manage
50  * possible, present, active and online cpus.
51  *
52  *     cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
53  *     cpu_present_mask - has bit 'cpu' set iff cpu is populated
54  *     cpu_online_mask  - has bit 'cpu' set iff cpu available to scheduler
55  *     cpu_active_mask  - has bit 'cpu' set iff cpu available to migration
56  *
57  *  If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
58  *
59  *  The cpu_possible_mask is fixed at boot time, as the set of CPU id's
60  *  that it is possible might ever be plugged in at anytime during the
61  *  life of that system boot.  The cpu_present_mask is dynamic(*),
62  *  representing which CPUs are currently plugged in.  And
63  *  cpu_online_mask is the dynamic subset of cpu_present_mask,
64  *  indicating those CPUs available for scheduling.
65  *
66  *  If HOTPLUG is enabled, then cpu_possible_mask is forced to have
67  *  all NR_CPUS bits set, otherwise it is just the set of CPUs that
68  *  ACPI reports present at boot.
69  *
70  *  If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
71  *  depending on what ACPI reports as currently plugged in, otherwise
72  *  cpu_present_mask is just a copy of cpu_possible_mask.
73  *
74  *  (*) Well, cpu_present_mask is dynamic in the hotplug case.  If not
75  *      hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
76  *
77  * Subtleties:
78  * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
79  *    assumption that their single CPU is online.  The UP
80  *    cpu_{online,possible,present}_masks are placebos.  Changing them
81  *    will have no useful affect on the following num_*_cpus()
82  *    and cpu_*() macros in the UP case.  This ugliness is a UP
83  *    optimization - don't waste any instructions or memory references
84  *    asking if you're online or how many CPUs there are if there is
85  *    only one CPU.
86  */
87 
88 extern struct cpumask __cpu_possible_mask;
89 extern struct cpumask __cpu_online_mask;
90 extern struct cpumask __cpu_present_mask;
91 extern struct cpumask __cpu_active_mask;
92 #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
93 #define cpu_online_mask   ((const struct cpumask *)&__cpu_online_mask)
94 #define cpu_present_mask  ((const struct cpumask *)&__cpu_present_mask)
95 #define cpu_active_mask   ((const struct cpumask *)&__cpu_active_mask)
96 
97 #if NR_CPUS > 1
98 #define num_online_cpus()	cpumask_weight(cpu_online_mask)
99 #define num_possible_cpus()	cpumask_weight(cpu_possible_mask)
100 #define num_present_cpus()	cpumask_weight(cpu_present_mask)
101 #define num_active_cpus()	cpumask_weight(cpu_active_mask)
102 #define cpu_online(cpu)		cpumask_test_cpu((cpu), cpu_online_mask)
103 #define cpu_possible(cpu)	cpumask_test_cpu((cpu), cpu_possible_mask)
104 #define cpu_present(cpu)	cpumask_test_cpu((cpu), cpu_present_mask)
105 #define cpu_active(cpu)		cpumask_test_cpu((cpu), cpu_active_mask)
106 #else
107 #define num_online_cpus()	1U
108 #define num_possible_cpus()	1U
109 #define num_present_cpus()	1U
110 #define num_active_cpus()	1U
111 #define cpu_online(cpu)		((cpu) == 0)
112 #define cpu_possible(cpu)	((cpu) == 0)
113 #define cpu_present(cpu)	((cpu) == 0)
114 #define cpu_active(cpu)		((cpu) == 0)
115 #endif
116 
117 /* verify cpu argument to cpumask_* operators */
cpumask_check(unsigned int cpu)118 static inline unsigned int cpumask_check(unsigned int cpu)
119 {
120 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
121 	WARN_ON_ONCE(cpu >= nr_cpumask_bits);
122 #endif /* CONFIG_DEBUG_PER_CPU_MAPS */
123 	return cpu;
124 }
125 
126 #if NR_CPUS == 1
127 /* Uniprocessor.  Assume all masks are "1". */
cpumask_first(const struct cpumask * srcp)128 static inline unsigned int cpumask_first(const struct cpumask *srcp)
129 {
130 	return 0;
131 }
132 
133 /* Valid inputs for n are -1 and 0. */
cpumask_next(int n,const struct cpumask * srcp)134 static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
135 {
136 	return n+1;
137 }
138 
cpumask_next_zero(int n,const struct cpumask * srcp)139 static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
140 {
141 	return n+1;
142 }
143 
cpumask_next_and(int n,const struct cpumask * srcp,const struct cpumask * andp)144 static inline unsigned int cpumask_next_and(int n,
145 					    const struct cpumask *srcp,
146 					    const struct cpumask *andp)
147 {
148 	return n+1;
149 }
150 
151 /* cpu must be a valid cpu, ie 0, so there's no other choice. */
cpumask_any_but(const struct cpumask * mask,unsigned int cpu)152 static inline unsigned int cpumask_any_but(const struct cpumask *mask,
153 					   unsigned int cpu)
154 {
155 	return 1;
156 }
157 
cpumask_local_spread(unsigned int i,int node)158 static inline unsigned int cpumask_local_spread(unsigned int i, int node)
159 {
160 	return 0;
161 }
162 
163 #define for_each_cpu(cpu, mask)			\
164 	for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
165 #define for_each_cpu_not(cpu, mask)		\
166 	for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
167 #define for_each_cpu_and(cpu, mask, and)	\
168 	for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and)
169 #else
170 /**
171  * cpumask_first - get the first cpu in a cpumask
172  * @srcp: the cpumask pointer
173  *
174  * Returns >= nr_cpu_ids if no cpus set.
175  */
cpumask_first(const struct cpumask * srcp)176 static inline unsigned int cpumask_first(const struct cpumask *srcp)
177 {
178 	return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
179 }
180 
181 /**
182  * cpumask_next - get the next cpu in a cpumask
183  * @n: the cpu prior to the place to search (ie. return will be > @n)
184  * @srcp: the cpumask pointer
185  *
186  * Returns >= nr_cpu_ids if no further cpus set.
187  */
cpumask_next(int n,const struct cpumask * srcp)188 static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
189 {
190 	/* -1 is a legal arg here. */
191 	if (n != -1)
192 		cpumask_check(n);
193 	return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
194 }
195 
196 /**
197  * cpumask_next_zero - get the next unset cpu in a cpumask
198  * @n: the cpu prior to the place to search (ie. return will be > @n)
199  * @srcp: the cpumask pointer
200  *
201  * Returns >= nr_cpu_ids if no further cpus unset.
202  */
cpumask_next_zero(int n,const struct cpumask * srcp)203 static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
204 {
205 	/* -1 is a legal arg here. */
206 	if (n != -1)
207 		cpumask_check(n);
208 	return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
209 }
210 
211 int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
212 int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
213 unsigned int cpumask_local_spread(unsigned int i, int node);
214 
215 /**
216  * for_each_cpu - iterate over every cpu in a mask
217  * @cpu: the (optionally unsigned) integer iterator
218  * @mask: the cpumask pointer
219  *
220  * After the loop, cpu is >= nr_cpu_ids.
221  */
222 #define for_each_cpu(cpu, mask)				\
223 	for ((cpu) = -1;				\
224 		(cpu) = cpumask_next((cpu), (mask)),	\
225 		(cpu) < nr_cpu_ids;)
226 
227 /**
228  * for_each_cpu_not - iterate over every cpu in a complemented mask
229  * @cpu: the (optionally unsigned) integer iterator
230  * @mask: the cpumask pointer
231  *
232  * After the loop, cpu is >= nr_cpu_ids.
233  */
234 #define for_each_cpu_not(cpu, mask)				\
235 	for ((cpu) = -1;					\
236 		(cpu) = cpumask_next_zero((cpu), (mask)),	\
237 		(cpu) < nr_cpu_ids;)
238 
239 extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
240 
241 /**
242  * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
243  * @cpu: the (optionally unsigned) integer iterator
244  * @mask: the cpumask poiter
245  * @start: the start location
246  *
247  * The implementation does not assume any bit in @mask is set (including @start).
248  *
249  * After the loop, cpu is >= nr_cpu_ids.
250  */
251 #define for_each_cpu_wrap(cpu, mask, start)					\
252 	for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false);	\
253 	     (cpu) < nr_cpumask_bits;						\
254 	     (cpu) = cpumask_next_wrap((cpu), (mask), (start), true))
255 
256 /**
257  * for_each_cpu_and - iterate over every cpu in both masks
258  * @cpu: the (optionally unsigned) integer iterator
259  * @mask: the first cpumask pointer
260  * @and: the second cpumask pointer
261  *
262  * This saves a temporary CPU mask in many places.  It is equivalent to:
263  *	struct cpumask tmp;
264  *	cpumask_and(&tmp, &mask, &and);
265  *	for_each_cpu(cpu, &tmp)
266  *		...
267  *
268  * After the loop, cpu is >= nr_cpu_ids.
269  */
270 #define for_each_cpu_and(cpu, mask, and)				\
271 	for ((cpu) = -1;						\
272 		(cpu) = cpumask_next_and((cpu), (mask), (and)),		\
273 		(cpu) < nr_cpu_ids;)
274 #endif /* SMP */
275 
276 #define CPU_BITS_NONE						\
277 {								\
278 	[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL			\
279 }
280 
281 #define CPU_BITS_CPU0						\
282 {								\
283 	[0] =  1UL						\
284 }
285 
286 /**
287  * cpumask_set_cpu - set a cpu in a cpumask
288  * @cpu: cpu number (< nr_cpu_ids)
289  * @dstp: the cpumask pointer
290  */
cpumask_set_cpu(unsigned int cpu,struct cpumask * dstp)291 static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
292 {
293 	set_bit(cpumask_check(cpu), cpumask_bits(dstp));
294 }
295 
296 /**
297  * cpumask_clear_cpu - clear a cpu in a cpumask
298  * @cpu: cpu number (< nr_cpu_ids)
299  * @dstp: the cpumask pointer
300  */
cpumask_clear_cpu(int cpu,struct cpumask * dstp)301 static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
302 {
303 	clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
304 }
305 
306 /**
307  * cpumask_test_cpu - test for a cpu in a cpumask
308  * @cpu: cpu number (< nr_cpu_ids)
309  * @cpumask: the cpumask pointer
310  *
311  * Returns 1 if @cpu is set in @cpumask, else returns 0
312  */
cpumask_test_cpu(int cpu,const struct cpumask * cpumask)313 static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
314 {
315 	return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
316 }
317 
318 /**
319  * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
320  * @cpu: cpu number (< nr_cpu_ids)
321  * @cpumask: the cpumask pointer
322  *
323  * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
324  *
325  * test_and_set_bit wrapper for cpumasks.
326  */
cpumask_test_and_set_cpu(int cpu,struct cpumask * cpumask)327 static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
328 {
329 	return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
330 }
331 
332 /**
333  * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
334  * @cpu: cpu number (< nr_cpu_ids)
335  * @cpumask: the cpumask pointer
336  *
337  * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
338  *
339  * test_and_clear_bit wrapper for cpumasks.
340  */
cpumask_test_and_clear_cpu(int cpu,struct cpumask * cpumask)341 static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
342 {
343 	return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
344 }
345 
346 /**
347  * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
348  * @dstp: the cpumask pointer
349  */
cpumask_setall(struct cpumask * dstp)350 static inline void cpumask_setall(struct cpumask *dstp)
351 {
352 	bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
353 }
354 
355 /**
356  * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
357  * @dstp: the cpumask pointer
358  */
cpumask_clear(struct cpumask * dstp)359 static inline void cpumask_clear(struct cpumask *dstp)
360 {
361 	bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
362 }
363 
364 /**
365  * cpumask_and - *dstp = *src1p & *src2p
366  * @dstp: the cpumask result
367  * @src1p: the first input
368  * @src2p: the second input
369  *
370  * If *@dstp is empty, returns 0, else returns 1
371  */
cpumask_and(struct cpumask * dstp,const struct cpumask * src1p,const struct cpumask * src2p)372 static inline int cpumask_and(struct cpumask *dstp,
373 			       const struct cpumask *src1p,
374 			       const struct cpumask *src2p)
375 {
376 	return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
377 				       cpumask_bits(src2p), nr_cpumask_bits);
378 }
379 
380 /**
381  * cpumask_or - *dstp = *src1p | *src2p
382  * @dstp: the cpumask result
383  * @src1p: the first input
384  * @src2p: the second input
385  */
cpumask_or(struct cpumask * dstp,const struct cpumask * src1p,const struct cpumask * src2p)386 static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
387 			      const struct cpumask *src2p)
388 {
389 	bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
390 				      cpumask_bits(src2p), nr_cpumask_bits);
391 }
392 
393 /**
394  * cpumask_xor - *dstp = *src1p ^ *src2p
395  * @dstp: the cpumask result
396  * @src1p: the first input
397  * @src2p: the second input
398  */
cpumask_xor(struct cpumask * dstp,const struct cpumask * src1p,const struct cpumask * src2p)399 static inline void cpumask_xor(struct cpumask *dstp,
400 			       const struct cpumask *src1p,
401 			       const struct cpumask *src2p)
402 {
403 	bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
404 				       cpumask_bits(src2p), nr_cpumask_bits);
405 }
406 
407 /**
408  * cpumask_andnot - *dstp = *src1p & ~*src2p
409  * @dstp: the cpumask result
410  * @src1p: the first input
411  * @src2p: the second input
412  *
413  * If *@dstp is empty, returns 0, else returns 1
414  */
cpumask_andnot(struct cpumask * dstp,const struct cpumask * src1p,const struct cpumask * src2p)415 static inline int cpumask_andnot(struct cpumask *dstp,
416 				  const struct cpumask *src1p,
417 				  const struct cpumask *src2p)
418 {
419 	return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
420 					  cpumask_bits(src2p), nr_cpumask_bits);
421 }
422 
423 /**
424  * cpumask_complement - *dstp = ~*srcp
425  * @dstp: the cpumask result
426  * @srcp: the input to invert
427  */
cpumask_complement(struct cpumask * dstp,const struct cpumask * srcp)428 static inline void cpumask_complement(struct cpumask *dstp,
429 				      const struct cpumask *srcp)
430 {
431 	bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
432 					      nr_cpumask_bits);
433 }
434 
435 /**
436  * cpumask_equal - *src1p == *src2p
437  * @src1p: the first input
438  * @src2p: the second input
439  */
cpumask_equal(const struct cpumask * src1p,const struct cpumask * src2p)440 static inline bool cpumask_equal(const struct cpumask *src1p,
441 				const struct cpumask *src2p)
442 {
443 	return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
444 						 nr_cpumask_bits);
445 }
446 
447 /**
448  * cpumask_intersects - (*src1p & *src2p) != 0
449  * @src1p: the first input
450  * @src2p: the second input
451  */
cpumask_intersects(const struct cpumask * src1p,const struct cpumask * src2p)452 static inline bool cpumask_intersects(const struct cpumask *src1p,
453 				     const struct cpumask *src2p)
454 {
455 	return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
456 						      nr_cpumask_bits);
457 }
458 
459 /**
460  * cpumask_subset - (*src1p & ~*src2p) == 0
461  * @src1p: the first input
462  * @src2p: the second input
463  *
464  * Returns 1 if *@src1p is a subset of *@src2p, else returns 0
465  */
cpumask_subset(const struct cpumask * src1p,const struct cpumask * src2p)466 static inline int cpumask_subset(const struct cpumask *src1p,
467 				 const struct cpumask *src2p)
468 {
469 	return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
470 						  nr_cpumask_bits);
471 }
472 
473 /**
474  * cpumask_empty - *srcp == 0
475  * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
476  */
cpumask_empty(const struct cpumask * srcp)477 static inline bool cpumask_empty(const struct cpumask *srcp)
478 {
479 	return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
480 }
481 
482 /**
483  * cpumask_full - *srcp == 0xFFFFFFFF...
484  * @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
485  */
cpumask_full(const struct cpumask * srcp)486 static inline bool cpumask_full(const struct cpumask *srcp)
487 {
488 	return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
489 }
490 
491 /**
492  * cpumask_weight - Count of bits in *srcp
493  * @srcp: the cpumask to count bits (< nr_cpu_ids) in.
494  */
cpumask_weight(const struct cpumask * srcp)495 static inline unsigned int cpumask_weight(const struct cpumask *srcp)
496 {
497 	return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
498 }
499 
500 /**
501  * cpumask_shift_right - *dstp = *srcp >> n
502  * @dstp: the cpumask result
503  * @srcp: the input to shift
504  * @n: the number of bits to shift by
505  */
cpumask_shift_right(struct cpumask * dstp,const struct cpumask * srcp,int n)506 static inline void cpumask_shift_right(struct cpumask *dstp,
507 				       const struct cpumask *srcp, int n)
508 {
509 	bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
510 					       nr_cpumask_bits);
511 }
512 
513 /**
514  * cpumask_shift_left - *dstp = *srcp << n
515  * @dstp: the cpumask result
516  * @srcp: the input to shift
517  * @n: the number of bits to shift by
518  */
cpumask_shift_left(struct cpumask * dstp,const struct cpumask * srcp,int n)519 static inline void cpumask_shift_left(struct cpumask *dstp,
520 				      const struct cpumask *srcp, int n)
521 {
522 	bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
523 					      nr_cpumask_bits);
524 }
525 
526 /**
527  * cpumask_copy - *dstp = *srcp
528  * @dstp: the result
529  * @srcp: the input cpumask
530  */
cpumask_copy(struct cpumask * dstp,const struct cpumask * srcp)531 static inline void cpumask_copy(struct cpumask *dstp,
532 				const struct cpumask *srcp)
533 {
534 	bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
535 }
536 
537 /**
538  * cpumask_any - pick a "random" cpu from *srcp
539  * @srcp: the input cpumask
540  *
541  * Returns >= nr_cpu_ids if no cpus set.
542  */
543 #define cpumask_any(srcp) cpumask_first(srcp)
544 
545 /**
546  * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
547  * @src1p: the first input
548  * @src2p: the second input
549  *
550  * Returns >= nr_cpu_ids if no cpus set in both.  See also cpumask_next_and().
551  */
552 #define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
553 
554 /**
555  * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
556  * @mask1: the first input cpumask
557  * @mask2: the second input cpumask
558  *
559  * Returns >= nr_cpu_ids if no cpus set.
560  */
561 #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
562 
563 /**
564  * cpumask_of - the cpumask containing just a given cpu
565  * @cpu: the cpu (<= nr_cpu_ids)
566  */
567 #define cpumask_of(cpu) (get_cpu_mask(cpu))
568 
569 /**
570  * cpumask_parse_user - extract a cpumask from a user string
571  * @buf: the buffer to extract from
572  * @len: the length of the buffer
573  * @dstp: the cpumask to set.
574  *
575  * Returns -errno, or 0 for success.
576  */
cpumask_parse_user(const char __user * buf,int len,struct cpumask * dstp)577 static inline int cpumask_parse_user(const char __user *buf, int len,
578 				     struct cpumask *dstp)
579 {
580 	return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
581 }
582 
583 /**
584  * cpumask_parselist_user - extract a cpumask from a user string
585  * @buf: the buffer to extract from
586  * @len: the length of the buffer
587  * @dstp: the cpumask to set.
588  *
589  * Returns -errno, or 0 for success.
590  */
cpumask_parselist_user(const char __user * buf,int len,struct cpumask * dstp)591 static inline int cpumask_parselist_user(const char __user *buf, int len,
592 				     struct cpumask *dstp)
593 {
594 	return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
595 				     nr_cpumask_bits);
596 }
597 
598 /**
599  * cpumask_parse - extract a cpumask from a string
600  * @buf: the buffer to extract from
601  * @dstp: the cpumask to set.
602  *
603  * Returns -errno, or 0 for success.
604  */
cpumask_parse(const char * buf,struct cpumask * dstp)605 static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
606 {
607 	char *nl = strchr(buf, '\n');
608 	unsigned int len = nl ? (unsigned int)(nl - buf) : strlen(buf);
609 
610 	return bitmap_parse(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
611 }
612 
613 /**
614  * cpulist_parse - extract a cpumask from a user string of ranges
615  * @buf: the buffer to extract from
616  * @dstp: the cpumask to set.
617  *
618  * Returns -errno, or 0 for success.
619  */
cpulist_parse(const char * buf,struct cpumask * dstp)620 static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
621 {
622 	return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
623 }
624 
625 /**
626  * cpumask_size - size to allocate for a 'struct cpumask' in bytes
627  */
cpumask_size(void)628 static inline size_t cpumask_size(void)
629 {
630 	return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
631 }
632 
633 /*
634  * cpumask_var_t: struct cpumask for stack usage.
635  *
636  * Oh, the wicked games we play!  In order to make kernel coding a
637  * little more difficult, we typedef cpumask_var_t to an array or a
638  * pointer: doing &mask on an array is a noop, so it still works.
639  *
640  * ie.
641  *	cpumask_var_t tmpmask;
642  *	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
643  *		return -ENOMEM;
644  *
645  *	  ... use 'tmpmask' like a normal struct cpumask * ...
646  *
647  *	free_cpumask_var(tmpmask);
648  *
649  *
650  * However, one notable exception is there. alloc_cpumask_var() allocates
651  * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
652  * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
653  *
654  *	cpumask_var_t tmpmask;
655  *	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
656  *		return -ENOMEM;
657  *
658  *	var = *tmpmask;
659  *
660  * This code makes NR_CPUS length memcopy and brings to a memory corruption.
661  * cpumask_copy() provide safe copy functionality.
662  *
663  * Note that there is another evil here: If you define a cpumask_var_t
664  * as a percpu variable then the way to obtain the address of the cpumask
665  * structure differently influences what this_cpu_* operation needs to be
666  * used. Please use this_cpu_cpumask_var_t in those cases. The direct use
667  * of this_cpu_ptr() or this_cpu_read() will lead to failures when the
668  * other type of cpumask_var_t implementation is configured.
669  */
670 #ifdef CONFIG_CPUMASK_OFFSTACK
671 typedef struct cpumask *cpumask_var_t;
672 
673 #define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
674 
675 bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
676 bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
677 bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
678 bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
679 void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
680 void free_cpumask_var(cpumask_var_t mask);
681 void free_bootmem_cpumask_var(cpumask_var_t mask);
682 
cpumask_available(cpumask_var_t mask)683 static inline bool cpumask_available(cpumask_var_t mask)
684 {
685 	return mask != NULL;
686 }
687 
688 #else
689 typedef struct cpumask cpumask_var_t[1];
690 
691 #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
692 
alloc_cpumask_var(cpumask_var_t * mask,gfp_t flags)693 static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
694 {
695 	return true;
696 }
697 
alloc_cpumask_var_node(cpumask_var_t * mask,gfp_t flags,int node)698 static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
699 					  int node)
700 {
701 	return true;
702 }
703 
zalloc_cpumask_var(cpumask_var_t * mask,gfp_t flags)704 static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
705 {
706 	cpumask_clear(*mask);
707 	return true;
708 }
709 
zalloc_cpumask_var_node(cpumask_var_t * mask,gfp_t flags,int node)710 static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
711 					  int node)
712 {
713 	cpumask_clear(*mask);
714 	return true;
715 }
716 
alloc_bootmem_cpumask_var(cpumask_var_t * mask)717 static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
718 {
719 }
720 
free_cpumask_var(cpumask_var_t mask)721 static inline void free_cpumask_var(cpumask_var_t mask)
722 {
723 }
724 
free_bootmem_cpumask_var(cpumask_var_t mask)725 static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
726 {
727 }
728 
cpumask_available(cpumask_var_t mask)729 static inline bool cpumask_available(cpumask_var_t mask)
730 {
731 	return true;
732 }
733 #endif /* CONFIG_CPUMASK_OFFSTACK */
734 
735 /* It's common to want to use cpu_all_mask in struct member initializers,
736  * so it has to refer to an address rather than a pointer. */
737 extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
738 #define cpu_all_mask to_cpumask(cpu_all_bits)
739 
740 /* First bits of cpu_bit_bitmap are in fact unset. */
741 #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
742 
743 #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
744 #define for_each_online_cpu(cpu)   for_each_cpu((cpu), cpu_online_mask)
745 #define for_each_present_cpu(cpu)  for_each_cpu((cpu), cpu_present_mask)
746 
747 /* Wrappers for arch boot code to manipulate normally-constant masks */
748 void init_cpu_present(const struct cpumask *src);
749 void init_cpu_possible(const struct cpumask *src);
750 void init_cpu_online(const struct cpumask *src);
751 
752 static inline void
set_cpu_possible(unsigned int cpu,bool possible)753 set_cpu_possible(unsigned int cpu, bool possible)
754 {
755 	if (possible)
756 		cpumask_set_cpu(cpu, &__cpu_possible_mask);
757 	else
758 		cpumask_clear_cpu(cpu, &__cpu_possible_mask);
759 }
760 
761 static inline void
set_cpu_present(unsigned int cpu,bool present)762 set_cpu_present(unsigned int cpu, bool present)
763 {
764 	if (present)
765 		cpumask_set_cpu(cpu, &__cpu_present_mask);
766 	else
767 		cpumask_clear_cpu(cpu, &__cpu_present_mask);
768 }
769 
770 static inline void
set_cpu_online(unsigned int cpu,bool online)771 set_cpu_online(unsigned int cpu, bool online)
772 {
773 	if (online)
774 		cpumask_set_cpu(cpu, &__cpu_online_mask);
775 	else
776 		cpumask_clear_cpu(cpu, &__cpu_online_mask);
777 }
778 
779 static inline void
set_cpu_active(unsigned int cpu,bool active)780 set_cpu_active(unsigned int cpu, bool active)
781 {
782 	if (active)
783 		cpumask_set_cpu(cpu, &__cpu_active_mask);
784 	else
785 		cpumask_clear_cpu(cpu, &__cpu_active_mask);
786 }
787 
788 
789 /**
790  * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
791  * @bitmap: the bitmap
792  *
793  * There are a few places where cpumask_var_t isn't appropriate and
794  * static cpumasks must be used (eg. very early boot), yet we don't
795  * expose the definition of 'struct cpumask'.
796  *
797  * This does the conversion, and can be used as a constant initializer.
798  */
799 #define to_cpumask(bitmap)						\
800 	((struct cpumask *)(1 ? (bitmap)				\
801 			    : (void *)sizeof(__check_is_bitmap(bitmap))))
802 
__check_is_bitmap(const unsigned long * bitmap)803 static inline int __check_is_bitmap(const unsigned long *bitmap)
804 {
805 	return 1;
806 }
807 
808 /*
809  * Special-case data structure for "single bit set only" constant CPU masks.
810  *
811  * We pre-generate all the 64 (or 32) possible bit positions, with enough
812  * padding to the left and the right, and return the constant pointer
813  * appropriately offset.
814  */
815 extern const unsigned long
816 	cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
817 
get_cpu_mask(unsigned int cpu)818 static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
819 {
820 	const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
821 	p -= cpu / BITS_PER_LONG;
822 	return to_cpumask(p);
823 }
824 
825 #define cpu_is_offline(cpu)	unlikely(!cpu_online(cpu))
826 
827 #if NR_CPUS <= BITS_PER_LONG
828 #define CPU_BITS_ALL						\
829 {								\
830 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
831 }
832 
833 #else /* NR_CPUS > BITS_PER_LONG */
834 
835 #define CPU_BITS_ALL						\
836 {								\
837 	[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,		\
838 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
839 }
840 #endif /* NR_CPUS > BITS_PER_LONG */
841 
842 /**
843  * cpumap_print_to_pagebuf  - copies the cpumask into the buffer either
844  *	as comma-separated list of cpus or hex values of cpumask
845  * @list: indicates whether the cpumap must be list
846  * @mask: the cpumask to copy
847  * @buf: the buffer to copy into
848  *
849  * Returns the length of the (null-terminated) @buf string, zero if
850  * nothing is copied.
851  */
852 static inline ssize_t
cpumap_print_to_pagebuf(bool list,char * buf,const struct cpumask * mask)853 cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
854 {
855 	return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
856 				      nr_cpu_ids);
857 }
858 
859 #if NR_CPUS <= BITS_PER_LONG
860 #define CPU_MASK_ALL							\
861 (cpumask_t) { {								\
862 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
863 } }
864 #else
865 #define CPU_MASK_ALL							\
866 (cpumask_t) { {								\
867 	[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,			\
868 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
869 } }
870 #endif /* NR_CPUS > BITS_PER_LONG */
871 
872 #define CPU_MASK_NONE							\
873 (cpumask_t) { {								\
874 	[0 ... BITS_TO_LONGS(NR_CPUS)-1] =  0UL				\
875 } }
876 
877 #define CPU_MASK_CPU0							\
878 (cpumask_t) { {								\
879 	[0] =  1UL							\
880 } }
881 
882 #endif /* __LINUX_CPUMASK_H */
883