Lines Matching +full:1 +full:- +full:cpu
1 /* SPDX-License-Identifier: GPL-2.0 */
7 * set of CPU's in a system, one bit position per CPU number. In general,
22 * cpumask_bits - get the bits in a cpumask
26 * a macro so it's const-correct.
28 #define cpumask_bits(maskp) ((maskp)->bits)
31 * cpumask_pr_args - printf args to output a cpumask
38 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
46 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) in set_nr_cpu_ids()
58 * optimized routines that work for the single-word case, but only when
68 * they set bits or just don't have any faster fixed-sized versions. We
74 * optimization comes from being able to potentially use a compile-time
75 * constant instead of a run-time generated exact number of CPUs.
93 * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
94 * cpu_present_mask - has bit 'cpu' set iff cpu is populated
95 * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
96 * cpu_active_mask - has bit 'cpu' set iff cpu available to migration
100 * The cpu_possible_mask is fixed at boot time, as the set of CPU id's
115 * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
116 * assumption that their single CPU is online. The UP
120 * optimization - don't waste any instructions or memory references
122 * only one CPU.
140 static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits) in cpu_max_bits_warn() argument
143 WARN_ON_ONCE(cpu >= bits); in cpu_max_bits_warn()
147 /* verify cpu argument to cpumask_* operators */
148 static __always_inline unsigned int cpumask_check(unsigned int cpu) in cpumask_check() argument
150 cpu_max_bits_warn(cpu, small_cpumask_bits); in cpumask_check()
151 return cpu; in cpumask_check()
155 * cpumask_first - get the first cpu in a cpumask
166 * cpumask_first_zero - get the first unset cpu in a cpumask
177 * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
190 * cpumask_last - get the last CPU in a cpumask
191 * @srcp: - the cpumask pointer
201 * cpumask_next - get the next cpu in a cpumask
202 * @n: the cpu prior to the place to search (ie. return will be > @n)
210 /* -1 is a legal arg here. */ in cpumask_next()
211 if (n != -1) in cpumask_next()
213 return find_next_bit(cpumask_bits(srcp), small_cpumask_bits, n + 1); in cpumask_next()
217 * cpumask_next_zero - get the next unset cpu in a cpumask
218 * @n: the cpu prior to the place to search (ie. return will be > @n)
225 /* -1 is a legal arg here. */ in cpumask_next_zero()
226 if (n != -1) in cpumask_next_zero()
228 return find_next_zero_bit(cpumask_bits(srcp), small_cpumask_bits, n+1); in cpumask_next_zero()
231 #if NR_CPUS == 1
232 /* Uniprocessor: there is only one valid CPU */
256 * cpumask_next_and - get the next cpu in *src1p & *src2p
257 * @n: the cpu prior to the place to search (ie. return will be > @n)
267 /* -1 is a legal arg here. */ in cpumask_next_and()
268 if (n != -1) in cpumask_next_and()
271 small_cpumask_bits, n + 1); in cpumask_next_and()
275 * for_each_cpu - iterate over every cpu in a mask
276 * @cpu: the (optionally unsigned) integer iterator
279 * After the loop, cpu is >= nr_cpu_ids.
281 #define for_each_cpu(cpu, mask) \ argument
282 for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
284 #if NR_CPUS == 1
289 if (n != -1) in cpumask_next_wrap()
293 * Return the first available CPU when wrapping, or when starting before cpu0, in cpumask_next_wrap()
306 * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
307 * @cpu: the (optionally unsigned) integer iterator
313 * After the loop, cpu is >= nr_cpu_ids.
315 #define for_each_cpu_wrap(cpu, mask, start) \ argument
316 for_each_set_bit_wrap(cpu, cpumask_bits(mask), small_cpumask_bits, start)
319 * for_each_cpu_and - iterate over every cpu in both masks
320 * @cpu: the (optionally unsigned) integer iterator
324 * This saves a temporary CPU mask in many places. It is equivalent to:
327 * for_each_cpu(cpu, &tmp)
330 * After the loop, cpu is >= nr_cpu_ids.
332 #define for_each_cpu_and(cpu, mask1, mask2) \ argument
333 for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
336 * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
338 * @cpu: the (optionally unsigned) integer iterator
342 * This saves a temporary CPU mask in many places. It is equivalent to:
345 * for_each_cpu(cpu, &tmp)
348 * After the loop, cpu is >= nr_cpu_ids.
350 #define for_each_cpu_andnot(cpu, mask1, mask2) \ argument
351 for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
354 * for_each_cpu_or - iterate over every cpu present in either mask
355 * @cpu: the (optionally unsigned) integer iterator
359 * This saves a temporary CPU mask in many places. It is equivalent to:
362 * for_each_cpu(cpu, &tmp)
365 * After the loop, cpu is >= nr_cpu_ids.
367 #define for_each_cpu_or(cpu, mask1, mask2) \ argument
368 for_each_or_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
371 * cpumask_any_but - return a "random" in a cpumask, but not this one.
373 * @cpu: the cpu to ignore.
375 * Often used to find any cpu but smp_processor_id() in a mask.
379 unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) in cpumask_any_but() argument
383 cpumask_check(cpu); in cpumask_any_but()
385 if (i != cpu) in cpumask_any_but()
391 * cpumask_nth - get the first cpu in a cpumask
393 * @cpu: the N'th cpu to find, starting from 0
395 * Returns >= nr_cpu_ids if such cpu doesn't exist.
397 static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) in cpumask_nth() argument
399 return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu)); in cpumask_nth()
403 * cpumask_nth_and - get the first cpu in 2 cpumasks
406 * @cpu: the N'th cpu to find, starting from 0
408 * Returns >= nr_cpu_ids if such cpu doesn't exist.
411 unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, in cpumask_nth_and() argument
415 small_cpumask_bits, cpumask_check(cpu)); in cpumask_nth_and()
419 * cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd.
422 * @cpu: the N'th cpu to find, starting from 0
424 * Returns >= nr_cpu_ids if such cpu doesn't exist.
427 unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1, in cpumask_nth_andnot() argument
431 small_cpumask_bits, cpumask_check(cpu)); in cpumask_nth_andnot()
435 * cpumask_nth_and_andnot - get the Nth cpu set in 1st and 2nd cpumask, and clear in 3rd.
439 * @cpu: the N'th cpu to find, starting from 0
441 * Returns >= nr_cpu_ids if such cpu doesn't exist.
444 unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1, in cpumask_nth_and_andnot() argument
451 small_cpumask_bits, cpumask_check(cpu)); in cpumask_nth_and_andnot()
456 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
461 [0] = 1UL \
465 * cpumask_set_cpu - set a cpu in a cpumask
466 * @cpu: cpu number (< nr_cpu_ids)
469 static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) in cpumask_set_cpu() argument
471 set_bit(cpumask_check(cpu), cpumask_bits(dstp)); in cpumask_set_cpu()
474 static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) in __cpumask_set_cpu() argument
476 __set_bit(cpumask_check(cpu), cpumask_bits(dstp)); in __cpumask_set_cpu()
481 * cpumask_clear_cpu - clear a cpu in a cpumask
482 * @cpu: cpu number (< nr_cpu_ids)
485 static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp) in cpumask_clear_cpu() argument
487 clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); in cpumask_clear_cpu()
490 static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp) in __cpumask_clear_cpu() argument
492 __clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); in __cpumask_clear_cpu()
496 * cpumask_test_cpu - test for a cpu in a cpumask
497 * @cpu: cpu number (< nr_cpu_ids)
500 * Returns true if @cpu is set in @cpumask, else returns false
502 static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask) in cpumask_test_cpu() argument
504 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); in cpumask_test_cpu()
508 * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
509 * @cpu: cpu number (< nr_cpu_ids)
512 * Returns true if @cpu is set in old bitmap of @cpumask, else returns false
516 static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) in cpumask_test_and_set_cpu() argument
518 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask)); in cpumask_test_and_set_cpu()
522 * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
523 * @cpu: cpu number (< nr_cpu_ids)
526 * Returns true if @cpu is set in old bitmap of @cpumask, else returns false
530 static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) in cpumask_test_and_clear_cpu() argument
532 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask)); in cpumask_test_and_clear_cpu()
536 * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
549 * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
558 * cpumask_and - *dstp = *src1p & *src2p
574 * cpumask_or - *dstp = *src1p | *src2p
587 * cpumask_xor - *dstp = *src1p ^ *src2p
601 * cpumask_andnot - *dstp = *src1p & ~*src2p
617 * cpumask_equal - *src1p == *src2p
629 * cpumask_or_equal - *src1p | *src2p == *src3p
643 * cpumask_intersects - (*src1p & *src2p) != 0
655 * cpumask_subset - (*src1p & ~*src2p) == 0
669 * cpumask_empty - *srcp == 0
678 * cpumask_full - *srcp == 0xFFFFFFFF...
687 * cpumask_weight - Count of bits in *srcp
696 * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2)
707 * cpumask_shift_right - *dstp = *srcp >> n
720 * cpumask_shift_left - *dstp = *srcp << n
733 * cpumask_copy - *dstp = *srcp
744 * cpumask_any - pick a "random" cpu from *srcp
752 * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
761 * cpumask_of - the cpumask containing just a given cpu
762 * @cpu: the cpu (<= nr_cpu_ids)
764 #define cpumask_of(cpu) (get_cpu_mask(cpu)) argument
767 * cpumask_parse_user - extract a cpumask from a user string
772 * Returns -errno, or 0 for success.
781 * cpumask_parselist_user - extract a cpumask from a user string
786 * Returns -errno, or 0 for success.
796 * cpumask_parse - extract a cpumask from a string
800 * Returns -errno, or 0 for success.
808 * cpulist_parse - extract a cpumask from a user string of ranges
812 * Returns -errno, or 0 for success.
820 * cpumask_size - size to allocate for a 'struct cpumask' in bytes
837 * return -ENOMEM;
850 * return -ENOMEM;
882 * alloc_cpumask_var - allocate a struct cpumask
887 * a nop returning a constant 1 (in <linux/cpumask.h>).
913 typedef struct cpumask cpumask_var_t[1];
968 #if NR_CPUS == 1
969 /* Uniprocessor: the possible/online/present masks are always "1" */
970 #define for_each_possible_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) argument
971 #define for_each_online_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) argument
972 #define for_each_present_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) argument
974 #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask) argument
975 #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask) argument
976 #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask) argument
979 /* Wrappers for arch boot code to manipulate normally-constant masks */
990 set_cpu_possible(unsigned int cpu, bool possible) in set_cpu_possible() argument
993 cpumask_set_cpu(cpu, &__cpu_possible_mask); in set_cpu_possible()
995 cpumask_clear_cpu(cpu, &__cpu_possible_mask); in set_cpu_possible()
999 set_cpu_present(unsigned int cpu, bool present) in set_cpu_present() argument
1002 cpumask_set_cpu(cpu, &__cpu_present_mask); in set_cpu_present()
1004 cpumask_clear_cpu(cpu, &__cpu_present_mask); in set_cpu_present()
1007 void set_cpu_online(unsigned int cpu, bool online);
1010 set_cpu_active(unsigned int cpu, bool active) in set_cpu_active() argument
1013 cpumask_set_cpu(cpu, &__cpu_active_mask); in set_cpu_active()
1015 cpumask_clear_cpu(cpu, &__cpu_active_mask); in set_cpu_active()
1019 set_cpu_dying(unsigned int cpu, bool dying) in set_cpu_dying() argument
1022 cpumask_set_cpu(cpu, &__cpu_dying_mask); in set_cpu_dying()
1024 cpumask_clear_cpu(cpu, &__cpu_dying_mask); in set_cpu_dying()
1028 * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
1038 ((struct cpumask *)(1 ? (bitmap) \
1043 return 1; in __check_is_bitmap()
1047 * Special-case data structure for "single bit set only" constant CPU masks.
1049 * We pre-generate all the 64 (or 32) possible bit positions, with enough
1054 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
1056 static inline const struct cpumask *get_cpu_mask(unsigned int cpu) in get_cpu_mask() argument
1058 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG]; in get_cpu_mask()
1059 p -= cpu / BITS_PER_LONG; in get_cpu_mask()
1063 #if NR_CPUS > 1
1065 * num_online_cpus() - Read the number of online CPUs
1069 * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
1080 static inline bool cpu_online(unsigned int cpu) in cpu_online() argument
1082 return cpumask_test_cpu(cpu, cpu_online_mask); in cpu_online()
1085 static inline bool cpu_possible(unsigned int cpu) in cpu_possible() argument
1087 return cpumask_test_cpu(cpu, cpu_possible_mask); in cpu_possible()
1090 static inline bool cpu_present(unsigned int cpu) in cpu_present() argument
1092 return cpumask_test_cpu(cpu, cpu_present_mask); in cpu_present()
1095 static inline bool cpu_active(unsigned int cpu) in cpu_active() argument
1097 return cpumask_test_cpu(cpu, cpu_active_mask); in cpu_active()
1100 static inline bool cpu_dying(unsigned int cpu) in cpu_dying() argument
1102 return cpumask_test_cpu(cpu, cpu_dying_mask); in cpu_dying()
1107 #define num_online_cpus() 1U
1108 #define num_possible_cpus() 1U
1109 #define num_present_cpus() 1U
1110 #define num_active_cpus() 1U
1112 static inline bool cpu_online(unsigned int cpu) in cpu_online() argument
1114 return cpu == 0; in cpu_online()
1117 static inline bool cpu_possible(unsigned int cpu) in cpu_possible() argument
1119 return cpu == 0; in cpu_possible()
1122 static inline bool cpu_present(unsigned int cpu) in cpu_present() argument
1124 return cpu == 0; in cpu_present()
1127 static inline bool cpu_active(unsigned int cpu) in cpu_active() argument
1129 return cpu == 0; in cpu_active()
1132 static inline bool cpu_dying(unsigned int cpu) in cpu_dying() argument
1137 #endif /* NR_CPUS > 1 */
1139 #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu)) argument
1144 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1151 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
1152 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1157 * cpumap_print_to_pagebuf - copies the cpumask into the buffer either
1158 * as comma-separated list of cpus or hex values of cpumask
1163 * Returns the length of the (null-terminated) @buf string, zero if
1174 * cpumap_print_bitmask_to_buf - copies the cpumask into the buffer as
1194 nr_cpu_ids, off, count) - 1; in cpumap_print_bitmask_to_buf()
1198 * cpumap_print_list_to_buf - copies the cpumask into the buffer as
1199 * comma-separated list of cpus
1213 nr_cpu_ids, off, count) - 1; in cpumap_print_list_to_buf()
1219 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1224 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
1225 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1231 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
1236 [0] = 1UL \
1244 * for cpumap NR_CPUS * 9/32 - 1 should be an exact length.
1246 * For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up
1248 * cover a worst-case of every other cpu being on one of two nodes for a
1252 * unsigned comparison to -1.
1255 ? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE)