324 	int bit_off = ALIGN(block->contig_hint_start, align) -  in pcpu_check_block_hint()  local
327 	return bit_off + bits <= block->contig_hint;  in pcpu_check_block_hint()
369 static void pcpu_next_md_free_region(struct pcpu_chunk *chunk, int *bit_off,  in pcpu_next_md_free_region()  argument
372 	int i = pcpu_off_to_block_index(*bit_off);  in pcpu_next_md_free_region()
373 	int block_off = pcpu_off_to_block_off(*bit_off);  in pcpu_next_md_free_region()
398 			*bit_off = pcpu_block_off_to_off(i,  in pcpu_next_md_free_region()
406 		*bit_off = (i + 1) * PCPU_BITMAP_BLOCK_BITS - block->right_free;  in pcpu_next_md_free_region()
425 				 int align, int *bit_off, int *bits)  in pcpu_next_fit_region()  argument
427 	int i = pcpu_off_to_block_index(*bit_off);  in pcpu_next_fit_region()
428 	int block_off = pcpu_off_to_block_off(*bit_off);  in pcpu_next_fit_region()
457 			*bit_off = pcpu_block_off_to_off(i, start);  in pcpu_next_fit_region()
463 		*bit_off = ALIGN(PCPU_BITMAP_BLOCK_BITS - block->right_free,  in pcpu_next_fit_region()
465 		*bits = PCPU_BITMAP_BLOCK_BITS - *bit_off;  in pcpu_next_fit_region()
466 		*bit_off = pcpu_block_off_to_off(i, *bit_off);  in pcpu_next_fit_region()
472 	*bit_off = pcpu_chunk_map_bits(chunk);  in pcpu_next_fit_region()
481 #define pcpu_for_each_md_free_region(chunk, bit_off, bits)		\  argument
482 	for (pcpu_next_md_free_region((chunk), &(bit_off), &(bits));	\
483 	     (bit_off) < pcpu_chunk_map_bits((chunk));			\
484 	     (bit_off) += (bits) + 1,					\
485 	     pcpu_next_md_free_region((chunk), &(bit_off), &(bits)))
487 #define pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits)     \  argument
488 	for (pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
490 	     (bit_off) < pcpu_chunk_map_bits((chunk));			      \
491 	     (bit_off) += (bits),					      \
492 	     pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
716 static void pcpu_block_update_scan(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_scan()  argument
719 	int s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_scan()
727 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_scan()
752 	int bit_off, bits;  in pcpu_chunk_refresh_hint()  local
756 		bit_off = chunk_md->scan_hint_start + chunk_md->scan_hint;  in pcpu_chunk_refresh_hint()
761 		bit_off = chunk_md->first_free;  in pcpu_chunk_refresh_hint()
766 	pcpu_for_each_md_free_region(chunk, bit_off, bits)  in pcpu_chunk_refresh_hint()
767 		pcpu_block_update(chunk_md, bit_off, bit_off + bits);  in pcpu_chunk_refresh_hint()
813 static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_hint_alloc()  argument
828 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_hint_alloc()
829 	e_index = pcpu_off_to_block_index(bit_off + bits - 1);  in pcpu_block_update_hint_alloc()
830 	s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_hint_alloc()
831 	e_off = pcpu_off_to_block_off(bit_off + bits - 1) + 1;  in pcpu_block_update_hint_alloc()
925 				bit_off,  in pcpu_block_update_hint_alloc()
926 				bit_off + bits))  in pcpu_block_update_hint_alloc()
937 				bit_off,  in pcpu_block_update_hint_alloc()
938 				bit_off + bits))  in pcpu_block_update_hint_alloc()
960 static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_hint_free()  argument
975 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_hint_free()
976 	e_index = pcpu_off_to_block_index(bit_off + bits - 1);  in pcpu_block_update_hint_free()
977 	s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_hint_free()
978 	e_off = pcpu_off_to_block_off(bit_off + bits - 1) + 1;  in pcpu_block_update_hint_free()
1070 static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits,  in pcpu_is_populated()  argument
1075 	page_start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE);  in pcpu_is_populated()
1076 	page_end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);  in pcpu_is_populated()
1110 	int bit_off, bits, next_off;  in pcpu_find_block_fit()  local
1120 	bit_off = pcpu_next_hint(chunk_md, alloc_bits);  in pcpu_find_block_fit()
1122 	pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) {  in pcpu_find_block_fit()
1123 		if (!pop_only || pcpu_is_populated(chunk, bit_off, bits,  in pcpu_find_block_fit()
1127 		bit_off = next_off;  in pcpu_find_block_fit()
1131 	if (bit_off == pcpu_chunk_map_bits(chunk))  in pcpu_find_block_fit()
1134 	return bit_off;  in pcpu_find_block_fit()
1218 	int bit_off, end, oslot;  in pcpu_alloc_area()  local
1229 	bit_off = pcpu_find_zero_area(chunk->alloc_map, end, start, alloc_bits,  in pcpu_alloc_area()
1231 	if (bit_off >= end)  in pcpu_alloc_area()
1238 	bitmap_set(chunk->alloc_map, bit_off, alloc_bits);  in pcpu_alloc_area()
1241 	set_bit(bit_off, chunk->bound_map);  in pcpu_alloc_area()
1242 	bitmap_clear(chunk->bound_map, bit_off + 1, alloc_bits - 1);  in pcpu_alloc_area()
1243 	set_bit(bit_off + alloc_bits, chunk->bound_map);  in pcpu_alloc_area()
1248 	if (bit_off == chunk_md->first_free)  in pcpu_alloc_area()
1252 					bit_off + alloc_bits);  in pcpu_alloc_area()
1254 	pcpu_block_update_hint_alloc(chunk, bit_off, alloc_bits);  in pcpu_alloc_area()
1258 	return bit_off * PCPU_MIN_ALLOC_SIZE;  in pcpu_alloc_area()
1275 	int bit_off, bits, end, oslot, freed;  in pcpu_free_area()  local
1282 	bit_off = off / PCPU_MIN_ALLOC_SIZE;  in pcpu_free_area()
1286 			    bit_off + 1);  in pcpu_free_area()
1287 	bits = end - bit_off;  in pcpu_free_area()
1288 	bitmap_clear(chunk->alloc_map, bit_off, bits);  in pcpu_free_area()
1296 	chunk_md->first_free = min(chunk_md->first_free, bit_off);  in pcpu_free_area()
1298 	pcpu_block_update_hint_free(chunk, bit_off, bits);  in pcpu_free_area()