364 static void pcpu_next_md_free_region(struct pcpu_chunk *chunk, int *bit_off,  in pcpu_next_md_free_region()  argument
367 	int i = pcpu_off_to_block_index(*bit_off);  in pcpu_next_md_free_region()
368 	int block_off = pcpu_off_to_block_off(*bit_off);  in pcpu_next_md_free_region()
393 			*bit_off = pcpu_block_off_to_off(i,  in pcpu_next_md_free_region()
401 		*bit_off = (i + 1) * PCPU_BITMAP_BLOCK_BITS - block->right_free;  in pcpu_next_md_free_region()
420 				 int align, int *bit_off, int *bits)  in pcpu_next_fit_region()  argument
422 	int i = pcpu_off_to_block_index(*bit_off);  in pcpu_next_fit_region()
423 	int block_off = pcpu_off_to_block_off(*bit_off);  in pcpu_next_fit_region()
452 			*bit_off = pcpu_block_off_to_off(i, start);  in pcpu_next_fit_region()
458 		*bit_off = ALIGN(PCPU_BITMAP_BLOCK_BITS - block->right_free,  in pcpu_next_fit_region()
460 		*bits = PCPU_BITMAP_BLOCK_BITS - *bit_off;  in pcpu_next_fit_region()
461 		*bit_off = pcpu_block_off_to_off(i, *bit_off);  in pcpu_next_fit_region()
467 	*bit_off = pcpu_chunk_map_bits(chunk);  in pcpu_next_fit_region()
476 #define pcpu_for_each_md_free_region(chunk, bit_off, bits)		\  argument
477 	for (pcpu_next_md_free_region((chunk), &(bit_off), &(bits));	\
478 	     (bit_off) < pcpu_chunk_map_bits((chunk));			\
479 	     (bit_off) += (bits) + 1,					\
480 	     pcpu_next_md_free_region((chunk), &(bit_off), &(bits)))
482 #define pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits)     \  argument
483 	for (pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
485 	     (bit_off) < pcpu_chunk_map_bits((chunk));			      \
486 	     (bit_off) += (bits),					      \
487 	     pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
685 static void pcpu_block_update_scan(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_scan()  argument
688 	int s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_scan()
696 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_scan()
721 	int bit_off, bits;  in pcpu_chunk_refresh_hint()  local
725 		bit_off = chunk_md->scan_hint_start + chunk_md->scan_hint;  in pcpu_chunk_refresh_hint()
730 		bit_off = chunk_md->first_free;  in pcpu_chunk_refresh_hint()
735 	pcpu_for_each_md_free_region(chunk, bit_off, bits) {  in pcpu_chunk_refresh_hint()
736 		pcpu_block_update(chunk_md, bit_off, bit_off + bits);  in pcpu_chunk_refresh_hint()
784 static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_hint_alloc()  argument
799 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_hint_alloc()
800 	e_index = pcpu_off_to_block_index(bit_off + bits - 1);  in pcpu_block_update_hint_alloc()
801 	s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_hint_alloc()
802 	e_off = pcpu_off_to_block_off(bit_off + bits - 1) + 1;  in pcpu_block_update_hint_alloc()
896 				bit_off,  in pcpu_block_update_hint_alloc()
897 				bit_off + bits))  in pcpu_block_update_hint_alloc()
908 				bit_off,  in pcpu_block_update_hint_alloc()
909 				bit_off + bits))  in pcpu_block_update_hint_alloc()
931 static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_hint_free()  argument
946 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_hint_free()
947 	e_index = pcpu_off_to_block_index(bit_off + bits - 1);  in pcpu_block_update_hint_free()
948 	s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_hint_free()
949 	e_off = pcpu_off_to_block_off(bit_off + bits - 1) + 1;  in pcpu_block_update_hint_free()
1041 static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits,  in pcpu_is_populated()  argument
1046 	page_start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE);  in pcpu_is_populated()
1047 	page_end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);  in pcpu_is_populated()
1081 	int bit_off, bits, next_off;  in pcpu_find_block_fit()  local
1089 	bit_off = ALIGN(chunk_md->contig_hint_start, align) -  in pcpu_find_block_fit()
1091 	if (bit_off + alloc_bits > chunk_md->contig_hint)  in pcpu_find_block_fit()
1094 	bit_off = pcpu_next_hint(chunk_md, alloc_bits);  in pcpu_find_block_fit()
1096 	pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) {  in pcpu_find_block_fit()
1097 		if (!pop_only || pcpu_is_populated(chunk, bit_off, bits,  in pcpu_find_block_fit()
1101 		bit_off = next_off;  in pcpu_find_block_fit()
1105 	if (bit_off == pcpu_chunk_map_bits(chunk))  in pcpu_find_block_fit()
1108 	return bit_off;  in pcpu_find_block_fit()
1192 	int bit_off, end, oslot;  in pcpu_alloc_area()  local
1203 	bit_off = pcpu_find_zero_area(chunk->alloc_map, end, start, alloc_bits,  in pcpu_alloc_area()
1205 	if (bit_off >= end)  in pcpu_alloc_area()
1212 	bitmap_set(chunk->alloc_map, bit_off, alloc_bits);  in pcpu_alloc_area()
1215 	set_bit(bit_off, chunk->bound_map);  in pcpu_alloc_area()
1216 	bitmap_clear(chunk->bound_map, bit_off + 1, alloc_bits - 1);  in pcpu_alloc_area()
1217 	set_bit(bit_off + alloc_bits, chunk->bound_map);  in pcpu_alloc_area()
1222 	if (bit_off == chunk_md->first_free)  in pcpu_alloc_area()
1226 					bit_off + alloc_bits);  in pcpu_alloc_area()
1228 	pcpu_block_update_hint_alloc(chunk, bit_off, alloc_bits);  in pcpu_alloc_area()
1232 	return bit_off * PCPU_MIN_ALLOC_SIZE;  in pcpu_alloc_area()
1246 	int bit_off, bits, end, oslot;  in pcpu_free_area()  local
1253 	bit_off = off / PCPU_MIN_ALLOC_SIZE;  in pcpu_free_area()
1257 			    bit_off + 1);  in pcpu_free_area()
1258 	bits = end - bit_off;  in pcpu_free_area()
1259 	bitmap_clear(chunk->alloc_map, bit_off, bits);  in pcpu_free_area()
1265 	chunk_md->first_free = min(chunk_md->first_free, bit_off);  in pcpu_free_area()
1267 	pcpu_block_update_hint_free(chunk, bit_off, bits);  in pcpu_free_area()