1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
18 #include <linux/backing-dev.h>
23  *   - test ext4_ext_search_left() and ext4_ext_search_right()
24  *   - search for metadata in few groups
27  *   - normalization should take into account whether file is still open
28  *   - discard preallocations if no free space left (policy?)
29  *   - don't normalize tails
30  *   - quota
31  *   - reservation for superuser
34  *   - bitmap read-ahead (proposed by Oleg Drokin aka green)
35  *   - track min/max extents in each group for better group selection
36  *   - mb_mark_used() may allocate chunk right after splitting buddy
37  *   - tree of groups sorted by number of free blocks
38  *   - error handling
49  * is larger. If the size is less than sbi->s_mb_stream_request we
59  * ext4_inode_info->i_prealloc_list, which contains list of prealloc
63  * pa_lstart -> the logical start block for this prealloc space
64  * pa_pstart -> the physical start block for this prealloc space
65  * pa_len    -> length for this prealloc space (in clusters)
66  * pa_free   ->  free space available in this prealloc space (in clusters)
78  * have the group allocation flag set then we look at the locality group
83  * The reason for having a per cpu locality group is to reduce the contention
86  * The locality group prealloc space is used looking at whether we have
89  * If we can't allocate blocks via inode prealloc or/and locality group
121  * sbi->s_mb_group_prealloc.  The default value of s_mb_group_prealloc is
122  * dependent on the cluster size; for non-bigalloc file systems, it is
125  * terms of number of blocks. If we have mounted the file system with -O
127  * smallest multiple of the stripe value (sbi->s_stripe) which is
137  * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The
140  * stripe size (sbi->s_stripe), we try to search for contiguous block in
142  * not, we search in the specific group using bitmap for best extents. The
160  *  - on-disk bitmap
161  *  - in-core buddy (actually includes buddy and bitmap)
162  *  - preallocation descriptors (PAs)
165  *  - inode
166  *    assiged to specific inode and can be used for this inode only.
167  *    it describes part of inode's space preallocated to specific
174  *  - locality group
175  *    assigned to specific locality group which does not translate to
181  *    in-core buddy = on-disk bitmap + preallocation descriptors
184  *  - allocated blocks (persistent)
185  *  - preallocated blocks (non-persistent)
189  * literally -- time is discrete and delimited by locks.
192  *  blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA.
195  *  - init buddy:			buddy = on-disk + PAs
196  *  - new PA:				buddy += N; PA = N
197  *  - use inode PA:			on-disk += N; PA -= N
198  *  - discard inode PA			buddy -= on-disk - PA; PA = 0
199  *  - use locality group PA		on-disk += N; PA -= N
200  *  - discard locality group PA		buddy -= PA; PA = 0
201  *  note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap
203  *        bits from PA, only from on-disk bitmap
207  * killing performance on high-end SMP hardware. let's try to relax it using
211  *     nobody can re-allocate that block
212  *  3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has
214  *     on-disk bitmap if buddy has same bit set or/and PA covers corresponded
218  *  - init buddy vs.
219  *    - new PA
222  *    - use inode PA
223  *      we need to make sure that either on-disk bitmap or PA has uptodate data
224  *      given (3) we care that PA-=N operation doesn't interfere with init
225  *    - discard inode PA
227  *    - use locality group PA
228  *      again PA-=N must be serialized with init
229  *    - discard locality group PA
231  *  - new PA vs.
232  *    - use inode PA
234  *    - discard inode PA
236  *    - use locality group PA
238  *    - discard locality group PA
240  *  - use inode PA
241  *    - use inode PA
243  *    - discard inode PA
245  *    - use locality group PA
246  *      nothing wrong here -- they're different PAs covering different blocks
247  *    - discard locality group PA
251  *  - PA is referenced and while it is no discard is possible
252  *  - PA is referenced until block isn't marked in on-disk bitmap
253  *  - PA changes only after on-disk bitmap
254  *  - discard must not compete with init. either init is done before
256  *  - buddy init as sum of on-disk bitmap and PAs is done atomically
266  *  - allocation:
269  *    mark bits in on-disk bitmap
272  *  - use preallocation:
273  *    find proper PA (per-inode or group)
275  *    mark bits in on-disk bitmap
279  *  - free:
281  *    mark bits in on-disk bitmap
284  *  - discard preallocations in group:
287  *    load on-disk bitmap
289  *    remove PA from object (inode or locality group)
290  *    mark free blocks in-core
292  *  - discard inode's preallocations:
299  *  - bitlock on a group	(group)
300  *  - object (inode/locality)	(object)
301  *  - per-pa lock		(pa)
304  *  - new pa
308  *  - find and use pa:
311  *  - release consumed pa:
316  *  - generate in-core bitmap:
320  *  - discard all for given object (inode, locality group):
325  *  - discard all for given group:
370  * sure that all the PAs on grp->bb_prealloc_list got freed or if it's empty.
432 	ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix;  in mb_find_next_zero_bit()
445 	ret = ext4_find_next_bit(addr, tmpmax, start) - fix;  in mb_find_next_bit()
455 	BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);  in mb_find_buddy()
458 	if (order > e4b->bd_blkbits + 1) {  in mb_find_buddy()
465 		*max = 1 << (e4b->bd_blkbits + 3);  in mb_find_buddy()
466 		return e4b->bd_bitmap;  in mb_find_buddy()
469 	bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order];  in mb_find_buddy()
470 	*max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order];  in mb_find_buddy()
480 	struct super_block *sb = e4b->bd_sb;  in mb_free_blocks_double()
482 	if (unlikely(e4b->bd_info->bb_bitmap == NULL))  in mb_free_blocks_double()
484 	assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));  in mb_free_blocks_double()
486 		if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) {  in mb_free_blocks_double()
489 			blocknr = ext4_group_first_block_no(sb, e4b->bd_group);  in mb_free_blocks_double()
491 			ext4_grp_locked_error(sb, e4b->bd_group,  in mb_free_blocks_double()
492 					      inode ? inode->i_ino : 0,  in mb_free_blocks_double()
497 			ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in mb_free_blocks_double()
500 		mb_clear_bit(first + i, e4b->bd_info->bb_bitmap);  in mb_free_blocks_double()
508 	if (unlikely(e4b->bd_info->bb_bitmap == NULL))  in mb_mark_used_double()
510 	assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));  in mb_mark_used_double()
512 		BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap));  in mb_mark_used_double()
513 		mb_set_bit(first + i, e4b->bd_info->bb_bitmap);  in mb_mark_used_double()
519 	if (unlikely(e4b->bd_info->bb_bitmap == NULL))  in mb_cmp_bitmaps()
521 	if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) {  in mb_cmp_bitmaps()
524 		b1 = (unsigned char *) e4b->bd_info->bb_bitmap;  in mb_cmp_bitmaps()
526 		for (i = 0; i < e4b->bd_sb->s_blocksize; i++) {  in mb_cmp_bitmaps()
528 				ext4_msg(e4b->bd_sb, KERN_ERR,  in mb_cmp_bitmaps()
532 					 e4b->bd_group, i, i * 8, b1[i], b2[i]);  in mb_cmp_bitmaps()
544 	grp->bb_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS);  in mb_group_bb_bitmap_alloc()
545 	if (!grp->bb_bitmap)  in mb_group_bb_bitmap_alloc()
550 		kfree(grp->bb_bitmap);  in mb_group_bb_bitmap_alloc()
551 		grp->bb_bitmap = NULL;  in mb_group_bb_bitmap_alloc()
555 	memcpy(grp->bb_bitmap, bh->b_data, sb->s_blocksize);  in mb_group_bb_bitmap_alloc()
561 	kfree(grp->bb_bitmap);  in mb_group_bb_bitmap_free()
607 	struct super_block *sb = e4b->bd_sb;  in __mb_check_buddy()
608 	int order = e4b->bd_blkbits + 1;  in __mb_check_buddy()
622 	if (e4b->bd_info->bb_check_counter++ % 10)  in __mb_check_buddy()
628 		buddy2 = mb_find_buddy(e4b, order - 1, &max2);  in __mb_check_buddy()
655 					!mb_test_bit(k, e4b->bd_bitmap));  in __mb_check_buddy()
659 		MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count);  in __mb_check_buddy()
660 		order--;  in __mb_check_buddy()
663 	fstart = -1;  in __mb_check_buddy()
667 			MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free);  in __mb_check_buddy()
668 			if (fstart == -1) {  in __mb_check_buddy()
674 		fstart = -1;  in __mb_check_buddy()
676 		for (j = 0; j < e4b->bd_blkbits + 1; j++) {  in __mb_check_buddy()
683 	MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info));  in __mb_check_buddy()
684 	MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments);  in __mb_check_buddy()
686 	grp = ext4_get_group_info(sb, e4b->bd_group);  in __mb_check_buddy()
687 	list_for_each(cur, &grp->bb_prealloc_list) {  in __mb_check_buddy()
691 		ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k);  in __mb_check_buddy()
692 		MB_CHECK_ASSERT(groupnr == e4b->bd_group);  in __mb_check_buddy()
693 		for (i = 0; i < pa->pa_len; i++)  in __mb_check_buddy()
723 	border = 2 << sb->s_blocksize_bits;  in ext4_mb_mark_free_simple()
727 		max = ffs(first | border) - 1;  in ext4_mb_mark_free_simple()
730 		min = fls(len) - 1;  in ext4_mb_mark_free_simple()
737 		grp->bb_counters[min]++;  in ext4_mb_mark_free_simple()
740 				     buddy + sbi->s_mb_offsets[min]);  in ext4_mb_mark_free_simple()
742 		len -= chunk;  in ext4_mb_mark_free_simple()
757 	grp->bb_largest_free_order = -1; /* uninit */  in mb_set_largest_free_order()
759 	bits = sb->s_blocksize_bits + 1;  in mb_set_largest_free_order()
760 	for (i = bits; i >= 0; i--) {  in mb_set_largest_free_order()
761 		if (grp->bb_counters[i] > 0) {  in mb_set_largest_free_order()
762 			grp->bb_largest_free_order = i;  in mb_set_largest_free_order()
783 	 * of on-disk bitmap and preallocations */  in ext4_mb_generate_buddy()
785 	grp->bb_first_free = i;  in ext4_mb_generate_buddy()
790 		len = i - first;  in ext4_mb_generate_buddy()
795 			grp->bb_counters[0]++;  in ext4_mb_generate_buddy()
799 	grp->bb_fragments = fragments;  in ext4_mb_generate_buddy()
801 	if (free != grp->bb_free) {  in ext4_mb_generate_buddy()
805 				      free, grp->bb_free);  in ext4_mb_generate_buddy()
810 		grp->bb_free = free;  in ext4_mb_generate_buddy()
816 	clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state));  in ext4_mb_generate_buddy()
818 	period = get_cycles() - period;  in ext4_mb_generate_buddy()
819 	spin_lock(&sbi->s_bal_lock);  in ext4_mb_generate_buddy()
820 	sbi->s_mb_buddies_generated++;  in ext4_mb_generate_buddy()
821 	sbi->s_mb_generation_time += period;  in ext4_mb_generate_buddy()
822 	spin_unlock(&sbi->s_bal_lock);  in ext4_mb_generate_buddy()
834 	e4b->bd_info->bb_fragments = 0;  in mb_regenerate_buddy()
835 	memset(e4b->bd_info->bb_counters, 0,  in mb_regenerate_buddy()
836 		sizeof(*e4b->bd_info->bb_counters) *  in mb_regenerate_buddy()
837 		(e4b->bd_sb->s_blocksize_bits + 2));  in mb_regenerate_buddy()
839 	ext4_mb_generate_buddy(e4b->bd_sb, e4b->bd_buddy,  in mb_regenerate_buddy()
840 		e4b->bd_bitmap, e4b->bd_group);  in mb_regenerate_buddy()
881 	inode = page->mapping->host;  in ext4_mb_init_cache()
882 	sb = inode->i_sb;  in ext4_mb_init_cache()
887 	mb_debug(sb, "init page %lu\n", page->index);  in ext4_mb_init_cache()
898 			err = -ENOMEM;  in ext4_mb_init_cache()
904 	first_group = page->index * blocks_per_page / 2;  in ext4_mb_init_cache()
942 	first_block = page->index * blocks_per_page;  in ext4_mb_init_cache()
948 		if (!bh[group - first_group])  in ext4_mb_init_cache()
952 		if (!buffer_verified(bh[group - first_group]))  in ext4_mb_init_cache()
964 		bitmap = bh[group - first_group]->b_data;  in ext4_mb_init_cache()
974 				group, page->index, i * blocksize);  in ext4_mb_init_cache()
977 			grinfo->bb_fragments = 0;  in ext4_mb_init_cache()
978 			memset(grinfo->bb_counters, 0,  in ext4_mb_init_cache()
979 			       sizeof(*grinfo->bb_counters) *  in ext4_mb_init_cache()
980 				(sb->s_blocksize_bits+2));  in ext4_mb_init_cache()
994 				group, page->index, i * blocksize);  in ext4_mb_init_cache()
1001 			/* mark all preallocated blks used in in-core bitmap */  in ext4_mb_init_cache()
1028  * are on the same page e4b->bd_buddy_page is NULL and return value is 0.
1033 	struct inode *inode = EXT4_SB(sb)->s_buddy_cache;  in ext4_mb_get_buddy_page_lock()
1038 	e4b->bd_buddy_page = NULL;  in ext4_mb_get_buddy_page_lock()
1039 	e4b->bd_bitmap_page = NULL;  in ext4_mb_get_buddy_page_lock()
1041 	blocks_per_page = PAGE_SIZE / sb->s_blocksize;  in ext4_mb_get_buddy_page_lock()
1050 	page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_get_buddy_page_lock()
1052 		return -ENOMEM;  in ext4_mb_get_buddy_page_lock()
1053 	BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_get_buddy_page_lock()
1054 	e4b->bd_bitmap_page = page;  in ext4_mb_get_buddy_page_lock()
1055 	e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);  in ext4_mb_get_buddy_page_lock()
1064 	page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_get_buddy_page_lock()
1066 		return -ENOMEM;  in ext4_mb_get_buddy_page_lock()
1067 	BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_get_buddy_page_lock()
1068 	e4b->bd_buddy_page = page;  in ext4_mb_get_buddy_page_lock()
1074 	if (e4b->bd_bitmap_page) {  in ext4_mb_put_buddy_page_lock()
1075 		unlock_page(e4b->bd_bitmap_page);  in ext4_mb_put_buddy_page_lock()
1076 		put_page(e4b->bd_bitmap_page);  in ext4_mb_put_buddy_page_lock()
1078 	if (e4b->bd_buddy_page) {  in ext4_mb_put_buddy_page_lock()
1079 		unlock_page(e4b->bd_buddy_page);  in ext4_mb_put_buddy_page_lock()
1080 		put_page(e4b->bd_buddy_page);  in ext4_mb_put_buddy_page_lock()
1124 		ret = -EIO;  in ext4_mb_init_group()
1143 		ret = -EIO;  in ext4_mb_init_group()
1168 	struct inode *inode = sbi->s_buddy_cache;  in ext4_mb_load_buddy_gfp()
1173 	blocks_per_page = PAGE_SIZE / sb->s_blocksize;  in ext4_mb_load_buddy_gfp()
1176 	e4b->bd_blkbits = sb->s_blocksize_bits;  in ext4_mb_load_buddy_gfp()
1177 	e4b->bd_info = grp;  in ext4_mb_load_buddy_gfp()
1178 	e4b->bd_sb = sb;  in ext4_mb_load_buddy_gfp()
1179 	e4b->bd_group = group;  in ext4_mb_load_buddy_gfp()
1180 	e4b->bd_buddy_page = NULL;  in ext4_mb_load_buddy_gfp()
1181 	e4b->bd_bitmap_page = NULL;  in ext4_mb_load_buddy_gfp()
1204 	page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED);  in ext4_mb_load_buddy_gfp()
1216 		page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_load_buddy_gfp()
1218 			BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_load_buddy_gfp()
1226 					       (poff * sb->s_blocksize));  in ext4_mb_load_buddy_gfp()
1232 		ret = -ENOMEM;  in ext4_mb_load_buddy_gfp()
1236 		ret = -EIO;  in ext4_mb_load_buddy_gfp()
1241 	e4b->bd_bitmap_page = page;  in ext4_mb_load_buddy_gfp()
1242 	e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);  in ext4_mb_load_buddy_gfp()
1248 	page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED);  in ext4_mb_load_buddy_gfp()
1252 		page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_load_buddy_gfp()
1254 			BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_load_buddy_gfp()
1256 				ret = ext4_mb_init_cache(page, e4b->bd_bitmap,  in ext4_mb_load_buddy_gfp()
1267 		ret = -ENOMEM;  in ext4_mb_load_buddy_gfp()
1271 		ret = -EIO;  in ext4_mb_load_buddy_gfp()
1276 	e4b->bd_buddy_page = page;  in ext4_mb_load_buddy_gfp()
1277 	e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize);  in ext4_mb_load_buddy_gfp()
1284 	if (e4b->bd_bitmap_page)  in ext4_mb_load_buddy_gfp()
1285 		put_page(e4b->bd_bitmap_page);  in ext4_mb_load_buddy_gfp()
1286 	if (e4b->bd_buddy_page)  in ext4_mb_load_buddy_gfp()
1287 		put_page(e4b->bd_buddy_page);  in ext4_mb_load_buddy_gfp()
1288 	e4b->bd_buddy = NULL;  in ext4_mb_load_buddy_gfp()
1289 	e4b->bd_bitmap = NULL;  in ext4_mb_load_buddy_gfp()
1301 	if (e4b->bd_bitmap_page)  in ext4_mb_unload_buddy()
1302 		put_page(e4b->bd_bitmap_page);  in ext4_mb_unload_buddy()
1303 	if (e4b->bd_buddy_page)  in ext4_mb_unload_buddy()
1304 		put_page(e4b->bd_buddy_page);  in ext4_mb_unload_buddy()
1311 	int bb_incr = 1 << (e4b->bd_blkbits - 1);  in mb_find_order_for_block()
1314 	BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);  in mb_find_order_for_block()
1315 	BUG_ON(block >= (1 << (e4b->bd_blkbits + 3)));  in mb_find_order_for_block()
1317 	bb = e4b->bd_buddy;  in mb_find_order_for_block()
1318 	while (order <= e4b->bd_blkbits + 1) {  in mb_find_order_for_block()
1337 		if ((cur & 31) == 0 && (len - cur) >= 32) {  in mb_clear_bits()
1350  * will return first found zero bit if any, -1 otherwise
1355 	int zero_bit = -1;  in mb_test_and_clear_bits()
1359 		if ((cur & 31) == 0 && (len - cur) >= 32) {  in mb_test_and_clear_bits()
1362 			if (*addr != (__u32)(-1) && zero_bit == -1)  in mb_test_and_clear_bits()
1368 		if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1)  in mb_test_and_clear_bits()
1382 		if ((cur & 31) == 0 && (len - cur) >= 32) {  in ext4_set_bits()
1398 		(*bit) -= side;  in mb_buddy_adjust_border()
1404 		return -1;  in mb_buddy_adjust_border()
1427 		 * ---------------------------------  in mb_buddy_mark_free()
1429 		 * ---------------------------------  in mb_buddy_mark_free()
1431 		 * ---------------------------------  in mb_buddy_mark_free()
1447 			e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1);  in mb_buddy_mark_free()
1449 			e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1);  in mb_buddy_mark_free()
1455 			mb_clear_bits(buddy, first, last - first + 1);  in mb_buddy_mark_free()
1456 			e4b->bd_info->bb_counters[order - 1] += last - first + 1;  in mb_buddy_mark_free()
1471 	int last = first + count - 1;  in mb_free_blocks()
1472 	struct super_block *sb = e4b->bd_sb;  in mb_free_blocks()
1476 	BUG_ON(last >= (sb->s_blocksize << 3));  in mb_free_blocks()
1477 	assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));  in mb_free_blocks()
1479 	if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))  in mb_free_blocks()
1486 	e4b->bd_info->bb_free += count;  in mb_free_blocks()
1487 	if (first < e4b->bd_info->bb_first_free)  in mb_free_blocks()
1488 		e4b->bd_info->bb_first_free = first;  in mb_free_blocks()
1494 		left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap);  in mb_free_blocks()
1495 	block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count);  in mb_free_blocks()
1496 	if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0])  in mb_free_blocks()
1497 		right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap);  in mb_free_blocks()
1499 	if (unlikely(block != -1)) {  in mb_free_blocks()
1503 		blocknr = ext4_group_first_block_no(sb, e4b->bd_group);  in mb_free_blocks()
1505 		if (!(sbi->s_mount_state & EXT4_FC_REPLAY)) {  in mb_free_blocks()
1506 			ext4_grp_locked_error(sb, e4b->bd_group,  in mb_free_blocks()
1507 					      inode ? inode->i_ino : 0,  in mb_free_blocks()
1512 				sb, e4b->bd_group,  in mb_free_blocks()
1521 		e4b->bd_info->bb_fragments--;  in mb_free_blocks()
1523 		e4b->bd_info->bb_fragments++;  in mb_free_blocks()
1533 		e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1;  in mb_free_blocks()
1536 		last -= !right_is_free;  in mb_free_blocks()
1537 		e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1;  in mb_free_blocks()
1544 	mb_set_largest_free_order(sb, e4b->bd_info);  in mb_free_blocks()
1555 	assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));  in mb_find_extent()
1562 		ex->fe_len = 0;  in mb_find_extent()
1563 		ex->fe_start = 0;  in mb_find_extent()
1564 		ex->fe_group = 0;  in mb_find_extent()
1572 	ex->fe_len = 1 << order;  in mb_find_extent()
1573 	ex->fe_start = block << order;  in mb_find_extent()
1574 	ex->fe_group = e4b->bd_group;  in mb_find_extent()
1577 	next = next - ex->fe_start;  in mb_find_extent()
1578 	ex->fe_len -= next;  in mb_find_extent()
1579 	ex->fe_start += next;  in mb_find_extent()
1581 	while (needed > ex->fe_len &&  in mb_find_extent()
1588 		if (mb_test_bit(next, e4b->bd_bitmap))  in mb_find_extent()
1594 		ex->fe_len += 1 << order;  in mb_find_extent()
1597 	if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) {  in mb_find_extent()
1600 		ext4_grp_locked_error(e4b->bd_sb, e4b->bd_group, 0, 0,  in mb_find_extent()
1603 			block, order, needed, ex->fe_group, ex->fe_start,  in mb_find_extent()
1604 			ex->fe_len, ex->fe_logical);  in mb_find_extent()
1605 		ex->fe_len = 0;  in mb_find_extent()
1606 		ex->fe_start = 0;  in mb_find_extent()
1607 		ex->fe_group = 0;  in mb_find_extent()
1609 	return ex->fe_len;  in mb_find_extent()
1618 	int start = ex->fe_start;  in mb_mark_used()
1619 	int len = ex->fe_len;  in mb_mark_used()
1624 	BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3));  in mb_mark_used()
1625 	BUG_ON(e4b->bd_group != ex->fe_group);  in mb_mark_used()
1626 	assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));  in mb_mark_used()
1631 	e4b->bd_info->bb_free -= len;  in mb_mark_used()
1632 	if (e4b->bd_info->bb_first_free == start)  in mb_mark_used()
1633 		e4b->bd_info->bb_first_free += len;  in mb_mark_used()
1637 		mlen = !mb_test_bit(start - 1, e4b->bd_bitmap);  in mb_mark_used()
1638 	if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0])  in mb_mark_used()
1639 		max = !mb_test_bit(start + len, e4b->bd_bitmap);  in mb_mark_used()
1641 		e4b->bd_info->bb_fragments++;  in mb_mark_used()
1643 		e4b->bd_info->bb_fragments--;  in mb_mark_used()
1655 			e4b->bd_info->bb_counters[ord]--;  in mb_mark_used()
1657 			len -= mlen;  in mb_mark_used()
1670 		e4b->bd_info->bb_counters[ord]--;  in mb_mark_used()
1672 		ord--;  in mb_mark_used()
1677 		e4b->bd_info->bb_counters[ord]++;  in mb_mark_used()
1678 		e4b->bd_info->bb_counters[ord]++;  in mb_mark_used()
1680 	mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info);  in mb_mark_used()
1682 	ext4_set_bits(e4b->bd_bitmap, ex->fe_start, len0);  in mb_mark_used()
1694 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_best_found()
1697 	BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);  in ext4_mb_use_best_found()
1698 	BUG_ON(ac->ac_status == AC_STATUS_FOUND);  in ext4_mb_use_best_found()
1700 	ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len);  in ext4_mb_use_best_found()
1701 	ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical;  in ext4_mb_use_best_found()
1702 	ret = mb_mark_used(e4b, &ac->ac_b_ex);  in ext4_mb_use_best_found()
1706 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_use_best_found()
1708 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_best_found()
1709 	ac->ac_tail = ret & 0xffff;  in ext4_mb_use_best_found()
1710 	ac->ac_buddy = ret >> 16;  in ext4_mb_use_best_found()
1719 	ac->ac_bitmap_page = e4b->bd_bitmap_page;  in ext4_mb_use_best_found()
1720 	get_page(ac->ac_bitmap_page);  in ext4_mb_use_best_found()
1721 	ac->ac_buddy_page = e4b->bd_buddy_page;  in ext4_mb_use_best_found()
1722 	get_page(ac->ac_buddy_page);  in ext4_mb_use_best_found()
1724 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {  in ext4_mb_use_best_found()
1725 		spin_lock(&sbi->s_md_lock);  in ext4_mb_use_best_found()
1726 		sbi->s_mb_last_group = ac->ac_f_ex.fe_group;  in ext4_mb_use_best_found()
1727 		sbi->s_mb_last_start = ac->ac_f_ex.fe_start;  in ext4_mb_use_best_found()
1728 		spin_unlock(&sbi->s_md_lock);  in ext4_mb_use_best_found()
1735 	if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len)  in ext4_mb_use_best_found()
1744 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_check_limits()
1745 	struct ext4_free_extent *bex = &ac->ac_b_ex;  in ext4_mb_check_limits()
1746 	struct ext4_free_extent *gex = &ac->ac_g_ex;  in ext4_mb_check_limits()
1750 	if (ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_check_limits()
1755 	if (ac->ac_found > sbi->s_mb_max_to_scan &&  in ext4_mb_check_limits()
1756 			!(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_check_limits()
1757 		ac->ac_status = AC_STATUS_BREAK;  in ext4_mb_check_limits()
1764 	if (bex->fe_len < gex->fe_len)  in ext4_mb_check_limits()
1767 	if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan)  in ext4_mb_check_limits()
1768 			&& bex->fe_group == e4b->bd_group) {  in ext4_mb_check_limits()
1769 		/* recheck chunk's availability - we don't know  in ext4_mb_check_limits()
1770 		 * when it was found (within this lock-unlock  in ext4_mb_check_limits()
1772 		max = mb_find_extent(e4b, bex->fe_start, gex->fe_len, &ex);  in ext4_mb_check_limits()
1773 		if (max >= gex->fe_len) {  in ext4_mb_check_limits()
1794 	struct ext4_free_extent *bex = &ac->ac_b_ex;  in ext4_mb_measure_extent()
1795 	struct ext4_free_extent *gex = &ac->ac_g_ex;  in ext4_mb_measure_extent()
1797 	BUG_ON(ex->fe_len <= 0);  in ext4_mb_measure_extent()
1798 	BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));  in ext4_mb_measure_extent()
1799 	BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));  in ext4_mb_measure_extent()
1800 	BUG_ON(ac->ac_status != AC_STATUS_CONTINUE);  in ext4_mb_measure_extent()
1802 	ac->ac_found++;  in ext4_mb_measure_extent()
1805 	 * The special case - take what you catch first  in ext4_mb_measure_extent()
1807 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_measure_extent()
1816 	if (ex->fe_len == gex->fe_len) {  in ext4_mb_measure_extent()
1825 	if (bex->fe_len == 0) {  in ext4_mb_measure_extent()
1833 	if (bex->fe_len < gex->fe_len) {  in ext4_mb_measure_extent()
1836 		if (ex->fe_len > bex->fe_len)  in ext4_mb_measure_extent()
1838 	} else if (ex->fe_len > gex->fe_len) {  in ext4_mb_measure_extent()
1842 		if (ex->fe_len < bex->fe_len)  in ext4_mb_measure_extent()
1853 	struct ext4_free_extent ex = ac->ac_b_ex;  in ext4_mb_try_best_found()
1859 	err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);  in ext4_mb_try_best_found()
1863 	ext4_lock_group(ac->ac_sb, group);  in ext4_mb_try_best_found()
1867 		ac->ac_b_ex = ex;  in ext4_mb_try_best_found()
1871 	ext4_unlock_group(ac->ac_sb, group);  in ext4_mb_try_best_found()
1881 	ext4_group_t group = ac->ac_g_ex.fe_group;  in ext4_mb_find_by_goal()
1884 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_find_by_goal()
1885 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1888 	if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL))  in ext4_mb_find_by_goal()
1890 	if (grp->bb_free == 0)  in ext4_mb_find_by_goal()
1893 	err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);  in ext4_mb_find_by_goal()
1897 	if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) {  in ext4_mb_find_by_goal()
1902 	ext4_lock_group(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1903 	max = mb_find_extent(e4b, ac->ac_g_ex.fe_start,  in ext4_mb_find_by_goal()
1904 			     ac->ac_g_ex.fe_len, &ex);  in ext4_mb_find_by_goal()
1907 	if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) {  in ext4_mb_find_by_goal()
1910 		start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) +  in ext4_mb_find_by_goal()
1912 		/* use do_div to get remainder (would be 64-bit modulo) */  in ext4_mb_find_by_goal()
1913 		if (do_div(start, sbi->s_stripe) == 0) {  in ext4_mb_find_by_goal()
1914 			ac->ac_found++;  in ext4_mb_find_by_goal()
1915 			ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1918 	} else if (max >= ac->ac_g_ex.fe_len) {  in ext4_mb_find_by_goal()
1920 		BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);  in ext4_mb_find_by_goal()
1921 		BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);  in ext4_mb_find_by_goal()
1922 		ac->ac_found++;  in ext4_mb_find_by_goal()
1923 		ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1925 	} else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) {  in ext4_mb_find_by_goal()
1929 		BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);  in ext4_mb_find_by_goal()
1930 		BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);  in ext4_mb_find_by_goal()
1931 		ac->ac_found++;  in ext4_mb_find_by_goal()
1932 		ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1935 	ext4_unlock_group(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1949 	struct super_block *sb = ac->ac_sb;  in ext4_mb_simple_scan_group()
1950 	struct ext4_group_info *grp = e4b->bd_info;  in ext4_mb_simple_scan_group()
1956 	BUG_ON(ac->ac_2order <= 0);  in ext4_mb_simple_scan_group()
1957 	for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) {  in ext4_mb_simple_scan_group()
1958 		if (grp->bb_counters[i] == 0)  in ext4_mb_simple_scan_group()
1966 			ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0,  in ext4_mb_simple_scan_group()
1968 				grp->bb_counters[i], i);  in ext4_mb_simple_scan_group()
1969 			ext4_mark_group_bitmap_corrupted(ac->ac_sb,  in ext4_mb_simple_scan_group()
1970 					 e4b->bd_group,  in ext4_mb_simple_scan_group()
1974 		ac->ac_found++;  in ext4_mb_simple_scan_group()
1976 		ac->ac_b_ex.fe_len = 1 << i;  in ext4_mb_simple_scan_group()
1977 		ac->ac_b_ex.fe_start = k << i;  in ext4_mb_simple_scan_group()
1978 		ac->ac_b_ex.fe_group = e4b->bd_group;  in ext4_mb_simple_scan_group()
1982 		BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len);  in ext4_mb_simple_scan_group()
1984 		if (EXT4_SB(sb)->s_mb_stats)  in ext4_mb_simple_scan_group()
1985 			atomic_inc(&EXT4_SB(sb)->s_bal_2orders);  in ext4_mb_simple_scan_group()
2000 	struct super_block *sb = ac->ac_sb;  in ext4_mb_complex_scan_group()
2001 	void *bitmap = e4b->bd_bitmap;  in ext4_mb_complex_scan_group()
2006 	free = e4b->bd_info->bb_free;  in ext4_mb_complex_scan_group()
2010 	i = e4b->bd_info->bb_first_free;  in ext4_mb_complex_scan_group()
2012 	while (free && ac->ac_status == AC_STATUS_CONTINUE) {  in ext4_mb_complex_scan_group()
2021 			ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,  in ext4_mb_complex_scan_group()
2025 			ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in ext4_mb_complex_scan_group()
2030 		mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex);  in ext4_mb_complex_scan_group()
2034 			ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,  in ext4_mb_complex_scan_group()
2038 			ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in ext4_mb_complex_scan_group()
2051 		free -= ex.fe_len;  in ext4_mb_complex_scan_group()
2059  * we try to find stripe-aligned chunks for stripe-size-multiple requests
2065 	struct super_block *sb = ac->ac_sb;  in ext4_mb_scan_aligned()
2067 	void *bitmap = e4b->bd_bitmap;  in ext4_mb_scan_aligned()
2074 	BUG_ON(sbi->s_stripe == 0);  in ext4_mb_scan_aligned()
2076 	/* find first stripe-aligned block in group */  in ext4_mb_scan_aligned()
2077 	first_group_block = ext4_group_first_block_no(sb, e4b->bd_group);  in ext4_mb_scan_aligned()
2079 	a = first_group_block + sbi->s_stripe - 1;  in ext4_mb_scan_aligned()
2080 	do_div(a, sbi->s_stripe);  in ext4_mb_scan_aligned()
2081 	i = (a * sbi->s_stripe) - first_group_block;  in ext4_mb_scan_aligned()
2085 			max = mb_find_extent(e4b, i, sbi->s_stripe, &ex);  in ext4_mb_scan_aligned()
2086 			if (max >= sbi->s_stripe) {  in ext4_mb_scan_aligned()
2087 				ac->ac_found++;  in ext4_mb_scan_aligned()
2089 				ac->ac_b_ex = ex;  in ext4_mb_scan_aligned()
2094 		i += sbi->s_stripe;  in ext4_mb_scan_aligned()
2107 	int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb));  in ext4_mb_good_group()
2108 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_good_group()
2115 	free = grp->bb_free;  in ext4_mb_good_group()
2119 	fragments = grp->bb_fragments;  in ext4_mb_good_group()
2125 		BUG_ON(ac->ac_2order == 0);  in ext4_mb_good_group()
2128 		if ((ac->ac_flags & EXT4_MB_HINT_DATA) &&  in ext4_mb_good_group()
2133 		if (free < ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2136 		if (ac->ac_2order > ac->ac_sb->s_blocksize_bits+1)  in ext4_mb_good_group()
2139 		if (grp->bb_largest_free_order < ac->ac_2order)  in ext4_mb_good_group()
2144 		if ((free / fragments) >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2148 		if (free >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2168 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_good_group_nolock()
2169 	struct super_block *sb = ac->ac_sb;  in ext4_mb_good_group_nolock()
2171 	bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK;  in ext4_mb_good_group_nolock()
2177 	free = grp->bb_free;  in ext4_mb_good_group_nolock()
2180 	if (cr <= 2 && free < ac->ac_g_ex.fe_len)  in ext4_mb_good_group_nolock()
2202 		    (!sbi->s_log_groups_per_flex ||  in ext4_mb_good_group_nolock()
2203 		     ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) &&  in ext4_mb_good_group_nolock()
2205 		      (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))))  in ext4_mb_good_group_nolock()
2233 	while (nr-- > 0) {  in ext4_mb_prefetch()
2249 		      (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) {  in ext4_mb_prefetch()
2279 	while (nr-- > 0) {  in ext4_mb_prefetch_fini()
2286 		group--;  in ext4_mb_prefetch_fini()
2292 		      (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) {  in ext4_mb_prefetch_fini()
2303 	int cr = -1;  in ext4_mb_regular_allocator()
2311 	sb = ac->ac_sb;  in ext4_mb_regular_allocator()
2314 	/* non-extent files are limited to low blocks/groups */  in ext4_mb_regular_allocator()
2315 	if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)))  in ext4_mb_regular_allocator()
2316 		ngroups = sbi->s_blockfile_groups;  in ext4_mb_regular_allocator()
2318 	BUG_ON(ac->ac_status == AC_STATUS_FOUND);  in ext4_mb_regular_allocator()
2322 	if (err || ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_regular_allocator()
2325 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_regular_allocator()
2329 	 * ac->ac_2order is set only if the fe_len is a power of 2  in ext4_mb_regular_allocator()
2330 	 * if ac->ac_2order is set we also set criteria to 0 so that we  in ext4_mb_regular_allocator()
2333 	i = fls(ac->ac_g_ex.fe_len);  in ext4_mb_regular_allocator()
2334 	ac->ac_2order = 0;  in ext4_mb_regular_allocator()
2339 	 * We also support searching for power-of-two requests only for  in ext4_mb_regular_allocator()
2342 	if (i >= sbi->s_mb_order2_reqs && i <= sb->s_blocksize_bits + 2) {  in ext4_mb_regular_allocator()
2346 		if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0)  in ext4_mb_regular_allocator()
2347 			ac->ac_2order = array_index_nospec(i - 1,  in ext4_mb_regular_allocator()
2348 							   sb->s_blocksize_bits + 2);  in ext4_mb_regular_allocator()
2352 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {  in ext4_mb_regular_allocator()
2354 		spin_lock(&sbi->s_md_lock);  in ext4_mb_regular_allocator()
2355 		ac->ac_g_ex.fe_group = sbi->s_mb_last_group;  in ext4_mb_regular_allocator()
2356 		ac->ac_g_ex.fe_start = sbi->s_mb_last_start;  in ext4_mb_regular_allocator()
2357 		spin_unlock(&sbi->s_md_lock);  in ext4_mb_regular_allocator()
2360 	/* Let's just scan groups to find more-less suitable blocks */  in ext4_mb_regular_allocator()
2361 	cr = ac->ac_2order ? 0 : 1;  in ext4_mb_regular_allocator()
2367 	for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) {  in ext4_mb_regular_allocator()
2368 		ac->ac_criteria = cr;  in ext4_mb_regular_allocator()
2373 		group = ac->ac_g_ex.fe_group;  in ext4_mb_regular_allocator()
2380 			 * Artificially restricted ngroups for non-extent  in ext4_mb_regular_allocator()
2394 			     prefetch_ios < sbi->s_mb_prefetch_limit)) {  in ext4_mb_regular_allocator()
2397 				nr = sbi->s_mb_prefetch;  in ext4_mb_regular_allocator()
2399 					nr = 1 << sbi->s_log_groups_per_flex;  in ext4_mb_regular_allocator()
2400 					nr -= group & (nr - 1);  in ext4_mb_regular_allocator()
2401 					nr = min(nr, sbi->s_mb_prefetch);  in ext4_mb_regular_allocator()
2434 			ac->ac_groups_scanned++;  in ext4_mb_regular_allocator()
2437 			else if (cr == 1 && sbi->s_stripe &&  in ext4_mb_regular_allocator()
2438 					!(ac->ac_g_ex.fe_len % sbi->s_stripe))  in ext4_mb_regular_allocator()
2446 			if (ac->ac_status != AC_STATUS_CONTINUE)  in ext4_mb_regular_allocator()
2451 	if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND &&  in ext4_mb_regular_allocator()
2452 	    !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_regular_allocator()
2458 		if (ac->ac_status != AC_STATUS_FOUND) {  in ext4_mb_regular_allocator()
2464 			lost = atomic_inc_return(&sbi->s_mb_lost_chunks);  in ext4_mb_regular_allocator()
2466 				 ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start,  in ext4_mb_regular_allocator()
2467 				 ac->ac_b_ex.fe_len, lost);  in ext4_mb_regular_allocator()
2469 			ac->ac_b_ex.fe_group = 0;  in ext4_mb_regular_allocator()
2470 			ac->ac_b_ex.fe_start = 0;  in ext4_mb_regular_allocator()
2471 			ac->ac_b_ex.fe_len = 0;  in ext4_mb_regular_allocator()
2472 			ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_regular_allocator()
2473 			ac->ac_flags |= EXT4_MB_HINT_FIRST;  in ext4_mb_regular_allocator()
2479 	if (!err && ac->ac_status != AC_STATUS_FOUND && first_err)  in ext4_mb_regular_allocator()
2483 		 ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status,  in ext4_mb_regular_allocator()
2484 		 ac->ac_flags, cr, err);  in ext4_mb_regular_allocator()
2494 	struct super_block *sb = PDE_DATA(file_inode(seq->file));  in ext4_mb_seq_groups_start()
2505 	struct super_block *sb = PDE_DATA(file_inode(seq->file));  in ext4_mb_seq_groups_next()
2517 	struct super_block *sb = PDE_DATA(file_inode(seq->file));  in ext4_mb_seq_groups_show()
2524 					     sb->s_blocksize_bits,  in ext4_mb_seq_groups_show()
2531 	group--;  in ext4_mb_seq_groups_show()
2545 			seq_printf(seq, "#%-5u: I/O error\n", group);  in ext4_mb_seq_groups_show()
2556 	seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free,  in ext4_mb_seq_groups_show()
2559 		seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ?  in ext4_mb_seq_groups_show()
2579 	int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE;  in get_groupinfo_cache()
2587  * Allocate the top-level s_group_info array for the specified number
2596 	size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >>  in ext4_mb_alloc_groupinfo()
2598 	if (size <= sbi->s_group_info_size)  in ext4_mb_alloc_groupinfo()
2601 	size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size);  in ext4_mb_alloc_groupinfo()
2605 		return -ENOMEM;  in ext4_mb_alloc_groupinfo()
2608 	old_groupinfo = rcu_dereference(sbi->s_group_info);  in ext4_mb_alloc_groupinfo()
2611 		       sbi->s_group_info_size * sizeof(*sbi->s_group_info));  in ext4_mb_alloc_groupinfo()
2613 	rcu_assign_pointer(sbi->s_group_info, new_groupinfo);  in ext4_mb_alloc_groupinfo()
2614 	sbi->s_group_info_size = size / sizeof(*sbi->s_group_info);  in ext4_mb_alloc_groupinfo()
2618 		   sbi->s_group_info_size);  in ext4_mb_alloc_groupinfo()
2631 	struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);  in ext4_mb_add_groupinfo()
2648 		rcu_dereference(sbi->s_group_info)[idx] = meta_group_info;  in ext4_mb_add_groupinfo()
2653 	i = group & (EXT4_DESC_PER_BLOCK(sb) - 1);  in ext4_mb_add_groupinfo()
2661 		&(meta_group_info[i]->bb_state));  in ext4_mb_add_groupinfo()
2668 	    (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {  in ext4_mb_add_groupinfo()
2669 		meta_group_info[i]->bb_free =  in ext4_mb_add_groupinfo()
2672 		meta_group_info[i]->bb_free =  in ext4_mb_add_groupinfo()
2676 	INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list);  in ext4_mb_add_groupinfo()
2677 	init_rwsem(&meta_group_info[i]->alloc_sem);  in ext4_mb_add_groupinfo()
2678 	meta_group_info[i]->bb_free_root = RB_ROOT;  in ext4_mb_add_groupinfo()
2679 	meta_group_info[i]->bb_largest_free_order = -1;  /* uninit */  in ext4_mb_add_groupinfo()
2690 		group_info = rcu_dereference(sbi->s_group_info);  in ext4_mb_add_groupinfo()
2696 	return -ENOMEM;  in ext4_mb_add_groupinfo()
2713 	sbi->s_buddy_cache = new_inode(sb);  in ext4_mb_init_backend()
2714 	if (sbi->s_buddy_cache == NULL) {  in ext4_mb_init_backend()
2718 	/* To avoid potentially colliding with an valid on-disk inode number,  in ext4_mb_init_backend()
2722 	sbi->s_buddy_cache->i_ino = EXT4_BAD_INO;  in ext4_mb_init_backend()
2723 	EXT4_I(sbi->s_buddy_cache)->i_disksize = 0;  in ext4_mb_init_backend()
2740 		if (sbi->s_es->s_log_groups_per_flex >= 32) {  in ext4_mb_init_backend()
2744 		sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex,  in ext4_mb_init_backend()
2745 			BLK_MAX_SEGMENT_SIZE >> (sb->s_blocksize_bits - 9));  in ext4_mb_init_backend()
2746 		sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */  in ext4_mb_init_backend()
2748 		sbi->s_mb_prefetch = 32;  in ext4_mb_init_backend()
2750 	if (sbi->s_mb_prefetch > ext4_get_groups_count(sb))  in ext4_mb_init_backend()
2751 		sbi->s_mb_prefetch = ext4_get_groups_count(sb);  in ext4_mb_init_backend()
2753 	 * given cr=0 is an CPU-related optimization we shouldn't try to  in ext4_mb_init_backend()
2759 	sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4;  in ext4_mb_init_backend()
2760 	if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb))  in ext4_mb_init_backend()
2761 		sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb);  in ext4_mb_init_backend()
2766 	cachep = get_groupinfo_cache(sb->s_blocksize_bits);  in ext4_mb_init_backend()
2767 	while (i-- > 0)  in ext4_mb_init_backend()
2769 	i = sbi->s_group_info_size;  in ext4_mb_init_backend()
2771 	group_info = rcu_dereference(sbi->s_group_info);  in ext4_mb_init_backend()
2772 	while (i-- > 0)  in ext4_mb_init_backend()
2775 	iput(sbi->s_buddy_cache);  in ext4_mb_init_backend()
2778 	kvfree(rcu_dereference(sbi->s_group_info));  in ext4_mb_init_backend()
2780 	return -ENOMEM;  in ext4_mb_init_backend()
2798 	int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE;  in ext4_groupinfo_create_slab()
2802 		return -EINVAL;  in ext4_groupinfo_create_slab()
2825 		       "EXT4-fs: no memory for groupinfo slab cache\n");  in ext4_groupinfo_create_slab()
2826 		return -ENOMEM;  in ext4_groupinfo_create_slab()
2840 	i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_offsets);  in ext4_mb_init()
2842 	sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL);  in ext4_mb_init()
2843 	if (sbi->s_mb_offsets == NULL) {  in ext4_mb_init()
2844 		ret = -ENOMEM;  in ext4_mb_init()
2848 	i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_maxs);  in ext4_mb_init()
2849 	sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL);  in ext4_mb_init()
2850 	if (sbi->s_mb_maxs == NULL) {  in ext4_mb_init()
2851 		ret = -ENOMEM;  in ext4_mb_init()
2855 	ret = ext4_groupinfo_create_slab(sb->s_blocksize);  in ext4_mb_init()
2860 	sbi->s_mb_maxs[0] = sb->s_blocksize << 3;  in ext4_mb_init()
2861 	sbi->s_mb_offsets[0] = 0;  in ext4_mb_init()
2865 	offset_incr = 1 << (sb->s_blocksize_bits - 1);  in ext4_mb_init()
2866 	max = sb->s_blocksize << 2;  in ext4_mb_init()
2868 		sbi->s_mb_offsets[i] = offset;  in ext4_mb_init()
2869 		sbi->s_mb_maxs[i] = max;  in ext4_mb_init()
2874 	} while (i <= sb->s_blocksize_bits + 1);  in ext4_mb_init()
2876 	spin_lock_init(&sbi->s_md_lock);  in ext4_mb_init()
2877 	spin_lock_init(&sbi->s_bal_lock);  in ext4_mb_init()
2878 	sbi->s_mb_free_pending = 0;  in ext4_mb_init()
2879 	INIT_LIST_HEAD(&sbi->s_freed_data_list);  in ext4_mb_init()
2881 	sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN;  in ext4_mb_init()
2882 	sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN;  in ext4_mb_init()
2883 	sbi->s_mb_stats = MB_DEFAULT_STATS;  in ext4_mb_init()
2884 	sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD;  in ext4_mb_init()
2885 	sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS;  in ext4_mb_init()
2886 	sbi->s_mb_max_inode_prealloc = MB_DEFAULT_MAX_INODE_PREALLOC;  in ext4_mb_init()
2899 	sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >>  in ext4_mb_init()
2900 				       sbi->s_cluster_bits, 32);  in ext4_mb_init()
2909 	if (sbi->s_stripe > 1) {  in ext4_mb_init()
2910 		sbi->s_mb_group_prealloc = roundup(  in ext4_mb_init()
2911 			sbi->s_mb_group_prealloc, sbi->s_stripe);  in ext4_mb_init()
2914 	sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group);  in ext4_mb_init()
2915 	if (sbi->s_locality_groups == NULL) {  in ext4_mb_init()
2916 		ret = -ENOMEM;  in ext4_mb_init()
2921 		lg = per_cpu_ptr(sbi->s_locality_groups, i);  in ext4_mb_init()
2922 		mutex_init(&lg->lg_mutex);  in ext4_mb_init()
2924 			INIT_LIST_HEAD(&lg->lg_prealloc_list[j]);  in ext4_mb_init()
2925 		spin_lock_init(&lg->lg_prealloc_lock);  in ext4_mb_init()
2936 	free_percpu(sbi->s_locality_groups);  in ext4_mb_init()
2937 	sbi->s_locality_groups = NULL;  in ext4_mb_init()
2939 	kfree(sbi->s_mb_offsets);  in ext4_mb_init()
2940 	sbi->s_mb_offsets = NULL;  in ext4_mb_init()
2941 	kfree(sbi->s_mb_maxs);  in ext4_mb_init()
2942 	sbi->s_mb_maxs = NULL;  in ext4_mb_init()
2953 	list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) {  in ext4_mb_cleanup_pa()
2955 		list_del(&pa->pa_group_list);  in ext4_mb_cleanup_pa()
2969 	struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);  in ext4_mb_release()
2972 	if (sbi->s_group_info) {  in ext4_mb_release()
2986 				EXT4_DESC_PER_BLOCK(sb) - 1) >>  in ext4_mb_release()
2989 		group_info = rcu_dereference(sbi->s_group_info);  in ext4_mb_release()
2995 	kfree(sbi->s_mb_offsets);  in ext4_mb_release()
2996 	kfree(sbi->s_mb_maxs);  in ext4_mb_release()
2997 	iput(sbi->s_buddy_cache);  in ext4_mb_release()
2998 	if (sbi->s_mb_stats) {  in ext4_mb_release()
3001 				atomic_read(&sbi->s_bal_allocated),  in ext4_mb_release()
3002 				atomic_read(&sbi->s_bal_reqs),  in ext4_mb_release()
3003 				atomic_read(&sbi->s_bal_success));  in ext4_mb_release()
3007 				atomic_read(&sbi->s_bal_ex_scanned),  in ext4_mb_release()
3008 				atomic_read(&sbi->s_bal_goals),  in ext4_mb_release()
3009 				atomic_read(&sbi->s_bal_2orders),  in ext4_mb_release()
3010 				atomic_read(&sbi->s_bal_breaks),  in ext4_mb_release()
3011 				atomic_read(&sbi->s_mb_lost_chunks));  in ext4_mb_release()
3014 				sbi->s_mb_buddies_generated,  in ext4_mb_release()
3015 				sbi->s_mb_generation_time);  in ext4_mb_release()
3018 				atomic_read(&sbi->s_mb_preallocated),  in ext4_mb_release()
3019 				atomic_read(&sbi->s_mb_discarded));  in ext4_mb_release()
3022 	free_percpu(sbi->s_locality_groups);  in ext4_mb_release()
3039 		return __blkdev_issue_discard(sb->s_bdev,  in ext4_issue_discard()
3040 			(sector_t)discard_block << (sb->s_blocksize_bits - 9),  in ext4_issue_discard()
3041 			(sector_t)count << (sb->s_blocksize_bits - 9),  in ext4_issue_discard()
3055 		 entry->efd_count, entry->efd_group, entry);  in ext4_free_data_in_buddy()
3057 	err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b);  in ext4_free_data_in_buddy()
3061 	spin_lock(&EXT4_SB(sb)->s_md_lock);  in ext4_free_data_in_buddy()
3062 	EXT4_SB(sb)->s_mb_free_pending -= entry->efd_count;  in ext4_free_data_in_buddy()
3063 	spin_unlock(&EXT4_SB(sb)->s_md_lock);  in ext4_free_data_in_buddy()
3067 	count += entry->efd_count;  in ext4_free_data_in_buddy()
3069 	ext4_lock_group(sb, entry->efd_group);  in ext4_free_data_in_buddy()
3071 	rb_erase(&entry->efd_node, &(db->bb_free_root));  in ext4_free_data_in_buddy()
3072 	mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count);  in ext4_free_data_in_buddy()
3077 	 * If the volume is mounted with -o discard, online discard  in ext4_free_data_in_buddy()
3083 	if (!db->bb_free_root.rb_node) {  in ext4_free_data_in_buddy()
3090 	ext4_unlock_group(sb, entry->efd_group);  in ext4_free_data_in_buddy()
3113 	spin_lock(&sbi->s_md_lock);  in ext4_process_freed_data()
3114 	list_for_each_entry(entry, &sbi->s_freed_data_list, efd_list) {  in ext4_process_freed_data()
3115 		if (entry->efd_tid != commit_tid)  in ext4_process_freed_data()
3117 		cut_pos = &entry->efd_list;  in ext4_process_freed_data()
3120 		list_cut_position(&freed_data_list, &sbi->s_freed_data_list,  in ext4_process_freed_data()
3122 	spin_unlock(&sbi->s_md_lock);  in ext4_process_freed_data()
3126 			err = ext4_issue_discard(sb, entry->efd_group,  in ext4_process_freed_data()
3127 						 entry->efd_start_cluster,  in ext4_process_freed_data()
3128 						 entry->efd_count,  in ext4_process_freed_data()
3130 			if (err && err != -EOPNOTSUPP) {  in ext4_process_freed_data()
3133 					 " with %d", entry->efd_group,  in ext4_process_freed_data()
3134 					 entry->efd_start_cluster,  in ext4_process_freed_data()
3135 					 entry->efd_count, err);  in ext4_process_freed_data()
3136 			} else if (err == -EOPNOTSUPP)  in ext4_process_freed_data()
3174 	return -ENOMEM;  in ext4_init_mballoc()
3192  * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
3207 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_mark_diskspace_used()
3208 	BUG_ON(ac->ac_b_ex.fe_len <= 0);  in ext4_mb_mark_diskspace_used()
3210 	sb = ac->ac_sb;  in ext4_mb_mark_diskspace_used()
3213 	bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3225 	err = -EIO;  in ext4_mb_mark_diskspace_used()
3226 	gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh);  in ext4_mb_mark_diskspace_used()
3230 	ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group,  in ext4_mb_mark_diskspace_used()
3238 	block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_mark_diskspace_used()
3240 	len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3241 	if (!ext4_inode_block_valid(ac->ac_inode, block, len)) {  in ext4_mb_mark_diskspace_used()
3242 		ext4_error(sb, "Allocating blocks %llu-%llu which overlap "  in ext4_mb_mark_diskspace_used()
3248 		ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3249 		ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,  in ext4_mb_mark_diskspace_used()
3250 			      ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3251 		ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3254 			err = -EFSCORRUPTED;  in ext4_mb_mark_diskspace_used()
3258 	ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3262 		for (i = 0; i < ac->ac_b_ex.fe_len; i++) {  in ext4_mb_mark_diskspace_used()
3263 			BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i,  in ext4_mb_mark_diskspace_used()
3264 						bitmap_bh->b_data));  in ext4_mb_mark_diskspace_used()
3268 	ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,  in ext4_mb_mark_diskspace_used()
3269 		      ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3271 	    (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {  in ext4_mb_mark_diskspace_used()
3272 		gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);  in ext4_mb_mark_diskspace_used()
3275 						ac->ac_b_ex.fe_group, gdp));  in ext4_mb_mark_diskspace_used()
3277 	len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len;  in ext4_mb_mark_diskspace_used()
3279 	ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh);  in ext4_mb_mark_diskspace_used()
3280 	ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp);  in ext4_mb_mark_diskspace_used()
3282 	ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3283 	percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3287 	if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED))  in ext4_mb_mark_diskspace_used()
3289 		percpu_counter_sub(&sbi->s_dirtyclusters_counter,  in ext4_mb_mark_diskspace_used()
3292 	if (sbi->s_log_groups_per_flex) {  in ext4_mb_mark_diskspace_used()
3294 							  ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3295 		atomic64_sub(ac->ac_b_ex.fe_len,  in ext4_mb_mark_diskspace_used()
3297 						  flex_group)->free_clusters);  in ext4_mb_mark_diskspace_used()
3336 	err = -EIO;  in ext4_mb_mark_bb()
3344 		if (!mb_test_bit(blkoff + i, bitmap_bh->b_data) == !state)  in ext4_mb_mark_bb()
3348 		ext4_set_bits(bitmap_bh->b_data, blkoff, clen);  in ext4_mb_mark_bb()
3350 		mb_test_and_clear_bits(bitmap_bh->b_data, blkoff, clen);  in ext4_mb_mark_bb()
3352 	    (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {  in ext4_mb_mark_bb()
3353 		gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);  in ext4_mb_mark_bb()
3359 		clen = ext4_free_group_clusters(sb, gdp) - clen + already;  in ext4_mb_mark_bb()
3361 		clen = ext4_free_group_clusters(sb, gdp) + clen - already;  in ext4_mb_mark_bb()
3369 	if (sbi->s_log_groups_per_flex) {  in ext4_mb_mark_bb()
3374 						  flex_group)->free_clusters);  in ext4_mb_mark_bb()
3389  * here we normalize request for locality group
3399 	struct super_block *sb = ac->ac_sb;  in ext4_mb_normalize_group_request()
3400 	struct ext4_locality_group *lg = ac->ac_lg;  in ext4_mb_normalize_group_request()
3403 	ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc;  in ext4_mb_normalize_group_request()
3404 	mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len);  in ext4_mb_normalize_group_request()
3415 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_normalize_request()
3421 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_normalize_request()
3426 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_normalize_request()
3430 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_normalize_request()
3435 	if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC)  in ext4_mb_normalize_request()
3438 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) {  in ext4_mb_normalize_request()
3443 	bsbits = ac->ac_sb->s_blocksize_bits;  in ext4_mb_normalize_request()
3447 	size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len);  in ext4_mb_normalize_request()
3449 	if (size < i_size_read(ac->ac_inode))  in ext4_mb_normalize_request()
3450 		size = i_size_read(ac->ac_inode);  in ext4_mb_normalize_request()
3477 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3478 						(21 - bsbits)) << 21;  in ext4_mb_normalize_request()
3481 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3482 							(22 - bsbits)) << 22;  in ext4_mb_normalize_request()
3484 	} else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len,  in ext4_mb_normalize_request()
3486 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3487 							(23 - bsbits)) << 23;  in ext4_mb_normalize_request()
3490 		start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits;  in ext4_mb_normalize_request()
3491 		size	  = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3492 					      ac->ac_o_ex.fe_len) << bsbits;  in ext4_mb_normalize_request()
3498 	if (ar->pleft && start <= ar->lleft) {  in ext4_mb_normalize_request()
3499 		size -= ar->lleft + 1 - start;  in ext4_mb_normalize_request()
3500 		start = ar->lleft + 1;  in ext4_mb_normalize_request()
3502 	if (ar->pright && start + size - 1 >= ar->lright)  in ext4_mb_normalize_request()
3503 		size -= start + size - ar->lright;  in ext4_mb_normalize_request()
3509 	if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb))  in ext4_mb_normalize_request()
3510 		size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb);  in ext4_mb_normalize_request()
3516 	list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {  in ext4_mb_normalize_request()
3519 		if (pa->pa_deleted)  in ext4_mb_normalize_request()
3521 		spin_lock(&pa->pa_lock);  in ext4_mb_normalize_request()
3522 		if (pa->pa_deleted) {  in ext4_mb_normalize_request()
3523 			spin_unlock(&pa->pa_lock);  in ext4_mb_normalize_request()
3527 		pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3528 						  pa->pa_len);  in ext4_mb_normalize_request()
3531 		BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end ||  in ext4_mb_normalize_request()
3532 			ac->ac_o_ex.fe_logical < pa->pa_lstart));  in ext4_mb_normalize_request()
3535 		if (pa->pa_lstart >= end || pa_end <= start) {  in ext4_mb_normalize_request()
3536 			spin_unlock(&pa->pa_lock);  in ext4_mb_normalize_request()
3539 		BUG_ON(pa->pa_lstart <= start && pa_end >= end);  in ext4_mb_normalize_request()
3542 		if (pa_end <= ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3545 		} else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3546 			BUG_ON(pa->pa_lstart > end);  in ext4_mb_normalize_request()
3547 			end = pa->pa_lstart;  in ext4_mb_normalize_request()
3549 		spin_unlock(&pa->pa_lock);  in ext4_mb_normalize_request()
3552 	size = end - start;  in ext4_mb_normalize_request()
3556 	list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {  in ext4_mb_normalize_request()
3559 		spin_lock(&pa->pa_lock);  in ext4_mb_normalize_request()
3560 		if (pa->pa_deleted == 0) {  in ext4_mb_normalize_request()
3561 			pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3562 							  pa->pa_len);  in ext4_mb_normalize_request()
3563 			BUG_ON(!(start >= pa_end || end <= pa->pa_lstart));  in ext4_mb_normalize_request()
3565 		spin_unlock(&pa->pa_lock);  in ext4_mb_normalize_request()
3569 	if (start + size <= ac->ac_o_ex.fe_logical &&  in ext4_mb_normalize_request()
3570 			start > ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3571 		ext4_msg(ac->ac_sb, KERN_ERR,  in ext4_mb_normalize_request()
3574 			 (unsigned long) ac->ac_o_ex.fe_logical);  in ext4_mb_normalize_request()
3577 	BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb));  in ext4_mb_normalize_request()
3583 	ac->ac_g_ex.fe_logical = start;  in ext4_mb_normalize_request()
3584 	ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size);  in ext4_mb_normalize_request()
3587 	if (ar->pright && (ar->lright == (start + size))) {  in ext4_mb_normalize_request()
3589 		ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size,  in ext4_mb_normalize_request()
3590 						&ac->ac_f_ex.fe_group,  in ext4_mb_normalize_request()
3591 						&ac->ac_f_ex.fe_start);  in ext4_mb_normalize_request()
3592 		ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;  in ext4_mb_normalize_request()
3594 	if (ar->pleft && (ar->lleft + 1 == start)) {  in ext4_mb_normalize_request()
3596 		ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1,  in ext4_mb_normalize_request()
3597 						&ac->ac_f_ex.fe_group,  in ext4_mb_normalize_request()
3598 						&ac->ac_f_ex.fe_start);  in ext4_mb_normalize_request()
3599 		ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;  in ext4_mb_normalize_request()
3602 	mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size,  in ext4_mb_normalize_request()
3608 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_collect_stats()
3610 	if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) {  in ext4_mb_collect_stats()
3611 		atomic_inc(&sbi->s_bal_reqs);  in ext4_mb_collect_stats()
3612 		atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);  in ext4_mb_collect_stats()
3613 		if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len)  in ext4_mb_collect_stats()
3614 			atomic_inc(&sbi->s_bal_success);  in ext4_mb_collect_stats()
3615 		atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);  in ext4_mb_collect_stats()
3616 		if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&  in ext4_mb_collect_stats()
3617 				ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)  in ext4_mb_collect_stats()
3618 			atomic_inc(&sbi->s_bal_goals);  in ext4_mb_collect_stats()
3619 		if (ac->ac_found > sbi->s_mb_max_to_scan)  in ext4_mb_collect_stats()
3620 			atomic_inc(&sbi->s_bal_breaks);  in ext4_mb_collect_stats()
3623 	if (ac->ac_op == EXT4_MB_HISTORY_ALLOC)  in ext4_mb_collect_stats()
3633  * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
3637 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_discard_allocated_blocks()
3642 		if (ac->ac_f_ex.fe_len == 0)  in ext4_discard_allocated_blocks()
3644 		err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b);  in ext4_discard_allocated_blocks()
3654 		ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group);  in ext4_discard_allocated_blocks()
3655 		mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start,  in ext4_discard_allocated_blocks()
3656 			       ac->ac_f_ex.fe_len);  in ext4_discard_allocated_blocks()
3657 		ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group);  in ext4_discard_allocated_blocks()
3661 	if (pa->pa_type == MB_INODE_PA)  in ext4_discard_allocated_blocks()
3662 		pa->pa_free += ac->ac_b_ex.fe_len;  in ext4_discard_allocated_blocks()
3671 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_inode_pa()
3677 	start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart);  in ext4_mb_use_inode_pa()
3678 	end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len),  in ext4_mb_use_inode_pa()
3679 		  start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len));  in ext4_mb_use_inode_pa()
3680 	len = EXT4_NUM_B2C(sbi, end - start);  in ext4_mb_use_inode_pa()
3681 	ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group,  in ext4_mb_use_inode_pa()
3682 					&ac->ac_b_ex.fe_start);  in ext4_mb_use_inode_pa()
3683 	ac->ac_b_ex.fe_len = len;  in ext4_mb_use_inode_pa()
3684 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_inode_pa()
3685 	ac->ac_pa = pa;  in ext4_mb_use_inode_pa()
3687 	BUG_ON(start < pa->pa_pstart);  in ext4_mb_use_inode_pa()
3688 	BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len));  in ext4_mb_use_inode_pa()
3689 	BUG_ON(pa->pa_free < len);  in ext4_mb_use_inode_pa()
3690 	pa->pa_free -= len;  in ext4_mb_use_inode_pa()
3692 	mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa);  in ext4_mb_use_inode_pa()
3696  * use blocks preallocated to locality group
3701 	unsigned int len = ac->ac_o_ex.fe_len;  in ext4_mb_use_group_pa()
3703 	ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart,  in ext4_mb_use_group_pa()
3704 					&ac->ac_b_ex.fe_group,  in ext4_mb_use_group_pa()
3705 					&ac->ac_b_ex.fe_start);  in ext4_mb_use_group_pa()
3706 	ac->ac_b_ex.fe_len = len;  in ext4_mb_use_group_pa()
3707 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_group_pa()
3708 	ac->ac_pa = pa;  in ext4_mb_use_group_pa()
3713 	 * in on-disk bitmap -- see ext4_mb_release_context()  in ext4_mb_use_group_pa()
3716 	mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n",  in ext4_mb_use_group_pa()
3717 		 pa->pa_lstart-len, len, pa);  in ext4_mb_use_group_pa()
3734 		atomic_inc(&pa->pa_count);  in ext4_mb_check_group_pa()
3737 	cur_distance = abs(goal_block - cpa->pa_pstart);  in ext4_mb_check_group_pa()
3738 	new_distance = abs(goal_block - pa->pa_pstart);  in ext4_mb_check_group_pa()
3744 	atomic_dec(&cpa->pa_count);  in ext4_mb_check_group_pa()
3745 	atomic_inc(&pa->pa_count);  in ext4_mb_check_group_pa()
3755 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_preallocated()
3757 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_use_preallocated()
3763 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_use_preallocated()
3766 	/* first, try per-file preallocation */  in ext4_mb_use_preallocated()
3768 	list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {  in ext4_mb_use_preallocated()
3772 		if (ac->ac_o_ex.fe_logical < pa->pa_lstart ||  in ext4_mb_use_preallocated()
3773 		    ac->ac_o_ex.fe_logical >= (pa->pa_lstart +  in ext4_mb_use_preallocated()
3774 					       EXT4_C2B(sbi, pa->pa_len)))  in ext4_mb_use_preallocated()
3777 		/* non-extent files can't have physical blocks past 2^32 */  in ext4_mb_use_preallocated()
3778 		if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) &&  in ext4_mb_use_preallocated()
3779 		    (pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len) >  in ext4_mb_use_preallocated()
3784 		spin_lock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3785 		if (pa->pa_deleted == 0 && pa->pa_free) {  in ext4_mb_use_preallocated()
3786 			atomic_inc(&pa->pa_count);  in ext4_mb_use_preallocated()
3788 			spin_unlock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3789 			ac->ac_criteria = 10;  in ext4_mb_use_preallocated()
3793 		spin_unlock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3798 	if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC))  in ext4_mb_use_preallocated()
3801 	/* inode may have no locality group for some reason */  in ext4_mb_use_preallocated()
3802 	lg = ac->ac_lg;  in ext4_mb_use_preallocated()
3805 	order  = fls(ac->ac_o_ex.fe_len) - 1;  in ext4_mb_use_preallocated()
3806 	if (order > PREALLOC_TB_SIZE - 1)  in ext4_mb_use_preallocated()
3808 		order = PREALLOC_TB_SIZE - 1;  in ext4_mb_use_preallocated()
3810 	goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex);  in ext4_mb_use_preallocated()
3817 		list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[i],  in ext4_mb_use_preallocated()
3819 			spin_lock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3820 			if (pa->pa_deleted == 0 &&  in ext4_mb_use_preallocated()
3821 					pa->pa_free >= ac->ac_o_ex.fe_len) {  in ext4_mb_use_preallocated()
3826 			spin_unlock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3832 		ac->ac_criteria = 20;  in ext4_mb_use_preallocated()
3840  * but not yet committed and marks them used in in-core bitmap.
3852 	n = rb_first(&(grp->bb_free_root));  in ext4_mb_generate_from_freelist()
3856 		ext4_set_bits(bitmap, entry->efd_start_cluster, entry->efd_count);  in ext4_mb_generate_from_freelist()
3864  * used in in-core bitmap. buddy must be generated from this bitmap
3887 	list_for_each(cur, &grp->bb_prealloc_list) {  in ext4_mb_generate_from_pa()
3889 		spin_lock(&pa->pa_lock);  in ext4_mb_generate_from_pa()
3890 		ext4_get_group_no_and_offset(sb, pa->pa_pstart,  in ext4_mb_generate_from_pa()
3892 		len = pa->pa_len;  in ext4_mb_generate_from_pa()
3893 		spin_unlock(&pa->pa_lock);  in ext4_mb_generate_from_pa()
3908 	if (pa->pa_deleted) {  in ext4_mb_mark_pa_deleted()
3910 			     pa->pa_type, pa->pa_pstart, pa->pa_lstart,  in ext4_mb_mark_pa_deleted()
3911 			     pa->pa_len);  in ext4_mb_mark_pa_deleted()
3915 	pa->pa_deleted = 1;  in ext4_mb_mark_pa_deleted()
3917 	if (pa->pa_type == MB_INODE_PA) {  in ext4_mb_mark_pa_deleted()
3918 		ei = EXT4_I(pa->pa_inode);  in ext4_mb_mark_pa_deleted()
3919 		atomic_dec(&ei->i_prealloc_active);  in ext4_mb_mark_pa_deleted()
3928 	BUG_ON(atomic_read(&pa->pa_count));  in ext4_mb_pa_callback()
3929 	BUG_ON(pa->pa_deleted == 0);  in ext4_mb_pa_callback()
3944 	spin_lock(&pa->pa_lock);  in ext4_mb_put_pa()
3945 	if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) {  in ext4_mb_put_pa()
3946 		spin_unlock(&pa->pa_lock);  in ext4_mb_put_pa()
3950 	if (pa->pa_deleted == 1) {  in ext4_mb_put_pa()
3951 		spin_unlock(&pa->pa_lock);  in ext4_mb_put_pa()
3956 	spin_unlock(&pa->pa_lock);  in ext4_mb_put_pa()
3958 	grp_blk = pa->pa_pstart;  in ext4_mb_put_pa()
3960 	 * If doing group-based preallocation, pa_pstart may be in the  in ext4_mb_put_pa()
3963 	if (pa->pa_type == MB_GROUP_PA)  in ext4_mb_put_pa()
3964 		grp_blk--;  in ext4_mb_put_pa()
3973 	 *  copy on-disk bitmap to buddy  in ext4_mb_put_pa()
3974 	 *  					mark B in on-disk bitmap  in ext4_mb_put_pa()
3983 	list_del(&pa->pa_group_list);  in ext4_mb_put_pa()
3986 	spin_lock(pa->pa_obj_lock);  in ext4_mb_put_pa()
3987 	list_del_rcu(&pa->pa_inode_list);  in ext4_mb_put_pa()
3988 	spin_unlock(pa->pa_obj_lock);  in ext4_mb_put_pa()
3990 	call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_mb_put_pa()
3999 	struct super_block *sb = ac->ac_sb;  in ext4_mb_new_inode_pa()
4006 	BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
4007 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_new_inode_pa()
4008 	BUG_ON(!S_ISREG(ac->ac_inode->i_mode));  in ext4_mb_new_inode_pa()
4009 	BUG_ON(ac->ac_pa == NULL);  in ext4_mb_new_inode_pa()
4011 	pa = ac->ac_pa;  in ext4_mb_new_inode_pa()
4013 	if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) {  in ext4_mb_new_inode_pa()
4022 		BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical);  in ext4_mb_new_inode_pa()
4023 		BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len);  in ext4_mb_new_inode_pa()
4028 		winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical;  in ext4_mb_new_inode_pa()
4031 		wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len);  in ext4_mb_new_inode_pa()
4036 		offs = ac->ac_o_ex.fe_logical %  in ext4_mb_new_inode_pa()
4037 			EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
4041 		ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical -  in ext4_mb_new_inode_pa()
4043 		BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical);  in ext4_mb_new_inode_pa()
4044 		BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
4049 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_new_inode_pa()
4051 	pa->pa_lstart = ac->ac_b_ex.fe_logical;  in ext4_mb_new_inode_pa()
4052 	pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_inode_pa()
4053 	pa->pa_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_inode_pa()
4054 	pa->pa_free = pa->pa_len;  in ext4_mb_new_inode_pa()
4055 	spin_lock_init(&pa->pa_lock);  in ext4_mb_new_inode_pa()
4056 	INIT_LIST_HEAD(&pa->pa_inode_list);  in ext4_mb_new_inode_pa()
4057 	INIT_LIST_HEAD(&pa->pa_group_list);  in ext4_mb_new_inode_pa()
4058 	pa->pa_deleted = 0;  in ext4_mb_new_inode_pa()
4059 	pa->pa_type = MB_INODE_PA;  in ext4_mb_new_inode_pa()
4061 	mb_debug(sb, "new inode pa %p: %llu/%d for %u\n", pa, pa->pa_pstart,  in ext4_mb_new_inode_pa()
4062 		 pa->pa_len, pa->pa_lstart);  in ext4_mb_new_inode_pa()
4066 	atomic_add(pa->pa_free, &sbi->s_mb_preallocated);  in ext4_mb_new_inode_pa()
4068 	ei = EXT4_I(ac->ac_inode);  in ext4_mb_new_inode_pa()
4069 	grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_inode_pa()
4071 	pa->pa_obj_lock = &ei->i_prealloc_lock;  in ext4_mb_new_inode_pa()
4072 	pa->pa_inode = ac->ac_inode;  in ext4_mb_new_inode_pa()
4074 	list_add(&pa->pa_group_list, &grp->bb_prealloc_list);  in ext4_mb_new_inode_pa()
4076 	spin_lock(pa->pa_obj_lock);  in ext4_mb_new_inode_pa()
4077 	list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list);  in ext4_mb_new_inode_pa()
4078 	spin_unlock(pa->pa_obj_lock);  in ext4_mb_new_inode_pa()
4079 	atomic_inc(&ei->i_prealloc_active);  in ext4_mb_new_inode_pa()
4083  * creates new preallocated space for locality group inodes belongs to
4088 	struct super_block *sb = ac->ac_sb;  in ext4_mb_new_group_pa()
4094 	BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);  in ext4_mb_new_group_pa()
4095 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_new_group_pa()
4096 	BUG_ON(!S_ISREG(ac->ac_inode->i_mode));  in ext4_mb_new_group_pa()
4097 	BUG_ON(ac->ac_pa == NULL);  in ext4_mb_new_group_pa()
4099 	pa = ac->ac_pa;  in ext4_mb_new_group_pa()
4103 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_new_group_pa()
4105 	pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_group_pa()
4106 	pa->pa_lstart = pa->pa_pstart;  in ext4_mb_new_group_pa()
4107 	pa->pa_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_group_pa()
4108 	pa->pa_free = pa->pa_len;  in ext4_mb_new_group_pa()
4109 	spin_lock_init(&pa->pa_lock);  in ext4_mb_new_group_pa()
4110 	INIT_LIST_HEAD(&pa->pa_inode_list);  in ext4_mb_new_group_pa()
4111 	INIT_LIST_HEAD(&pa->pa_group_list);  in ext4_mb_new_group_pa()
4112 	pa->pa_deleted = 0;  in ext4_mb_new_group_pa()
4113 	pa->pa_type = MB_GROUP_PA;  in ext4_mb_new_group_pa()
4115 	mb_debug(sb, "new group pa %p: %llu/%d for %u\n", pa, pa->pa_pstart,  in ext4_mb_new_group_pa()
4116 		 pa->pa_len, pa->pa_lstart);  in ext4_mb_new_group_pa()
4120 	atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);  in ext4_mb_new_group_pa()
4122 	grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_group_pa()
4123 	lg = ac->ac_lg;  in ext4_mb_new_group_pa()
4126 	pa->pa_obj_lock = &lg->lg_prealloc_lock;  in ext4_mb_new_group_pa()
4127 	pa->pa_inode = NULL;  in ext4_mb_new_group_pa()
4129 	list_add(&pa->pa_group_list, &grp->bb_prealloc_list);  in ext4_mb_new_group_pa()
4139 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)  in ext4_mb_new_preallocation()
4146  * finds all unused blocks in on-disk bitmap, frees them in
4147  * in-core bitmap and buddy.
4151  * TODO: optimize the case when there are no in-core structures yet
4157 	struct super_block *sb = e4b->bd_sb;  in ext4_mb_release_inode_pa()
4166 	BUG_ON(pa->pa_deleted == 0);  in ext4_mb_release_inode_pa()
4167 	ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);  in ext4_mb_release_inode_pa()
4168 	grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit);  in ext4_mb_release_inode_pa()
4169 	BUG_ON(group != e4b->bd_group && pa->pa_len != 0);  in ext4_mb_release_inode_pa()
4170 	end = bit + pa->pa_len;  in ext4_mb_release_inode_pa()
4173 		bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit);  in ext4_mb_release_inode_pa()
4176 		next = mb_find_next_bit(bitmap_bh->b_data, end, bit);  in ext4_mb_release_inode_pa()
4179 			 (unsigned) next - bit, (unsigned) group);  in ext4_mb_release_inode_pa()
4180 		free += next - bit;  in ext4_mb_release_inode_pa()
4182 		trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit);  in ext4_mb_release_inode_pa()
4185 					       next - bit);  in ext4_mb_release_inode_pa()
4186 		mb_free_blocks(pa->pa_inode, e4b, bit, next - bit);  in ext4_mb_release_inode_pa()
4189 	if (free != pa->pa_free) {  in ext4_mb_release_inode_pa()
4190 		ext4_msg(e4b->bd_sb, KERN_CRIT,  in ext4_mb_release_inode_pa()
4192 			 pa, (unsigned long) pa->pa_lstart,  in ext4_mb_release_inode_pa()
4193 			 (unsigned long) pa->pa_pstart,  in ext4_mb_release_inode_pa()
4194 			 pa->pa_len);  in ext4_mb_release_inode_pa()
4196 					free, pa->pa_free);  in ext4_mb_release_inode_pa()
4202 	atomic_add(free, &sbi->s_mb_discarded);  in ext4_mb_release_inode_pa()
4211 	struct super_block *sb = e4b->bd_sb;  in ext4_mb_release_group_pa()
4216 	BUG_ON(pa->pa_deleted == 0);  in ext4_mb_release_group_pa()
4217 	ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);  in ext4_mb_release_group_pa()
4218 	BUG_ON(group != e4b->bd_group && pa->pa_len != 0);  in ext4_mb_release_group_pa()
4219 	mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len);  in ext4_mb_release_group_pa()
4220 	atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded);  in ext4_mb_release_group_pa()
4221 	trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len);  in ext4_mb_release_group_pa()
4230  * - when do we discard
4232  * - how many do we discard
4248 	if (list_empty(&grp->bb_prealloc_list))  in ext4_mb_discard_group_preallocations()
4254 		ext4_error_err(sb, -err,  in ext4_mb_discard_group_preallocations()
4271 				&grp->bb_prealloc_list, pa_group_list) {  in ext4_mb_discard_group_preallocations()
4272 		spin_lock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
4273 		if (atomic_read(&pa->pa_count)) {  in ext4_mb_discard_group_preallocations()
4274 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
4278 		if (pa->pa_deleted) {  in ext4_mb_discard_group_preallocations()
4279 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
4290 		free += pa->pa_free;  in ext4_mb_discard_group_preallocations()
4292 		spin_unlock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
4294 		list_del(&pa->pa_group_list);  in ext4_mb_discard_group_preallocations()
4295 		list_add(&pa->u.pa_tmp_list, &list);  in ext4_mb_discard_group_preallocations()
4301 		/* remove from object (inode or locality group) */  in ext4_mb_discard_group_preallocations()
4302 		spin_lock(pa->pa_obj_lock);  in ext4_mb_discard_group_preallocations()
4303 		list_del_rcu(&pa->pa_inode_list);  in ext4_mb_discard_group_preallocations()
4304 		spin_unlock(pa->pa_obj_lock);  in ext4_mb_discard_group_preallocations()
4306 		if (pa->pa_type == MB_GROUP_PA)  in ext4_mb_discard_group_preallocations()
4311 		list_del(&pa->u.pa_tmp_list);  in ext4_mb_discard_group_preallocations()
4312 		call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_mb_discard_group_preallocations()
4320 		 free, group, grp->bb_free);  in ext4_mb_discard_group_preallocations()
4325  * releases all non-used preallocated blocks for given inode
4336 	struct super_block *sb = inode->i_sb;  in ext4_discard_preallocations()
4344 	if (!S_ISREG(inode->i_mode)) {  in ext4_discard_preallocations()
4345 		/*BUG_ON(!list_empty(&ei->i_prealloc_list));*/  in ext4_discard_preallocations()
4349 	if (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)  in ext4_discard_preallocations()
4353 		 inode->i_ino);  in ext4_discard_preallocations()
4355 			atomic_read(&ei->i_prealloc_active), needed);  in ext4_discard_preallocations()
4364 	spin_lock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
4365 	while (!list_empty(&ei->i_prealloc_list) && needed) {  in ext4_discard_preallocations()
4366 		pa = list_entry(ei->i_prealloc_list.prev,  in ext4_discard_preallocations()
4368 		BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock);  in ext4_discard_preallocations()
4369 		spin_lock(&pa->pa_lock);  in ext4_discard_preallocations()
4370 		if (atomic_read(&pa->pa_count)) {  in ext4_discard_preallocations()
4371 			/* this shouldn't happen often - nobody should  in ext4_discard_preallocations()
4373 			spin_unlock(&pa->pa_lock);  in ext4_discard_preallocations()
4374 			spin_unlock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
4376 				 "uh-oh! used pa while discarding");  in ext4_discard_preallocations()
4382 		if (pa->pa_deleted == 0) {  in ext4_discard_preallocations()
4384 			spin_unlock(&pa->pa_lock);  in ext4_discard_preallocations()
4385 			list_del_rcu(&pa->pa_inode_list);  in ext4_discard_preallocations()
4386 			list_add(&pa->u.pa_tmp_list, &list);  in ext4_discard_preallocations()
4387 			needed--;  in ext4_discard_preallocations()
4392 		spin_unlock(&pa->pa_lock);  in ext4_discard_preallocations()
4393 		spin_unlock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
4398 		 * ->clear_inode() the inode will get freed  in ext4_discard_preallocations()
4401 		 * freed memory, bad-bad-bad */  in ext4_discard_preallocations()
4405 		 * of ->clear_inode(), but not in case of  in ext4_discard_preallocations()
4410 	spin_unlock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
4413 		BUG_ON(pa->pa_type != MB_INODE_PA);  in ext4_discard_preallocations()
4414 		group = ext4_get_group_number(sb, pa->pa_pstart);  in ext4_discard_preallocations()
4419 			ext4_error_err(sb, -err, "Error %d loading buddy information for %u",  in ext4_discard_preallocations()
4427 			ext4_error_err(sb, -err, "Error %d reading block bitmap for %u",  in ext4_discard_preallocations()
4434 		list_del(&pa->pa_group_list);  in ext4_discard_preallocations()
4441 		list_del(&pa->u.pa_tmp_list);  in ext4_discard_preallocations()
4442 		call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_discard_preallocations()
4453 		return -ENOMEM;  in ext4_mb_pa_alloc()
4454 	atomic_set(&pa->pa_count, 1);  in ext4_mb_pa_alloc()
4455 	ac->ac_pa = pa;  in ext4_mb_pa_alloc()
4461 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_mb_pa_free()
4464 	ac->ac_pa = NULL;  in ext4_mb_pa_free()
4465 	WARN_ON(!atomic_dec_and_test(&pa->pa_count));  in ext4_mb_pa_free()
4485 		list_for_each(cur, &grp->bb_prealloc_list) {  in ext4_mb_show_pa()
4488 			spin_lock(&pa->pa_lock);  in ext4_mb_show_pa()
4489 			ext4_get_group_no_and_offset(sb, pa->pa_pstart,  in ext4_mb_show_pa()
4491 			spin_unlock(&pa->pa_lock);  in ext4_mb_show_pa()
4493 				 pa->pa_len);  in ext4_mb_show_pa()
4496 		mb_debug(sb, "%u: %d/%d\n", i, grp->bb_free,  in ext4_mb_show_pa()
4497 			 grp->bb_fragments);  in ext4_mb_show_pa()
4503 	struct super_block *sb = ac->ac_sb;  in ext4_mb_show_ac()
4511 			ac->ac_status, ac->ac_flags);  in ext4_mb_show_ac()
4515 			(unsigned long)ac->ac_o_ex.fe_group,  in ext4_mb_show_ac()
4516 			(unsigned long)ac->ac_o_ex.fe_start,  in ext4_mb_show_ac()
4517 			(unsigned long)ac->ac_o_ex.fe_len,  in ext4_mb_show_ac()
4518 			(unsigned long)ac->ac_o_ex.fe_logical,  in ext4_mb_show_ac()
4519 			(unsigned long)ac->ac_g_ex.fe_group,  in ext4_mb_show_ac()
4520 			(unsigned long)ac->ac_g_ex.fe_start,  in ext4_mb_show_ac()
4521 			(unsigned long)ac->ac_g_ex.fe_len,  in ext4_mb_show_ac()
4522 			(unsigned long)ac->ac_g_ex.fe_logical,  in ext4_mb_show_ac()
4523 			(unsigned long)ac->ac_b_ex.fe_group,  in ext4_mb_show_ac()
4524 			(unsigned long)ac->ac_b_ex.fe_start,  in ext4_mb_show_ac()
4525 			(unsigned long)ac->ac_b_ex.fe_len,  in ext4_mb_show_ac()
4526 			(unsigned long)ac->ac_b_ex.fe_logical,  in ext4_mb_show_ac()
4527 			(int)ac->ac_criteria);  in ext4_mb_show_ac()
4528 	mb_debug(sb, "%u found", ac->ac_found);  in ext4_mb_show_ac()
4538 	ext4_mb_show_pa(ac->ac_sb);  in ext4_mb_show_ac()
4544  * We use locality group preallocation for small size file. The size of the
4552 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_group_or_file()
4553 	int bsbits = ac->ac_sb->s_blocksize_bits;  in ext4_mb_group_or_file()
4556 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_group_or_file()
4559 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_group_or_file()
4562 	size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len);  in ext4_mb_group_or_file()
4563 	isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1)  in ext4_mb_group_or_file()
4567 	    !inode_is_open_for_write(ac->ac_inode)) {  in ext4_mb_group_or_file()
4568 		ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC;  in ext4_mb_group_or_file()
4572 	if (sbi->s_mb_group_prealloc <= 0) {  in ext4_mb_group_or_file()
4573 		ac->ac_flags |= EXT4_MB_STREAM_ALLOC;  in ext4_mb_group_or_file()
4579 	if (size > sbi->s_mb_stream_request) {  in ext4_mb_group_or_file()
4580 		ac->ac_flags |= EXT4_MB_STREAM_ALLOC;  in ext4_mb_group_or_file()
4584 	BUG_ON(ac->ac_lg != NULL);  in ext4_mb_group_or_file()
4586 	 * locality group prealloc space are per cpu. The reason for having  in ext4_mb_group_or_file()
4587 	 * per cpu locality group is to reduce the contention between block  in ext4_mb_group_or_file()
4590 	ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups);  in ext4_mb_group_or_file()
4593 	ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC;  in ext4_mb_group_or_file()
4596 	mutex_lock(&ac->ac_lg->lg_mutex);  in ext4_mb_group_or_file()
4603 	struct super_block *sb = ar->inode->i_sb;  in ext4_mb_initialize_context()
4605 	struct ext4_super_block *es = sbi->s_es;  in ext4_mb_initialize_context()
4612 	len = ar->len;  in ext4_mb_initialize_context()
4619 	goal = ar->goal;  in ext4_mb_initialize_context()
4620 	if (goal < le32_to_cpu(es->s_first_data_block) ||  in ext4_mb_initialize_context()
4622 		goal = le32_to_cpu(es->s_first_data_block);  in ext4_mb_initialize_context()
4626 	ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical);  in ext4_mb_initialize_context()
4627 	ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_initialize_context()
4628 	ac->ac_sb = sb;  in ext4_mb_initialize_context()
4629 	ac->ac_inode = ar->inode;  in ext4_mb_initialize_context()
4630 	ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical;  in ext4_mb_initialize_context()
4631 	ac->ac_o_ex.fe_group = group;  in ext4_mb_initialize_context()
4632 	ac->ac_o_ex.fe_start = block;  in ext4_mb_initialize_context()
4633 	ac->ac_o_ex.fe_len = len;  in ext4_mb_initialize_context()
4634 	ac->ac_g_ex = ac->ac_o_ex;  in ext4_mb_initialize_context()
4635 	ac->ac_flags = ar->flags;  in ext4_mb_initialize_context()
4638 	 * locality group. this is a policy, actually */  in ext4_mb_initialize_context()
4643 			(unsigned) ar->len, (unsigned) ar->logical,  in ext4_mb_initialize_context()
4644 			(unsigned) ar->goal, ac->ac_flags, ac->ac_2order,  in ext4_mb_initialize_context()
4645 			(unsigned) ar->lleft, (unsigned) ar->pleft,  in ext4_mb_initialize_context()
4646 			(unsigned) ar->lright, (unsigned) ar->pright,  in ext4_mb_initialize_context()
4647 			inode_is_open_for_write(ar->inode) ? "" : "non-");  in ext4_mb_initialize_context()
4662 	mb_debug(sb, "discard locality group preallocation\n");  in ext4_mb_discard_lg_preallocations()
4666 	spin_lock(&lg->lg_prealloc_lock);  in ext4_mb_discard_lg_preallocations()
4667 	list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order],  in ext4_mb_discard_lg_preallocations()
4669 				lockdep_is_held(&lg->lg_prealloc_lock)) {  in ext4_mb_discard_lg_preallocations()
4670 		spin_lock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
4671 		if (atomic_read(&pa->pa_count)) {  in ext4_mb_discard_lg_preallocations()
4677 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
4680 		if (pa->pa_deleted) {  in ext4_mb_discard_lg_preallocations()
4681 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
4685 		BUG_ON(pa->pa_type != MB_GROUP_PA);  in ext4_mb_discard_lg_preallocations()
4689 		spin_unlock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
4691 		list_del_rcu(&pa->pa_inode_list);  in ext4_mb_discard_lg_preallocations()
4692 		list_add(&pa->u.pa_tmp_list, &discard_list);  in ext4_mb_discard_lg_preallocations()
4694 		total_entries--;  in ext4_mb_discard_lg_preallocations()
4705 	spin_unlock(&lg->lg_prealloc_lock);  in ext4_mb_discard_lg_preallocations()
4710 		group = ext4_get_group_number(sb, pa->pa_pstart);  in ext4_mb_discard_lg_preallocations()
4714 			ext4_error_err(sb, -err, "Error %d loading buddy information for %u",  in ext4_mb_discard_lg_preallocations()
4719 		list_del(&pa->pa_group_list);  in ext4_mb_discard_lg_preallocations()
4724 		list_del(&pa->u.pa_tmp_list);  in ext4_mb_discard_lg_preallocations()
4725 		call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_mb_discard_lg_preallocations()
4741 	struct super_block *sb = ac->ac_sb;  in ext4_mb_add_n_trim()
4742 	struct ext4_locality_group *lg = ac->ac_lg;  in ext4_mb_add_n_trim()
4743 	struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa;  in ext4_mb_add_n_trim()
4745 	order = fls(pa->pa_free) - 1;  in ext4_mb_add_n_trim()
4746 	if (order > PREALLOC_TB_SIZE - 1)  in ext4_mb_add_n_trim()
4748 		order = PREALLOC_TB_SIZE - 1;  in ext4_mb_add_n_trim()
4750 	spin_lock(&lg->lg_prealloc_lock);  in ext4_mb_add_n_trim()
4751 	list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order],  in ext4_mb_add_n_trim()
4753 				lockdep_is_held(&lg->lg_prealloc_lock)) {  in ext4_mb_add_n_trim()
4754 		spin_lock(&tmp_pa->pa_lock);  in ext4_mb_add_n_trim()
4755 		if (tmp_pa->pa_deleted) {  in ext4_mb_add_n_trim()
4756 			spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_add_n_trim()
4759 		if (!added && pa->pa_free < tmp_pa->pa_free) {  in ext4_mb_add_n_trim()
4761 			list_add_tail_rcu(&pa->pa_inode_list,  in ext4_mb_add_n_trim()
4762 						&tmp_pa->pa_inode_list);  in ext4_mb_add_n_trim()
4769 		spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_add_n_trim()
4773 		list_add_tail_rcu(&pa->pa_inode_list,  in ext4_mb_add_n_trim()
4774 					&lg->lg_prealloc_list[order]);  in ext4_mb_add_n_trim()
4775 	spin_unlock(&lg->lg_prealloc_lock);  in ext4_mb_add_n_trim()
4787  * if per-inode prealloc list is too long, trim some PA
4792 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);  in ext4_mb_trim_inode_pa()
4795 	count = atomic_read(&ei->i_prealloc_active);  in ext4_mb_trim_inode_pa()
4796 	delta = (sbi->s_mb_max_inode_prealloc >> 2) + 1;  in ext4_mb_trim_inode_pa()
4797 	if (count > sbi->s_mb_max_inode_prealloc + delta) {  in ext4_mb_trim_inode_pa()
4798 		count -= sbi->s_mb_max_inode_prealloc;  in ext4_mb_trim_inode_pa()
4808 	struct inode *inode = ac->ac_inode;  in ext4_mb_release_context()
4810 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_release_context()
4811 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_mb_release_context()
4813 		if (pa->pa_type == MB_GROUP_PA) {  in ext4_mb_release_context()
4815 			spin_lock(&pa->pa_lock);  in ext4_mb_release_context()
4816 			pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
4817 			pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
4818 			pa->pa_free -= ac->ac_b_ex.fe_len;  in ext4_mb_release_context()
4819 			pa->pa_len -= ac->ac_b_ex.fe_len;  in ext4_mb_release_context()
4820 			spin_unlock(&pa->pa_lock);  in ext4_mb_release_context()
4828 			if (likely(pa->pa_free)) {  in ext4_mb_release_context()
4829 				spin_lock(pa->pa_obj_lock);  in ext4_mb_release_context()
4830 				list_del_rcu(&pa->pa_inode_list);  in ext4_mb_release_context()
4831 				spin_unlock(pa->pa_obj_lock);  in ext4_mb_release_context()
4836 		if (pa->pa_type == MB_INODE_PA) {  in ext4_mb_release_context()
4838 			 * treat per-inode prealloc list as a lru list, then try  in ext4_mb_release_context()
4841 			spin_lock(pa->pa_obj_lock);  in ext4_mb_release_context()
4842 			list_move(&pa->pa_inode_list, &ei->i_prealloc_list);  in ext4_mb_release_context()
4843 			spin_unlock(pa->pa_obj_lock);  in ext4_mb_release_context()
4846 		ext4_mb_put_pa(ac, ac->ac_sb, pa);  in ext4_mb_release_context()
4848 	if (ac->ac_bitmap_page)  in ext4_mb_release_context()
4849 		put_page(ac->ac_bitmap_page);  in ext4_mb_release_context()
4850 	if (ac->ac_buddy_page)  in ext4_mb_release_context()
4851 		put_page(ac->ac_buddy_page);  in ext4_mb_release_context()
4852 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)  in ext4_mb_release_context()
4853 		mutex_unlock(&ac->ac_lg->lg_mutex);  in ext4_mb_release_context()
4874 		needed -= ret;  in ext4_mb_discard_preallocations()
4893 	freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len);  in ext4_mb_discard_preallocations_should_retry()
4899 	if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) {  in ext4_mb_discard_preallocations_should_retry()
4900 		ac->ac_flags |= EXT4_MB_STRICT_CHECK;  in ext4_mb_discard_preallocations_should_retry()
4930 	sb = ar->inode->i_sb;  in ext4_mb_new_blocks()
4934 	if (sbi->s_mount_state & EXT4_FC_REPLAY)  in ext4_mb_new_blocks()
4938 	if (ext4_is_quota_file(ar->inode))  in ext4_mb_new_blocks()
4939 		ar->flags |= EXT4_MB_USE_ROOT_BLOCKS;  in ext4_mb_new_blocks()
4941 	if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) {  in ext4_mb_new_blocks()
4946 		while (ar->len &&  in ext4_mb_new_blocks()
4947 			ext4_claim_free_clusters(sbi, ar->len, ar->flags)) {  in ext4_mb_new_blocks()
4951 			ar->len = ar->len >> 1;  in ext4_mb_new_blocks()
4953 		if (!ar->len) {  in ext4_mb_new_blocks()
4955 			*errp = -ENOSPC;  in ext4_mb_new_blocks()
4958 		reserv_clstrs = ar->len;  in ext4_mb_new_blocks()
4959 		if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) {  in ext4_mb_new_blocks()
4960 			dquot_alloc_block_nofail(ar->inode,  in ext4_mb_new_blocks()
4961 						 EXT4_C2B(sbi, ar->len));  in ext4_mb_new_blocks()
4963 			while (ar->len &&  in ext4_mb_new_blocks()
4964 				dquot_alloc_block(ar->inode,  in ext4_mb_new_blocks()
4965 						  EXT4_C2B(sbi, ar->len))) {  in ext4_mb_new_blocks()
4967 				ar->flags |= EXT4_MB_HINT_NOPREALLOC;  in ext4_mb_new_blocks()
4968 				ar->len--;  in ext4_mb_new_blocks()
4971 		inquota = ar->len;  in ext4_mb_new_blocks()
4972 		if (ar->len == 0) {  in ext4_mb_new_blocks()
4973 			*errp = -EDQUOT;  in ext4_mb_new_blocks()
4980 		ar->len = 0;  in ext4_mb_new_blocks()
4981 		*errp = -ENOMEM;  in ext4_mb_new_blocks()
4987 		ar->len = 0;  in ext4_mb_new_blocks()
4991 	ac->ac_op = EXT4_MB_HISTORY_PREALLOC;  in ext4_mb_new_blocks()
4994 		ac->ac_op = EXT4_MB_HISTORY_ALLOC;  in ext4_mb_new_blocks()
5004 		 * pa allocated above is added to grp->bb_prealloc_list only  in ext4_mb_new_blocks()
5006 		 * ac->ac_status == AC_STATUS_FOUND.  in ext4_mb_new_blocks()
5007 		 * And error from above mean ac->ac_status != AC_STATUS_FOUND  in ext4_mb_new_blocks()
5015 		if (ac->ac_status == AC_STATUS_FOUND &&  in ext4_mb_new_blocks()
5016 			ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len)  in ext4_mb_new_blocks()
5019 	if (likely(ac->ac_status == AC_STATUS_FOUND)) {  in ext4_mb_new_blocks()
5025 			block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_blocks()
5026 			ar->len = ac->ac_b_ex.fe_len;  in ext4_mb_new_blocks()
5036 		*errp = -ENOSPC;  in ext4_mb_new_blocks()
5041 		ac->ac_b_ex.fe_len = 0;  in ext4_mb_new_blocks()
5042 		ar->len = 0;  in ext4_mb_new_blocks()
5049 	if (inquota && ar->len < inquota)  in ext4_mb_new_blocks()
5050 		dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len));  in ext4_mb_new_blocks()
5051 	if (!ar->len) {  in ext4_mb_new_blocks()
5052 		if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0)  in ext4_mb_new_blocks()
5054 			percpu_counter_sub(&sbi->s_dirtyclusters_counter,  in ext4_mb_new_blocks()
5073 	if ((entry->efd_tid != new_entry->efd_tid) ||  in ext4_try_merge_freed_extent()
5074 	    (entry->efd_group != new_entry->efd_group))  in ext4_try_merge_freed_extent()
5076 	if (entry->efd_start_cluster + entry->efd_count ==  in ext4_try_merge_freed_extent()
5077 	    new_entry->efd_start_cluster) {  in ext4_try_merge_freed_extent()
5078 		new_entry->efd_start_cluster = entry->efd_start_cluster;  in ext4_try_merge_freed_extent()
5079 		new_entry->efd_count += entry->efd_count;  in ext4_try_merge_freed_extent()
5080 	} else if (new_entry->efd_start_cluster + new_entry->efd_count ==  in ext4_try_merge_freed_extent()
5081 		   entry->efd_start_cluster) {  in ext4_try_merge_freed_extent()
5082 		new_entry->efd_count += entry->efd_count;  in ext4_try_merge_freed_extent()
5085 	spin_lock(&sbi->s_md_lock);  in ext4_try_merge_freed_extent()
5086 	list_del(&entry->efd_list);  in ext4_try_merge_freed_extent()
5087 	spin_unlock(&sbi->s_md_lock);  in ext4_try_merge_freed_extent()
5088 	rb_erase(&entry->efd_node, entry_rb_root);  in ext4_try_merge_freed_extent()
5096 	ext4_group_t group = e4b->bd_group;  in ext4_mb_free_metadata()
5098 	ext4_grpblk_t clusters = new_entry->efd_count;  in ext4_mb_free_metadata()
5100 	struct ext4_group_info *db = e4b->bd_info;  in ext4_mb_free_metadata()
5101 	struct super_block *sb = e4b->bd_sb;  in ext4_mb_free_metadata()
5103 	struct rb_node **n = &db->bb_free_root.rb_node, *node;  in ext4_mb_free_metadata()
5107 	BUG_ON(e4b->bd_bitmap_page == NULL);  in ext4_mb_free_metadata()
5108 	BUG_ON(e4b->bd_buddy_page == NULL);  in ext4_mb_free_metadata()
5110 	new_node = &new_entry->efd_node;  in ext4_mb_free_metadata()
5111 	cluster = new_entry->efd_start_cluster;  in ext4_mb_free_metadata()
5117 		 * on-disk bitmap and lose not-yet-available  in ext4_mb_free_metadata()
5119 		get_page(e4b->bd_buddy_page);  in ext4_mb_free_metadata()
5120 		get_page(e4b->bd_bitmap_page);  in ext4_mb_free_metadata()
5125 		if (cluster < entry->efd_start_cluster)  in ext4_mb_free_metadata()
5126 			n = &(*n)->rb_left;  in ext4_mb_free_metadata()
5127 		else if (cluster >= (entry->efd_start_cluster + entry->efd_count))  in ext4_mb_free_metadata()
5128 			n = &(*n)->rb_right;  in ext4_mb_free_metadata()
5133 				"Block already on to-be-freed list");  in ext4_mb_free_metadata()
5140 	rb_insert_color(new_node, &db->bb_free_root);  in ext4_mb_free_metadata()
5147 					    &(db->bb_free_root));  in ext4_mb_free_metadata()
5154 					    &(db->bb_free_root));  in ext4_mb_free_metadata()
5157 	spin_lock(&sbi->s_md_lock);  in ext4_mb_free_metadata()
5158 	list_add_tail(&new_entry->efd_list, &sbi->s_freed_data_list);  in ext4_mb_free_metadata()
5159 	sbi->s_mb_free_pending += clusters;  in ext4_mb_free_metadata()
5160 	spin_unlock(&sbi->s_md_lock);  in ext4_mb_free_metadata()
5173 	struct super_block *sb = ar->inode->i_sb;  in ext4_mb_new_blocks_simple()
5176 	int i = sb->s_blocksize;  in ext4_mb_new_blocks_simple()
5178 	struct ext4_super_block *es = EXT4_SB(sb)->s_es;  in ext4_mb_new_blocks_simple()
5180 	goal = ar->goal;  in ext4_mb_new_blocks_simple()
5181 	if (goal < le32_to_cpu(es->s_first_data_block) ||  in ext4_mb_new_blocks_simple()
5183 		goal = le32_to_cpu(es->s_first_data_block);  in ext4_mb_new_blocks_simple()
5185 	ar->len = 0;  in ext4_mb_new_blocks_simple()
5198 		i = mb_find_next_zero_bit(bitmap_bh->b_data, sb->s_blocksize,  in ext4_mb_new_blocks_simple()
5201 		if (i >= sb->s_blocksize)  in ext4_mb_new_blocks_simple()
5209 	if (group >= ext4_get_groups_count(sb) && i >= sb->s_blocksize)  in ext4_mb_new_blocks_simple()
5214 	ar->len = 1;  in ext4_mb_new_blocks_simple()
5223 	struct super_block *sb = inode->i_sb;  in ext4_free_blocks_simple()
5242 		if (!mb_test_bit(blkoff + i, bitmap_bh->b_data))  in ext4_free_blocks_simple()
5245 	mb_clear_bits(bitmap_bh->b_data, blkoff, count);  in ext4_free_blocks_simple()
5251 		count - already_freed);  in ext4_free_blocks_simple()
5261  * ext4_free_blocks() -- Free given blocks and update quota
5274 	struct super_block *sb = inode->i_sb;  in ext4_free_blocks()
5288 	if (sbi->s_mount_state & EXT4_FC_REPLAY) {  in ext4_free_blocks()
5296 			BUG_ON(block != bh->b_blocknr);  in ext4_free_blocks()
5298 			block = bh->b_blocknr;  in ext4_free_blocks()
5303 		ext4_error(sb, "Freeing blocks not in datazone - "  in ext4_free_blocks()
5328 			overflow = sbi->s_cluster_ratio - overflow;  in ext4_free_blocks()
5331 				count -= overflow;  in ext4_free_blocks()
5335 			block -= overflow;  in ext4_free_blocks()
5343 				count -= overflow;  in ext4_free_blocks()
5347 			count += sbi->s_cluster_ratio - overflow;  in ext4_free_blocks()
5357 				bh = sb_find_get_block(inode->i_sb, block + i);  in ext4_free_blocks()
5375 		overflow = EXT4_C2B(sbi, bit) + count -  in ext4_free_blocks()
5377 		count -= overflow;  in ext4_free_blocks()
5388 		err = -EIO;  in ext4_free_blocks()
5395 		     sbi->s_itb_per_group) ||  in ext4_free_blocks()
5396 	    in_range(block + count - 1, ext4_inode_table(sb, gdp),  in ext4_free_blocks()
5397 		     sbi->s_itb_per_group)) {  in ext4_free_blocks()
5399 		ext4_error(sb, "Freeing blocks in system zone - "  in ext4_free_blocks()
5412 	 * to unshare ->b_data if a currently-committing transaction is  in ext4_free_blocks()
5423 			BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data));  in ext4_free_blocks()
5450 		new_entry->efd_start_cluster = bit;  in ext4_free_blocks()
5451 		new_entry->efd_group = block_group;  in ext4_free_blocks()
5452 		new_entry->efd_count = count_clusters;  in ext4_free_blocks()
5453 		new_entry->efd_tid = handle->h_transaction->t_tid;  in ext4_free_blocks()
5456 		mb_clear_bits(bitmap_bh->b_data, bit, count_clusters);  in ext4_free_blocks()
5459 		/* need to update group_info->bb_free and bitmap  in ext4_free_blocks()
5466 			if (err && err != -EOPNOTSUPP)  in ext4_free_blocks()
5475 		mb_clear_bits(bitmap_bh->b_data, bit, count_clusters);  in ext4_free_blocks()
5485 	if (sbi->s_log_groups_per_flex) {  in ext4_free_blocks()
5489 						  flex_group)->free_clusters);  in ext4_free_blocks()
5500 		percpu_counter_add(&sbi->s_freeclusters_counter,  in ext4_free_blocks()
5529  * ext4_group_add_blocks() -- Add given blocks to an existing group
5551 	ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1);  in ext4_group_add_blocks()
5552 	unsigned long cluster_count = last_cluster - first_cluster + 1;  in ext4_group_add_blocks()
5554 	ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1);  in ext4_group_add_blocks()
5567 		err = -EINVAL;  in ext4_group_add_blocks()
5580 		err = -EIO;  in ext4_group_add_blocks()
5586 	    in_range(block, ext4_inode_table(sb, desc), sbi->s_itb_per_group) ||  in ext4_group_add_blocks()
5587 	    in_range(block + count - 1, ext4_inode_table(sb, desc),  in ext4_group_add_blocks()
5588 		     sbi->s_itb_per_group)) {  in ext4_group_add_blocks()
5589 		ext4_error(sb, "Adding blocks in system zones - "  in ext4_group_add_blocks()
5592 		err = -EINVAL;  in ext4_group_add_blocks()
5603 	 * to unshare ->b_data if a currently-committing transaction is  in ext4_group_add_blocks()
5613 		if (!mb_test_bit(bit + i, bitmap_bh->b_data)) {  in ext4_group_add_blocks()
5627 	 * need to update group_info->bb_free and bitmap  in ext4_group_add_blocks()
5632 	mb_clear_bits(bitmap_bh->b_data, bit, cluster_count);  in ext4_group_add_blocks()
5640 	percpu_counter_add(&sbi->s_freeclusters_counter,  in ext4_group_add_blocks()
5643 	if (sbi->s_log_groups_per_flex) {  in ext4_group_add_blocks()
5647 						  flex_group)->free_clusters);  in ext4_group_add_blocks()
5669  * ext4_trim_extent -- function to TRIM one single free extent in the group
5709  * ext4_trim_all_free -- function to trim all free space in alloc. group
5748 	    minblocks >= atomic_read(&EXT4_SB(sb)->s_last_trim_minblks))  in ext4_trim_all_free()
5751 	start = (e4b.bd_info->bb_first_free > start) ?  in ext4_trim_all_free()
5752 		e4b.bd_info->bb_first_free : start;  in ext4_trim_all_free()
5760 		if ((next - start) >= minblocks) {  in ext4_trim_all_free()
5762 					       next - start, group, &e4b);  in ext4_trim_all_free()
5763 			if (ret && ret != -EOPNOTSUPP)  in ext4_trim_all_free()
5766 			count += next - start;  in ext4_trim_all_free()
5768 		free_count += next - start;  in ext4_trim_all_free()
5772 			count = -ERESTARTSYS;  in ext4_trim_all_free()
5782 		if ((e4b.bd_info->bb_free - free_count) < minblocks)  in ext4_trim_all_free()
5801  * ext4_trim_fs() -- trim ioctl handle function
5814 	struct request_queue *q = bdev_get_queue(sb->s_bdev);  in ext4_trim_fs()
5820 			le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);  in ext4_trim_fs()
5821 	ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es);  in ext4_trim_fs()
5824 	start = range->start >> sb->s_blocksize_bits;  in ext4_trim_fs()
5825 	end = start + (range->len >> sb->s_blocksize_bits) - 1;  in ext4_trim_fs()
5827 			      range->minlen >> sb->s_blocksize_bits);  in ext4_trim_fs()
5831 	    range->len < sb->s_blocksize)  in ext4_trim_fs()
5832 		return -EINVAL;  in ext4_trim_fs()
5834 	if (range->minlen < q->limits.discard_granularity) {  in ext4_trim_fs()
5836 			q->limits.discard_granularity >> sb->s_blocksize_bits);  in ext4_trim_fs()
5841 		end = max_blks - 1;  in ext4_trim_fs()
5854 	end = EXT4_CLUSTERS_PER_GROUP(sb) - 1;  in ext4_trim_fs()
5867 		 * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to  in ext4_trim_fs()
5874 		if (grp->bb_free >= minlen) {  in ext4_trim_fs()
5892 		atomic_set(&EXT4_SB(sb)->s_last_trim_minblks, minlen);  in ext4_trim_fs()
5895 	range->len = EXT4_C2B(EXT4_SB(sb), trimmed) << sb->s_blocksize_bits;  in ext4_trim_fs()
5921 	start = (e4b.bd_info->bb_first_free > start) ?  in ext4_mballoc_query_range()
5922 		e4b.bd_info->bb_first_free : start;  in ext4_mballoc_query_range()
5924 		end = EXT4_CLUSTERS_PER_GROUP(sb) - 1;  in ext4_mballoc_query_range()
5933 		error = formatter(sb, group, start, next - start, priv);  in ext4_mballoc_query_range()