1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
18 #include <linux/backing-dev.h>
24  *   - test ext4_ext_search_left() and ext4_ext_search_right()
25  *   - search for metadata in few groups
28  *   - normalization should take into account whether file is still open
29  *   - discard preallocations if no free space left (policy?)
30  *   - don't normalize tails
31  *   - quota
32  *   - reservation for superuser
35  *   - bitmap read-ahead (proposed by Oleg Drokin aka green)
36  *   - track min/max extents in each group for better group selection
37  *   - mb_mark_used() may allocate chunk right after splitting buddy
38  *   - tree of groups sorted by number of free blocks
39  *   - error handling
50  * is larger. If the size is less than sbi->s_mb_stream_request we
60  * ext4_inode_info->i_prealloc_list, which contains list of prealloc
64  * pa_lstart -> the logical start block for this prealloc space
65  * pa_pstart -> the physical start block for this prealloc space
66  * pa_len    -> length for this prealloc space (in clusters)
67  * pa_free   ->  free space available in this prealloc space (in clusters)
79  * have the group allocation flag set then we look at the locality group
84  * The reason for having a per cpu locality group is to reduce the contention
87  * The locality group prealloc space is used looking at whether we have
90  * If we can't allocate blocks via inode prealloc or/and locality group
122  * sbi->s_mb_group_prealloc.  The default value of s_mb_group_prealloc is
123  * dependent on the cluster size; for non-bigalloc file systems, it is
126  * terms of number of blocks. If we have mounted the file system with -O
128  * smallest multiple of the stripe value (sbi->s_stripe) which is
138  * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The
141  * stripe size (sbi->s_stripe), we try to search for contiguous block in
143  * not, we search in the specific group using bitmap for best extents. The
161  *  - on-disk bitmap
162  *  - in-core buddy (actually includes buddy and bitmap)
163  *  - preallocation descriptors (PAs)
166  *  - inode
167  *    assiged to specific inode and can be used for this inode only.
168  *    it describes part of inode's space preallocated to specific
175  *  - locality group
176  *    assigned to specific locality group which does not translate to
182  *    in-core buddy = on-disk bitmap + preallocation descriptors
185  *  - allocated blocks (persistent)
186  *  - preallocated blocks (non-persistent)
190  * literally -- time is discrete and delimited by locks.
193  *  blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA.
196  *  - init buddy:			buddy = on-disk + PAs
197  *  - new PA:				buddy += N; PA = N
198  *  - use inode PA:			on-disk += N; PA -= N
199  *  - discard inode PA			buddy -= on-disk - PA; PA = 0
200  *  - use locality group PA		on-disk += N; PA -= N
201  *  - discard locality group PA		buddy -= PA; PA = 0
202  *  note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap
204  *        bits from PA, only from on-disk bitmap
208  * killing performance on high-end SMP hardware. let's try to relax it using
212  *     nobody can re-allocate that block
213  *  3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has
215  *     on-disk bitmap if buddy has same bit set or/and PA covers corresponded
219  *  - init buddy vs.
220  *    - new PA
223  *    - use inode PA
224  *      we need to make sure that either on-disk bitmap or PA has uptodate data
225  *      given (3) we care that PA-=N operation doesn't interfere with init
226  *    - discard inode PA
228  *    - use locality group PA
229  *      again PA-=N must be serialized with init
230  *    - discard locality group PA
232  *  - new PA vs.
233  *    - use inode PA
235  *    - discard inode PA
237  *    - use locality group PA
239  *    - discard locality group PA
241  *  - use inode PA
242  *    - use inode PA
244  *    - discard inode PA
246  *    - use locality group PA
247  *      nothing wrong here -- they're different PAs covering different blocks
248  *    - discard locality group PA
252  *  - PA is referenced and while it is no discard is possible
253  *  - PA is referenced until block isn't marked in on-disk bitmap
254  *  - PA changes only after on-disk bitmap
255  *  - discard must not compete with init. either init is done before
257  *  - buddy init as sum of on-disk bitmap and PAs is done atomically
267  *  - allocation:
270  *    mark bits in on-disk bitmap
273  *  - use preallocation:
274  *    find proper PA (per-inode or group)
276  *    mark bits in on-disk bitmap
280  *  - free:
282  *    mark bits in on-disk bitmap
285  *  - discard preallocations in group:
288  *    load on-disk bitmap
290  *    remove PA from object (inode or locality group)
291  *    mark free blocks in-core
293  *  - discard inode's preallocations:
300  *  - bitlock on a group	(group)
301  *  - object (inode/locality)	(object)
302  *  - per-pa lock		(pa)
305  *  - new pa
309  *  - find and use pa:
312  *  - release consumed pa:
317  *  - generate in-core bitmap:
321  *  - discard all for given object (inode, locality group):
326  *  - discard all for given group:
371  * sure that all the PAs on grp->bb_prealloc_list got freed or if it's empty.
433 	ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix;  in mb_find_next_zero_bit()
446 	ret = ext4_find_next_bit(addr, tmpmax, start) - fix;  in mb_find_next_bit()
456 	BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);  in mb_find_buddy()
459 	if (order > e4b->bd_blkbits + 1) {  in mb_find_buddy()
466 		*max = 1 << (e4b->bd_blkbits + 3);  in mb_find_buddy()
467 		return e4b->bd_bitmap;  in mb_find_buddy()
470 	bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order];  in mb_find_buddy()
471 	*max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order];  in mb_find_buddy()
481 	struct super_block *sb = e4b->bd_sb;  in mb_free_blocks_double()
483 	if (unlikely(e4b->bd_info->bb_bitmap == NULL))  in mb_free_blocks_double()
485 	assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));  in mb_free_blocks_double()
487 		if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) {  in mb_free_blocks_double()
490 			blocknr = ext4_group_first_block_no(sb, e4b->bd_group);  in mb_free_blocks_double()
492 			ext4_grp_locked_error(sb, e4b->bd_group,  in mb_free_blocks_double()
493 					      inode ? inode->i_ino : 0,  in mb_free_blocks_double()
498 			ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in mb_free_blocks_double()
501 		mb_clear_bit(first + i, e4b->bd_info->bb_bitmap);  in mb_free_blocks_double()
509 	if (unlikely(e4b->bd_info->bb_bitmap == NULL))  in mb_mark_used_double()
511 	assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));  in mb_mark_used_double()
513 		BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap));  in mb_mark_used_double()
514 		mb_set_bit(first + i, e4b->bd_info->bb_bitmap);  in mb_mark_used_double()
520 	if (unlikely(e4b->bd_info->bb_bitmap == NULL))  in mb_cmp_bitmaps()
522 	if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) {  in mb_cmp_bitmaps()
525 		b1 = (unsigned char *) e4b->bd_info->bb_bitmap;  in mb_cmp_bitmaps()
527 		for (i = 0; i < e4b->bd_sb->s_blocksize; i++) {  in mb_cmp_bitmaps()
529 				ext4_msg(e4b->bd_sb, KERN_ERR,  in mb_cmp_bitmaps()
533 					 e4b->bd_group, i, i * 8, b1[i], b2[i]);  in mb_cmp_bitmaps()
545 	grp->bb_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS);  in mb_group_bb_bitmap_alloc()
546 	if (!grp->bb_bitmap)  in mb_group_bb_bitmap_alloc()
551 		kfree(grp->bb_bitmap);  in mb_group_bb_bitmap_alloc()
552 		grp->bb_bitmap = NULL;  in mb_group_bb_bitmap_alloc()
556 	memcpy(grp->bb_bitmap, bh->b_data, sb->s_blocksize);  in mb_group_bb_bitmap_alloc()
562 	kfree(grp->bb_bitmap);  in mb_group_bb_bitmap_free()
608 	struct super_block *sb = e4b->bd_sb;  in __mb_check_buddy()
609 	int order = e4b->bd_blkbits + 1;  in __mb_check_buddy()
623 	if (e4b->bd_info->bb_check_counter++ % 10)  in __mb_check_buddy()
629 		buddy2 = mb_find_buddy(e4b, order - 1, &max2);  in __mb_check_buddy()
656 					!mb_test_bit(k, e4b->bd_bitmap));  in __mb_check_buddy()
660 		MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count);  in __mb_check_buddy()
661 		order--;  in __mb_check_buddy()
664 	fstart = -1;  in __mb_check_buddy()
668 			MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free);  in __mb_check_buddy()
669 			if (fstart == -1) {  in __mb_check_buddy()
675 		fstart = -1;  in __mb_check_buddy()
677 		for (j = 0; j < e4b->bd_blkbits + 1; j++) {  in __mb_check_buddy()
684 	MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info));  in __mb_check_buddy()
685 	MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments);  in __mb_check_buddy()
687 	grp = ext4_get_group_info(sb, e4b->bd_group);  in __mb_check_buddy()
690 	list_for_each(cur, &grp->bb_prealloc_list) {  in __mb_check_buddy()
694 		ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k);  in __mb_check_buddy()
695 		MB_CHECK_ASSERT(groupnr == e4b->bd_group);  in __mb_check_buddy()
696 		for (i = 0; i < pa->pa_len; i++)  in __mb_check_buddy()
726 	border = 2 << sb->s_blocksize_bits;  in ext4_mb_mark_free_simple()
730 		max = ffs(first | border) - 1;  in ext4_mb_mark_free_simple()
733 		min = fls(len) - 1;  in ext4_mb_mark_free_simple()
740 		grp->bb_counters[min]++;  in ext4_mb_mark_free_simple()
743 				     buddy + sbi->s_mb_offsets[min]);  in ext4_mb_mark_free_simple()
745 		len -= chunk;  in ext4_mb_mark_free_simple()
760 	grp->bb_largest_free_order = -1; /* uninit */  in mb_set_largest_free_order()
762 	bits = sb->s_blocksize_bits + 1;  in mb_set_largest_free_order()
763 	for (i = bits; i >= 0; i--) {  in mb_set_largest_free_order()
764 		if (grp->bb_counters[i] > 0) {  in mb_set_largest_free_order()
765 			grp->bb_largest_free_order = i;  in mb_set_largest_free_order()
786 	 * of on-disk bitmap and preallocations */  in ext4_mb_generate_buddy()
788 	grp->bb_first_free = i;  in ext4_mb_generate_buddy()
793 		len = i - first;  in ext4_mb_generate_buddy()
798 			grp->bb_counters[0]++;  in ext4_mb_generate_buddy()
802 	grp->bb_fragments = fragments;  in ext4_mb_generate_buddy()
804 	if (free != grp->bb_free) {  in ext4_mb_generate_buddy()
808 				      free, grp->bb_free);  in ext4_mb_generate_buddy()
813 		grp->bb_free = free;  in ext4_mb_generate_buddy()
819 	clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state));  in ext4_mb_generate_buddy()
821 	period = get_cycles() - period;  in ext4_mb_generate_buddy()
822 	atomic_inc(&sbi->s_mb_buddies_generated);  in ext4_mb_generate_buddy()
823 	atomic64_add(period, &sbi->s_mb_generation_time);  in ext4_mb_generate_buddy()
835 	e4b->bd_info->bb_fragments = 0;  in mb_regenerate_buddy()
836 	memset(e4b->bd_info->bb_counters, 0,  in mb_regenerate_buddy()
837 		sizeof(*e4b->bd_info->bb_counters) *  in mb_regenerate_buddy()
838 		(e4b->bd_sb->s_blocksize_bits + 2));  in mb_regenerate_buddy()
840 	ext4_mb_generate_buddy(e4b->bd_sb, e4b->bd_buddy,  in mb_regenerate_buddy()
841 		e4b->bd_bitmap, e4b->bd_group, e4b->bd_info);  in mb_regenerate_buddy()
882 	inode = page->mapping->host;  in ext4_mb_init_cache()
883 	sb = inode->i_sb;  in ext4_mb_init_cache()
888 	mb_debug(sb, "init page %lu\n", page->index);  in ext4_mb_init_cache()
899 			err = -ENOMEM;  in ext4_mb_init_cache()
905 	first_group = page->index * blocks_per_page / 2;  in ext4_mb_init_cache()
945 	first_block = page->index * blocks_per_page;  in ext4_mb_init_cache()
951 		if (!bh[group - first_group])  in ext4_mb_init_cache()
955 		if (!buffer_verified(bh[group - first_group]))  in ext4_mb_init_cache()
967 		bitmap = bh[group - first_group]->b_data;  in ext4_mb_init_cache()
977 				group, page->index, i * blocksize);  in ext4_mb_init_cache()
981 				err = -EFSCORRUPTED;  in ext4_mb_init_cache()
984 			grinfo->bb_fragments = 0;  in ext4_mb_init_cache()
985 			memset(grinfo->bb_counters, 0,  in ext4_mb_init_cache()
986 			       sizeof(*grinfo->bb_counters) *  in ext4_mb_init_cache()
987 				(sb->s_blocksize_bits+2));  in ext4_mb_init_cache()
1001 				group, page->index, i * blocksize);  in ext4_mb_init_cache()
1008 			/* mark all preallocated blks used in in-core bitmap */  in ext4_mb_init_cache()
1035  * are on the same page e4b->bd_buddy_page is NULL and return value is 0.
1040 	struct inode *inode = EXT4_SB(sb)->s_buddy_cache;  in ext4_mb_get_buddy_page_lock()
1045 	e4b->bd_buddy_page = NULL;  in ext4_mb_get_buddy_page_lock()
1046 	e4b->bd_bitmap_page = NULL;  in ext4_mb_get_buddy_page_lock()
1048 	blocks_per_page = PAGE_SIZE / sb->s_blocksize;  in ext4_mb_get_buddy_page_lock()
1057 	page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_get_buddy_page_lock()
1059 		return -ENOMEM;  in ext4_mb_get_buddy_page_lock()
1060 	BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_get_buddy_page_lock()
1061 	e4b->bd_bitmap_page = page;  in ext4_mb_get_buddy_page_lock()
1062 	e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);  in ext4_mb_get_buddy_page_lock()
1071 	page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_get_buddy_page_lock()
1073 		return -ENOMEM;  in ext4_mb_get_buddy_page_lock()
1074 	BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_get_buddy_page_lock()
1075 	e4b->bd_buddy_page = page;  in ext4_mb_get_buddy_page_lock()
1081 	if (e4b->bd_bitmap_page) {  in ext4_mb_put_buddy_page_lock()
1082 		unlock_page(e4b->bd_bitmap_page);  in ext4_mb_put_buddy_page_lock()
1083 		put_page(e4b->bd_bitmap_page);  in ext4_mb_put_buddy_page_lock()
1085 	if (e4b->bd_buddy_page) {  in ext4_mb_put_buddy_page_lock()
1086 		unlock_page(e4b->bd_buddy_page);  in ext4_mb_put_buddy_page_lock()
1087 		put_page(e4b->bd_buddy_page);  in ext4_mb_put_buddy_page_lock()
1109 		return -EFSCORRUPTED;  in ext4_mb_init_group()
1134 		ret = -EIO;  in ext4_mb_init_group()
1153 		ret = -EIO;  in ext4_mb_init_group()
1178 	struct inode *inode = sbi->s_buddy_cache;  in ext4_mb_load_buddy_gfp()
1183 	blocks_per_page = PAGE_SIZE / sb->s_blocksize;  in ext4_mb_load_buddy_gfp()
1186 		return -EFSCORRUPTED;  in ext4_mb_load_buddy_gfp()
1188 	e4b->bd_blkbits = sb->s_blocksize_bits;  in ext4_mb_load_buddy_gfp()
1189 	e4b->bd_info = grp;  in ext4_mb_load_buddy_gfp()
1190 	e4b->bd_sb = sb;  in ext4_mb_load_buddy_gfp()
1191 	e4b->bd_group = group;  in ext4_mb_load_buddy_gfp()
1192 	e4b->bd_buddy_page = NULL;  in ext4_mb_load_buddy_gfp()
1193 	e4b->bd_bitmap_page = NULL;  in ext4_mb_load_buddy_gfp()
1216 	page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED);  in ext4_mb_load_buddy_gfp()
1228 		page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_load_buddy_gfp()
1230 			BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_load_buddy_gfp()
1238 					       (poff * sb->s_blocksize));  in ext4_mb_load_buddy_gfp()
1244 		ret = -ENOMEM;  in ext4_mb_load_buddy_gfp()
1248 		ret = -EIO;  in ext4_mb_load_buddy_gfp()
1253 	e4b->bd_bitmap_page = page;  in ext4_mb_load_buddy_gfp()
1254 	e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);  in ext4_mb_load_buddy_gfp()
1260 	page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED);  in ext4_mb_load_buddy_gfp()
1264 		page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_load_buddy_gfp()
1266 			BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_load_buddy_gfp()
1268 				ret = ext4_mb_init_cache(page, e4b->bd_bitmap,  in ext4_mb_load_buddy_gfp()
1279 		ret = -ENOMEM;  in ext4_mb_load_buddy_gfp()
1283 		ret = -EIO;  in ext4_mb_load_buddy_gfp()
1288 	e4b->bd_buddy_page = page;  in ext4_mb_load_buddy_gfp()
1289 	e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize);  in ext4_mb_load_buddy_gfp()
1296 	if (e4b->bd_bitmap_page)  in ext4_mb_load_buddy_gfp()
1297 		put_page(e4b->bd_bitmap_page);  in ext4_mb_load_buddy_gfp()
1298 	if (e4b->bd_buddy_page)  in ext4_mb_load_buddy_gfp()
1299 		put_page(e4b->bd_buddy_page);  in ext4_mb_load_buddy_gfp()
1300 	e4b->bd_buddy = NULL;  in ext4_mb_load_buddy_gfp()
1301 	e4b->bd_bitmap = NULL;  in ext4_mb_load_buddy_gfp()
1313 	if (e4b->bd_bitmap_page)  in ext4_mb_unload_buddy()
1314 		put_page(e4b->bd_bitmap_page);  in ext4_mb_unload_buddy()
1315 	if (e4b->bd_buddy_page)  in ext4_mb_unload_buddy()
1316 		put_page(e4b->bd_buddy_page);  in ext4_mb_unload_buddy()
1323 	int bb_incr = 1 << (e4b->bd_blkbits - 1);  in mb_find_order_for_block()
1326 	BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);  in mb_find_order_for_block()
1327 	BUG_ON(block >= (1 << (e4b->bd_blkbits + 3)));  in mb_find_order_for_block()
1329 	bb = e4b->bd_buddy;  in mb_find_order_for_block()
1330 	while (order <= e4b->bd_blkbits + 1) {  in mb_find_order_for_block()
1349 		if ((cur & 31) == 0 && (len - cur) >= 32) {  in mb_clear_bits()
1362  * will return first found zero bit if any, -1 otherwise
1367 	int zero_bit = -1;  in mb_test_and_clear_bits()
1371 		if ((cur & 31) == 0 && (len - cur) >= 32) {  in mb_test_and_clear_bits()
1374 			if (*addr != (__u32)(-1) && zero_bit == -1)  in mb_test_and_clear_bits()
1380 		if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1)  in mb_test_and_clear_bits()
1394 		if ((cur & 31) == 0 && (len - cur) >= 32) {  in ext4_set_bits()
1410 		(*bit) -= side;  in mb_buddy_adjust_border()
1416 		return -1;  in mb_buddy_adjust_border()
1439 		 * ---------------------------------  in mb_buddy_mark_free()
1441 		 * ---------------------------------  in mb_buddy_mark_free()
1443 		 * ---------------------------------  in mb_buddy_mark_free()
1459 			e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1);  in mb_buddy_mark_free()
1461 			e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1);  in mb_buddy_mark_free()
1467 			mb_clear_bits(buddy, first, last - first + 1);  in mb_buddy_mark_free()
1468 			e4b->bd_info->bb_counters[order - 1] += last - first + 1;  in mb_buddy_mark_free()
1483 	int last = first + count - 1;  in mb_free_blocks()
1484 	struct super_block *sb = e4b->bd_sb;  in mb_free_blocks()
1488 	BUG_ON(last >= (sb->s_blocksize << 3));  in mb_free_blocks()
1489 	assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));  in mb_free_blocks()
1491 	if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))  in mb_free_blocks()
1498 	e4b->bd_info->bb_free += count;  in mb_free_blocks()
1499 	if (first < e4b->bd_info->bb_first_free)  in mb_free_blocks()
1500 		e4b->bd_info->bb_first_free = first;  in mb_free_blocks()
1506 		left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap);  in mb_free_blocks()
1507 	block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count);  in mb_free_blocks()
1508 	if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0])  in mb_free_blocks()
1509 		right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap);  in mb_free_blocks()
1511 	if (unlikely(block != -1)) {  in mb_free_blocks()
1515 		blocknr = ext4_group_first_block_no(sb, e4b->bd_group);  in mb_free_blocks()
1517 		if (!(sbi->s_mount_state & EXT4_FC_REPLAY)) {  in mb_free_blocks()
1518 			ext4_grp_locked_error(sb, e4b->bd_group,  in mb_free_blocks()
1519 					      inode ? inode->i_ino : 0,  in mb_free_blocks()
1524 				sb, e4b->bd_group,  in mb_free_blocks()
1534 		e4b->bd_info->bb_fragments--;  in mb_free_blocks()
1536 		e4b->bd_info->bb_fragments++;  in mb_free_blocks()
1546 		e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1;  in mb_free_blocks()
1549 		last -= !right_is_free;  in mb_free_blocks()
1550 		e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1;  in mb_free_blocks()
1557 	mb_set_largest_free_order(sb, e4b->bd_info);  in mb_free_blocks()
1568 	assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));  in mb_find_extent()
1575 		ex->fe_len = 0;  in mb_find_extent()
1576 		ex->fe_start = 0;  in mb_find_extent()
1577 		ex->fe_group = 0;  in mb_find_extent()
1585 	ex->fe_len = 1 << order;  in mb_find_extent()
1586 	ex->fe_start = block << order;  in mb_find_extent()
1587 	ex->fe_group = e4b->bd_group;  in mb_find_extent()
1590 	next = next - ex->fe_start;  in mb_find_extent()
1591 	ex->fe_len -= next;  in mb_find_extent()
1592 	ex->fe_start += next;  in mb_find_extent()
1594 	while (needed > ex->fe_len &&  in mb_find_extent()
1601 		if (mb_test_bit(next, e4b->bd_bitmap))  in mb_find_extent()
1607 		ex->fe_len += 1 << order;  in mb_find_extent()
1610 	if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) {  in mb_find_extent()
1613 		ext4_grp_locked_error(e4b->bd_sb, e4b->bd_group, 0, 0,  in mb_find_extent()
1616 			block, order, needed, ex->fe_group, ex->fe_start,  in mb_find_extent()
1617 			ex->fe_len, ex->fe_logical);  in mb_find_extent()
1618 		ex->fe_len = 0;  in mb_find_extent()
1619 		ex->fe_start = 0;  in mb_find_extent()
1620 		ex->fe_group = 0;  in mb_find_extent()
1622 	return ex->fe_len;  in mb_find_extent()
1631 	int start = ex->fe_start;  in mb_mark_used()
1632 	int len = ex->fe_len;  in mb_mark_used()
1637 	BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3));  in mb_mark_used()
1638 	BUG_ON(e4b->bd_group != ex->fe_group);  in mb_mark_used()
1639 	assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));  in mb_mark_used()
1644 	e4b->bd_info->bb_free -= len;  in mb_mark_used()
1645 	if (e4b->bd_info->bb_first_free == start)  in mb_mark_used()
1646 		e4b->bd_info->bb_first_free += len;  in mb_mark_used()
1650 		mlen = !mb_test_bit(start - 1, e4b->bd_bitmap);  in mb_mark_used()
1651 	if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0])  in mb_mark_used()
1652 		max = !mb_test_bit(start + len, e4b->bd_bitmap);  in mb_mark_used()
1654 		e4b->bd_info->bb_fragments++;  in mb_mark_used()
1656 		e4b->bd_info->bb_fragments--;  in mb_mark_used()
1668 			e4b->bd_info->bb_counters[ord]--;  in mb_mark_used()
1670 			len -= mlen;  in mb_mark_used()
1683 		e4b->bd_info->bb_counters[ord]--;  in mb_mark_used()
1685 		ord--;  in mb_mark_used()
1690 		e4b->bd_info->bb_counters[ord]++;  in mb_mark_used()
1691 		e4b->bd_info->bb_counters[ord]++;  in mb_mark_used()
1693 	mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info);  in mb_mark_used()
1695 	ext4_set_bits(e4b->bd_bitmap, ex->fe_start, len0);  in mb_mark_used()
1707 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_best_found()
1710 	BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);  in ext4_mb_use_best_found()
1711 	BUG_ON(ac->ac_status == AC_STATUS_FOUND);  in ext4_mb_use_best_found()
1713 	ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len);  in ext4_mb_use_best_found()
1714 	ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical;  in ext4_mb_use_best_found()
1715 	ret = mb_mark_used(e4b, &ac->ac_b_ex);  in ext4_mb_use_best_found()
1719 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_use_best_found()
1721 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_best_found()
1722 	ac->ac_tail = ret & 0xffff;  in ext4_mb_use_best_found()
1723 	ac->ac_buddy = ret >> 16;  in ext4_mb_use_best_found()
1732 	ac->ac_bitmap_page = e4b->bd_bitmap_page;  in ext4_mb_use_best_found()
1733 	get_page(ac->ac_bitmap_page);  in ext4_mb_use_best_found()
1734 	ac->ac_buddy_page = e4b->bd_buddy_page;  in ext4_mb_use_best_found()
1735 	get_page(ac->ac_buddy_page);  in ext4_mb_use_best_found()
1737 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {  in ext4_mb_use_best_found()
1738 		spin_lock(&sbi->s_md_lock);  in ext4_mb_use_best_found()
1739 		sbi->s_mb_last_group = ac->ac_f_ex.fe_group;  in ext4_mb_use_best_found()
1740 		sbi->s_mb_last_start = ac->ac_f_ex.fe_start;  in ext4_mb_use_best_found()
1741 		spin_unlock(&sbi->s_md_lock);  in ext4_mb_use_best_found()
1748 	if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len)  in ext4_mb_use_best_found()
1757 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_check_limits()
1758 	struct ext4_free_extent *bex = &ac->ac_b_ex;  in ext4_mb_check_limits()
1759 	struct ext4_free_extent *gex = &ac->ac_g_ex;  in ext4_mb_check_limits()
1763 	if (ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_check_limits()
1768 	if (ac->ac_found > sbi->s_mb_max_to_scan &&  in ext4_mb_check_limits()
1769 			!(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_check_limits()
1770 		ac->ac_status = AC_STATUS_BREAK;  in ext4_mb_check_limits()
1777 	if (bex->fe_len < gex->fe_len)  in ext4_mb_check_limits()
1780 	if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan)  in ext4_mb_check_limits()
1781 			&& bex->fe_group == e4b->bd_group) {  in ext4_mb_check_limits()
1782 		/* recheck chunk's availability - we don't know  in ext4_mb_check_limits()
1783 		 * when it was found (within this lock-unlock  in ext4_mb_check_limits()
1785 		max = mb_find_extent(e4b, bex->fe_start, gex->fe_len, &ex);  in ext4_mb_check_limits()
1786 		if (max >= gex->fe_len) {  in ext4_mb_check_limits()
1807 	struct ext4_free_extent *bex = &ac->ac_b_ex;  in ext4_mb_measure_extent()
1808 	struct ext4_free_extent *gex = &ac->ac_g_ex;  in ext4_mb_measure_extent()
1810 	BUG_ON(ex->fe_len <= 0);  in ext4_mb_measure_extent()
1811 	BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));  in ext4_mb_measure_extent()
1812 	BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));  in ext4_mb_measure_extent()
1813 	BUG_ON(ac->ac_status != AC_STATUS_CONTINUE);  in ext4_mb_measure_extent()
1815 	ac->ac_found++;  in ext4_mb_measure_extent()
1818 	 * The special case - take what you catch first  in ext4_mb_measure_extent()
1820 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_measure_extent()
1829 	if (ex->fe_len == gex->fe_len) {  in ext4_mb_measure_extent()
1838 	if (bex->fe_len == 0) {  in ext4_mb_measure_extent()
1846 	if (bex->fe_len < gex->fe_len) {  in ext4_mb_measure_extent()
1849 		if (ex->fe_len > bex->fe_len)  in ext4_mb_measure_extent()
1851 	} else if (ex->fe_len > gex->fe_len) {  in ext4_mb_measure_extent()
1855 		if (ex->fe_len < bex->fe_len)  in ext4_mb_measure_extent()
1866 	struct ext4_free_extent ex = ac->ac_b_ex;  in ext4_mb_try_best_found()
1872 	err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);  in ext4_mb_try_best_found()
1876 	ext4_lock_group(ac->ac_sb, group);  in ext4_mb_try_best_found()
1877 	if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))  in ext4_mb_try_best_found()
1883 		ac->ac_b_ex = ex;  in ext4_mb_try_best_found()
1888 	ext4_unlock_group(ac->ac_sb, group);  in ext4_mb_try_best_found()
1898 	ext4_group_t group = ac->ac_g_ex.fe_group;  in ext4_mb_find_by_goal()
1901 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_find_by_goal()
1902 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1906 		return -EFSCORRUPTED;  in ext4_mb_find_by_goal()
1907 	if (!(ac->ac_flags & (EXT4_MB_HINT_TRY_GOAL | EXT4_MB_HINT_GOAL_ONLY)))  in ext4_mb_find_by_goal()
1909 	if (grp->bb_free == 0)  in ext4_mb_find_by_goal()
1912 	err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);  in ext4_mb_find_by_goal()
1916 	ext4_lock_group(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1917 	if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))  in ext4_mb_find_by_goal()
1920 	max = mb_find_extent(e4b, ac->ac_g_ex.fe_start,  in ext4_mb_find_by_goal()
1921 			     ac->ac_g_ex.fe_len, &ex);  in ext4_mb_find_by_goal()
1924 	if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) {  in ext4_mb_find_by_goal()
1927 		start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) +  in ext4_mb_find_by_goal()
1929 		/* use do_div to get remainder (would be 64-bit modulo) */  in ext4_mb_find_by_goal()
1930 		if (do_div(start, sbi->s_stripe) == 0) {  in ext4_mb_find_by_goal()
1931 			ac->ac_found++;  in ext4_mb_find_by_goal()
1932 			ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1935 	} else if (max >= ac->ac_g_ex.fe_len) {  in ext4_mb_find_by_goal()
1937 		BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);  in ext4_mb_find_by_goal()
1938 		BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);  in ext4_mb_find_by_goal()
1939 		ac->ac_found++;  in ext4_mb_find_by_goal()
1940 		ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1942 	} else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) {  in ext4_mb_find_by_goal()
1946 		BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);  in ext4_mb_find_by_goal()
1947 		BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);  in ext4_mb_find_by_goal()
1948 		ac->ac_found++;  in ext4_mb_find_by_goal()
1949 		ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1953 	ext4_unlock_group(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1967 	struct super_block *sb = ac->ac_sb;  in ext4_mb_simple_scan_group()
1968 	struct ext4_group_info *grp = e4b->bd_info;  in ext4_mb_simple_scan_group()
1974 	BUG_ON(ac->ac_2order <= 0);  in ext4_mb_simple_scan_group()
1975 	for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) {  in ext4_mb_simple_scan_group()
1976 		if (grp->bb_counters[i] == 0)  in ext4_mb_simple_scan_group()
1984 			ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0,  in ext4_mb_simple_scan_group()
1986 				grp->bb_counters[i], i);  in ext4_mb_simple_scan_group()
1987 			ext4_mark_group_bitmap_corrupted(ac->ac_sb,  in ext4_mb_simple_scan_group()
1988 					 e4b->bd_group,  in ext4_mb_simple_scan_group()
1992 		ac->ac_found++;  in ext4_mb_simple_scan_group()
1994 		ac->ac_b_ex.fe_len = 1 << i;  in ext4_mb_simple_scan_group()
1995 		ac->ac_b_ex.fe_start = k << i;  in ext4_mb_simple_scan_group()
1996 		ac->ac_b_ex.fe_group = e4b->bd_group;  in ext4_mb_simple_scan_group()
2000 		BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len);  in ext4_mb_simple_scan_group()
2002 		if (EXT4_SB(sb)->s_mb_stats)  in ext4_mb_simple_scan_group()
2003 			atomic_inc(&EXT4_SB(sb)->s_bal_2orders);  in ext4_mb_simple_scan_group()
2018 	struct super_block *sb = ac->ac_sb;  in ext4_mb_complex_scan_group()
2019 	void *bitmap = e4b->bd_bitmap;  in ext4_mb_complex_scan_group()
2024 	free = e4b->bd_info->bb_free;  in ext4_mb_complex_scan_group()
2028 	i = e4b->bd_info->bb_first_free;  in ext4_mb_complex_scan_group()
2030 	while (free && ac->ac_status == AC_STATUS_CONTINUE) {  in ext4_mb_complex_scan_group()
2039 			ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,  in ext4_mb_complex_scan_group()
2043 			ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in ext4_mb_complex_scan_group()
2048 		mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex);  in ext4_mb_complex_scan_group()
2052 			ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,  in ext4_mb_complex_scan_group()
2056 			ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in ext4_mb_complex_scan_group()
2069 		free -= ex.fe_len;  in ext4_mb_complex_scan_group()
2077  * we try to find stripe-aligned chunks for stripe-size-multiple requests
2083 	struct super_block *sb = ac->ac_sb;  in ext4_mb_scan_aligned()
2085 	void *bitmap = e4b->bd_bitmap;  in ext4_mb_scan_aligned()
2092 	BUG_ON(sbi->s_stripe == 0);  in ext4_mb_scan_aligned()
2094 	/* find first stripe-aligned block in group */  in ext4_mb_scan_aligned()
2095 	first_group_block = ext4_group_first_block_no(sb, e4b->bd_group);  in ext4_mb_scan_aligned()
2097 	a = first_group_block + sbi->s_stripe - 1;  in ext4_mb_scan_aligned()
2098 	do_div(a, sbi->s_stripe);  in ext4_mb_scan_aligned()
2099 	i = (a * sbi->s_stripe) - first_group_block;  in ext4_mb_scan_aligned()
2103 			max = mb_find_extent(e4b, i, sbi->s_stripe, &ex);  in ext4_mb_scan_aligned()
2104 			if (max >= sbi->s_stripe) {  in ext4_mb_scan_aligned()
2105 				ac->ac_found++;  in ext4_mb_scan_aligned()
2107 				ac->ac_b_ex = ex;  in ext4_mb_scan_aligned()
2112 		i += sbi->s_stripe;  in ext4_mb_scan_aligned()
2125 	int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb));  in ext4_mb_good_group()
2126 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_good_group()
2133 	free = grp->bb_free;  in ext4_mb_good_group()
2137 	fragments = grp->bb_fragments;  in ext4_mb_good_group()
2143 		BUG_ON(ac->ac_2order == 0);  in ext4_mb_good_group()
2146 		if ((ac->ac_flags & EXT4_MB_HINT_DATA) &&  in ext4_mb_good_group()
2151 		if (free < ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2154 		if (ac->ac_2order > ac->ac_sb->s_blocksize_bits+1)  in ext4_mb_good_group()
2157 		if (grp->bb_largest_free_order < ac->ac_2order)  in ext4_mb_good_group()
2162 		if ((free / fragments) >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2166 		if (free >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2186 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_good_group_nolock()
2187 	struct super_block *sb = ac->ac_sb;  in ext4_mb_good_group_nolock()
2189 	bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK;  in ext4_mb_good_group_nolock()
2194 		return -EFSCORRUPTED;  in ext4_mb_good_group_nolock()
2195 	if (sbi->s_mb_stats)  in ext4_mb_good_group_nolock()
2196 		atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]);  in ext4_mb_good_group_nolock()
2199 	free = grp->bb_free;  in ext4_mb_good_group_nolock()
2202 	if (cr <= 2 && free < ac->ac_g_ex.fe_len)  in ext4_mb_good_group_nolock()
2224 		    (!sbi->s_log_groups_per_flex ||  in ext4_mb_good_group_nolock()
2225 		     ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) &&  in ext4_mb_good_group_nolock()
2227 		      (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))))  in ext4_mb_good_group_nolock()
2255 	while (nr-- > 0) {  in ext4_mb_prefetch()
2271 		      (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) {  in ext4_mb_prefetch()
2301 	while (nr-- > 0) {  in ext4_mb_prefetch_fini()
2308 		group--;  in ext4_mb_prefetch_fini()
2314 		      (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) {  in ext4_mb_prefetch_fini()
2325 	int cr = -1;  in ext4_mb_regular_allocator()
2333 	sb = ac->ac_sb;  in ext4_mb_regular_allocator()
2336 	/* non-extent files are limited to low blocks/groups */  in ext4_mb_regular_allocator()
2337 	if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)))  in ext4_mb_regular_allocator()
2338 		ngroups = sbi->s_blockfile_groups;  in ext4_mb_regular_allocator()
2340 	BUG_ON(ac->ac_status == AC_STATUS_FOUND);  in ext4_mb_regular_allocator()
2344 	if (err || ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_regular_allocator()
2347 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_regular_allocator()
2351 	 * ac->ac_2order is set only if the fe_len is a power of 2  in ext4_mb_regular_allocator()
2352 	 * if ac->ac_2order is set we also set criteria to 0 so that we  in ext4_mb_regular_allocator()
2355 	i = fls(ac->ac_g_ex.fe_len);  in ext4_mb_regular_allocator()
2356 	ac->ac_2order = 0;  in ext4_mb_regular_allocator()
2361 	 * We also support searching for power-of-two requests only for  in ext4_mb_regular_allocator()
2364 	if (i >= sbi->s_mb_order2_reqs && i <= sb->s_blocksize_bits + 2) {  in ext4_mb_regular_allocator()
2368 		if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0)  in ext4_mb_regular_allocator()
2369 			ac->ac_2order = array_index_nospec(i - 1,  in ext4_mb_regular_allocator()
2370 							   sb->s_blocksize_bits + 2);  in ext4_mb_regular_allocator()
2374 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {  in ext4_mb_regular_allocator()
2376 		spin_lock(&sbi->s_md_lock);  in ext4_mb_regular_allocator()
2377 		ac->ac_g_ex.fe_group = sbi->s_mb_last_group;  in ext4_mb_regular_allocator()
2378 		ac->ac_g_ex.fe_start = sbi->s_mb_last_start;  in ext4_mb_regular_allocator()
2379 		spin_unlock(&sbi->s_md_lock);  in ext4_mb_regular_allocator()
2382 	/* Let's just scan groups to find more-less suitable blocks */  in ext4_mb_regular_allocator()
2383 	cr = ac->ac_2order ? 0 : 1;  in ext4_mb_regular_allocator()
2389 	for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) {  in ext4_mb_regular_allocator()
2390 		ac->ac_criteria = cr;  in ext4_mb_regular_allocator()
2395 		group = ac->ac_g_ex.fe_group;  in ext4_mb_regular_allocator()
2402 			 * Artificially restricted ngroups for non-extent  in ext4_mb_regular_allocator()
2416 			     prefetch_ios < sbi->s_mb_prefetch_limit)) {  in ext4_mb_regular_allocator()
2419 				nr = sbi->s_mb_prefetch;  in ext4_mb_regular_allocator()
2421 					nr = 1 << sbi->s_log_groups_per_flex;  in ext4_mb_regular_allocator()
2422 					nr -= group & (nr - 1);  in ext4_mb_regular_allocator()
2423 					nr = min(nr, sbi->s_mb_prefetch);  in ext4_mb_regular_allocator()
2456 			ac->ac_groups_scanned++;  in ext4_mb_regular_allocator()
2459 			else if (cr == 1 && sbi->s_stripe &&  in ext4_mb_regular_allocator()
2460 					!(ac->ac_g_ex.fe_len % sbi->s_stripe))  in ext4_mb_regular_allocator()
2468 			if (ac->ac_status != AC_STATUS_CONTINUE)  in ext4_mb_regular_allocator()
2472 		if (sbi->s_mb_stats && i == ngroups)  in ext4_mb_regular_allocator()
2473 			atomic64_inc(&sbi->s_bal_cX_failed[cr]);  in ext4_mb_regular_allocator()
2476 	if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND &&  in ext4_mb_regular_allocator()
2477 	    !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_regular_allocator()
2483 		if (ac->ac_status != AC_STATUS_FOUND) {  in ext4_mb_regular_allocator()
2489 			lost = atomic_inc_return(&sbi->s_mb_lost_chunks);  in ext4_mb_regular_allocator()
2491 				 ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start,  in ext4_mb_regular_allocator()
2492 				 ac->ac_b_ex.fe_len, lost);  in ext4_mb_regular_allocator()
2494 			ac->ac_b_ex.fe_group = 0;  in ext4_mb_regular_allocator()
2495 			ac->ac_b_ex.fe_start = 0;  in ext4_mb_regular_allocator()
2496 			ac->ac_b_ex.fe_len = 0;  in ext4_mb_regular_allocator()
2497 			ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_regular_allocator()
2498 			ac->ac_flags |= EXT4_MB_HINT_FIRST;  in ext4_mb_regular_allocator()
2504 	if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_regular_allocator()
2505 		atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]);  in ext4_mb_regular_allocator()
2507 	if (!err && ac->ac_status != AC_STATUS_FOUND && first_err)  in ext4_mb_regular_allocator()
2511 		 ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status,  in ext4_mb_regular_allocator()
2512 		 ac->ac_flags, cr, err);  in ext4_mb_regular_allocator()
2522 	struct super_block *sb = PDE_DATA(file_inode(seq->file));  in ext4_mb_seq_groups_start()
2533 	struct super_block *sb = PDE_DATA(file_inode(seq->file));  in ext4_mb_seq_groups_next()
2545 	struct super_block *sb = PDE_DATA(file_inode(seq->file));  in ext4_mb_seq_groups_show()
2552 					     sb->s_blocksize_bits,  in ext4_mb_seq_groups_show()
2559 	group--;  in ext4_mb_seq_groups_show()
2575 			seq_printf(seq, "#%-5u: I/O error\n", group);  in ext4_mb_seq_groups_show()
2586 	seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free,  in ext4_mb_seq_groups_show()
2589 		seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ?  in ext4_mb_seq_groups_show()
2609 	struct super_block *sb = (struct super_block *)seq->private;  in ext4_seq_mb_stats_show()
2613 	if (!sbi->s_mb_stats) {  in ext4_seq_mb_stats_show()
2618 	seq_printf(seq, "\treqs: %u\n", atomic_read(&sbi->s_bal_reqs));  in ext4_seq_mb_stats_show()
2619 	seq_printf(seq, "\tsuccess: %u\n", atomic_read(&sbi->s_bal_success));  in ext4_seq_mb_stats_show()
2621 	seq_printf(seq, "\tgroups_scanned: %u\n",  atomic_read(&sbi->s_bal_groups_scanned));  in ext4_seq_mb_stats_show()
2624 	seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[0]));  in ext4_seq_mb_stats_show()
2626 		   atomic64_read(&sbi->s_bal_cX_groups_considered[0]));  in ext4_seq_mb_stats_show()
2628 		   atomic64_read(&sbi->s_bal_cX_failed[0]));  in ext4_seq_mb_stats_show()
2631 	seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[1]));  in ext4_seq_mb_stats_show()
2633 		   atomic64_read(&sbi->s_bal_cX_groups_considered[1]));  in ext4_seq_mb_stats_show()
2635 		   atomic64_read(&sbi->s_bal_cX_failed[1]));  in ext4_seq_mb_stats_show()
2638 	seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[2]));  in ext4_seq_mb_stats_show()
2640 		   atomic64_read(&sbi->s_bal_cX_groups_considered[2]));  in ext4_seq_mb_stats_show()
2642 		   atomic64_read(&sbi->s_bal_cX_failed[2]));  in ext4_seq_mb_stats_show()
2645 	seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[3]));  in ext4_seq_mb_stats_show()
2647 		   atomic64_read(&sbi->s_bal_cX_groups_considered[3]));  in ext4_seq_mb_stats_show()
2649 		   atomic64_read(&sbi->s_bal_cX_failed[3]));  in ext4_seq_mb_stats_show()
2650 	seq_printf(seq, "\textents_scanned: %u\n", atomic_read(&sbi->s_bal_ex_scanned));  in ext4_seq_mb_stats_show()
2651 	seq_printf(seq, "\t\tgoal_hits: %u\n", atomic_read(&sbi->s_bal_goals));  in ext4_seq_mb_stats_show()
2652 	seq_printf(seq, "\t\t2^n_hits: %u\n", atomic_read(&sbi->s_bal_2orders));  in ext4_seq_mb_stats_show()
2653 	seq_printf(seq, "\t\tbreaks: %u\n", atomic_read(&sbi->s_bal_breaks));  in ext4_seq_mb_stats_show()
2654 	seq_printf(seq, "\t\tlost: %u\n", atomic_read(&sbi->s_mb_lost_chunks));  in ext4_seq_mb_stats_show()
2657 		   atomic_read(&sbi->s_mb_buddies_generated),  in ext4_seq_mb_stats_show()
2660 		   atomic64_read(&sbi->s_mb_generation_time));  in ext4_seq_mb_stats_show()
2662 		   atomic_read(&sbi->s_mb_preallocated));  in ext4_seq_mb_stats_show()
2664 		   atomic_read(&sbi->s_mb_discarded));  in ext4_seq_mb_stats_show()
2670 	int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE;  in get_groupinfo_cache()
2678  * Allocate the top-level s_group_info array for the specified number
2687 	size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >>  in ext4_mb_alloc_groupinfo()
2689 	if (size <= sbi->s_group_info_size)  in ext4_mb_alloc_groupinfo()
2692 	size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size);  in ext4_mb_alloc_groupinfo()
2696 		return -ENOMEM;  in ext4_mb_alloc_groupinfo()
2699 	old_groupinfo = rcu_dereference(sbi->s_group_info);  in ext4_mb_alloc_groupinfo()
2702 		       sbi->s_group_info_size * sizeof(*sbi->s_group_info));  in ext4_mb_alloc_groupinfo()
2704 	rcu_assign_pointer(sbi->s_group_info, new_groupinfo);  in ext4_mb_alloc_groupinfo()
2705 	sbi->s_group_info_size = size / sizeof(*sbi->s_group_info);  in ext4_mb_alloc_groupinfo()
2709 		   sbi->s_group_info_size);  in ext4_mb_alloc_groupinfo()
2722 	struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);  in ext4_mb_add_groupinfo()
2739 		rcu_dereference(sbi->s_group_info)[idx] = meta_group_info;  in ext4_mb_add_groupinfo()
2744 	i = group & (EXT4_DESC_PER_BLOCK(sb) - 1);  in ext4_mb_add_groupinfo()
2752 		&(meta_group_info[i]->bb_state));  in ext4_mb_add_groupinfo()
2759 	    (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {  in ext4_mb_add_groupinfo()
2760 		meta_group_info[i]->bb_free =  in ext4_mb_add_groupinfo()
2763 		meta_group_info[i]->bb_free =  in ext4_mb_add_groupinfo()
2767 	INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list);  in ext4_mb_add_groupinfo()
2768 	init_rwsem(&meta_group_info[i]->alloc_sem);  in ext4_mb_add_groupinfo()
2769 	meta_group_info[i]->bb_free_root = RB_ROOT;  in ext4_mb_add_groupinfo()
2770 	meta_group_info[i]->bb_largest_free_order = -1;  /* uninit */  in ext4_mb_add_groupinfo()
2781 		group_info = rcu_dereference(sbi->s_group_info);  in ext4_mb_add_groupinfo()
2787 	return -ENOMEM;  in ext4_mb_add_groupinfo()
2804 	sbi->s_buddy_cache = new_inode(sb);  in ext4_mb_init_backend()
2805 	if (sbi->s_buddy_cache == NULL) {  in ext4_mb_init_backend()
2809 	/* To avoid potentially colliding with an valid on-disk inode number,  in ext4_mb_init_backend()
2813 	sbi->s_buddy_cache->i_ino = EXT4_BAD_INO;  in ext4_mb_init_backend()
2814 	EXT4_I(sbi->s_buddy_cache)->i_disksize = 0;  in ext4_mb_init_backend()
2831 		if (sbi->s_es->s_log_groups_per_flex >= 32) {  in ext4_mb_init_backend()
2835 		sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex,  in ext4_mb_init_backend()
2836 			BLK_MAX_SEGMENT_SIZE >> (sb->s_blocksize_bits - 9));  in ext4_mb_init_backend()
2837 		sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */  in ext4_mb_init_backend()
2839 		sbi->s_mb_prefetch = 32;  in ext4_mb_init_backend()
2841 	if (sbi->s_mb_prefetch > ext4_get_groups_count(sb))  in ext4_mb_init_backend()
2842 		sbi->s_mb_prefetch = ext4_get_groups_count(sb);  in ext4_mb_init_backend()
2844 	 * given cr=0 is an CPU-related optimization we shouldn't try to  in ext4_mb_init_backend()
2850 	sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4;  in ext4_mb_init_backend()
2851 	if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb))  in ext4_mb_init_backend()
2852 		sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb);  in ext4_mb_init_backend()
2857 	cachep = get_groupinfo_cache(sb->s_blocksize_bits);  in ext4_mb_init_backend()
2858 	while (i-- > 0) {  in ext4_mb_init_backend()
2864 	i = sbi->s_group_info_size;  in ext4_mb_init_backend()
2866 	group_info = rcu_dereference(sbi->s_group_info);  in ext4_mb_init_backend()
2867 	while (i-- > 0)  in ext4_mb_init_backend()
2870 	iput(sbi->s_buddy_cache);  in ext4_mb_init_backend()
2873 	kvfree(rcu_dereference(sbi->s_group_info));  in ext4_mb_init_backend()
2875 	return -ENOMEM;  in ext4_mb_init_backend()
2893 	int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE;  in ext4_groupinfo_create_slab()
2897 		return -EINVAL;  in ext4_groupinfo_create_slab()
2920 		       "EXT4-fs: no memory for groupinfo slab cache\n");  in ext4_groupinfo_create_slab()
2921 		return -ENOMEM;  in ext4_groupinfo_create_slab()
2935 	i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_offsets);  in ext4_mb_init()
2937 	sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL);  in ext4_mb_init()
2938 	if (sbi->s_mb_offsets == NULL) {  in ext4_mb_init()
2939 		ret = -ENOMEM;  in ext4_mb_init()
2943 	i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_maxs);  in ext4_mb_init()
2944 	sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL);  in ext4_mb_init()
2945 	if (sbi->s_mb_maxs == NULL) {  in ext4_mb_init()
2946 		ret = -ENOMEM;  in ext4_mb_init()
2950 	ret = ext4_groupinfo_create_slab(sb->s_blocksize);  in ext4_mb_init()
2955 	sbi->s_mb_maxs[0] = sb->s_blocksize << 3;  in ext4_mb_init()
2956 	sbi->s_mb_offsets[0] = 0;  in ext4_mb_init()
2960 	offset_incr = 1 << (sb->s_blocksize_bits - 1);  in ext4_mb_init()
2961 	max = sb->s_blocksize << 2;  in ext4_mb_init()
2963 		sbi->s_mb_offsets[i] = offset;  in ext4_mb_init()
2964 		sbi->s_mb_maxs[i] = max;  in ext4_mb_init()
2969 	} while (i <= sb->s_blocksize_bits + 1);  in ext4_mb_init()
2971 	spin_lock_init(&sbi->s_md_lock);  in ext4_mb_init()
2972 	sbi->s_mb_free_pending = 0;  in ext4_mb_init()
2973 	INIT_LIST_HEAD(&sbi->s_freed_data_list);  in ext4_mb_init()
2975 	sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN;  in ext4_mb_init()
2976 	sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN;  in ext4_mb_init()
2977 	sbi->s_mb_stats = MB_DEFAULT_STATS;  in ext4_mb_init()
2978 	sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD;  in ext4_mb_init()
2979 	sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS;  in ext4_mb_init()
2980 	sbi->s_mb_max_inode_prealloc = MB_DEFAULT_MAX_INODE_PREALLOC;  in ext4_mb_init()
2993 	sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >>  in ext4_mb_init()
2994 				       sbi->s_cluster_bits, 32);  in ext4_mb_init()
3003 	if (sbi->s_stripe > 1) {  in ext4_mb_init()
3004 		sbi->s_mb_group_prealloc = roundup(  in ext4_mb_init()
3005 			sbi->s_mb_group_prealloc, sbi->s_stripe);  in ext4_mb_init()
3008 	sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group);  in ext4_mb_init()
3009 	if (sbi->s_locality_groups == NULL) {  in ext4_mb_init()
3010 		ret = -ENOMEM;  in ext4_mb_init()
3015 		lg = per_cpu_ptr(sbi->s_locality_groups, i);  in ext4_mb_init()
3016 		mutex_init(&lg->lg_mutex);  in ext4_mb_init()
3018 			INIT_LIST_HEAD(&lg->lg_prealloc_list[j]);  in ext4_mb_init()
3019 		spin_lock_init(&lg->lg_prealloc_lock);  in ext4_mb_init()
3030 	free_percpu(sbi->s_locality_groups);  in ext4_mb_init()
3031 	sbi->s_locality_groups = NULL;  in ext4_mb_init()
3033 	kfree(sbi->s_mb_offsets);  in ext4_mb_init()
3034 	sbi->s_mb_offsets = NULL;  in ext4_mb_init()
3035 	kfree(sbi->s_mb_maxs);  in ext4_mb_init()
3036 	sbi->s_mb_maxs = NULL;  in ext4_mb_init()
3047 	list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) {  in ext4_mb_cleanup_pa()
3049 		list_del(&pa->pa_group_list);  in ext4_mb_cleanup_pa()
3063 	struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);  in ext4_mb_release()
3066 	if (sbi->s_group_info) {  in ext4_mb_release()
3082 				EXT4_DESC_PER_BLOCK(sb) - 1) >>  in ext4_mb_release()
3085 		group_info = rcu_dereference(sbi->s_group_info);  in ext4_mb_release()
3091 	kfree(sbi->s_mb_offsets);  in ext4_mb_release()
3092 	kfree(sbi->s_mb_maxs);  in ext4_mb_release()
3093 	iput(sbi->s_buddy_cache);  in ext4_mb_release()
3094 	if (sbi->s_mb_stats) {  in ext4_mb_release()
3097 				atomic_read(&sbi->s_bal_allocated),  in ext4_mb_release()
3098 				atomic_read(&sbi->s_bal_reqs),  in ext4_mb_release()
3099 				atomic_read(&sbi->s_bal_success));  in ext4_mb_release()
3103 				atomic_read(&sbi->s_bal_ex_scanned),  in ext4_mb_release()
3104 				atomic_read(&sbi->s_bal_groups_scanned),  in ext4_mb_release()
3105 				atomic_read(&sbi->s_bal_goals),  in ext4_mb_release()
3106 				atomic_read(&sbi->s_bal_2orders),  in ext4_mb_release()
3107 				atomic_read(&sbi->s_bal_breaks),  in ext4_mb_release()
3108 				atomic_read(&sbi->s_mb_lost_chunks));  in ext4_mb_release()
3111 				atomic_read(&sbi->s_mb_buddies_generated),  in ext4_mb_release()
3112 				atomic64_read(&sbi->s_mb_generation_time));  in ext4_mb_release()
3115 				atomic_read(&sbi->s_mb_preallocated),  in ext4_mb_release()
3116 				atomic_read(&sbi->s_mb_discarded));  in ext4_mb_release()
3119 	free_percpu(sbi->s_locality_groups);  in ext4_mb_release()
3136 		return __blkdev_issue_discard(sb->s_bdev,  in ext4_issue_discard()
3137 			(sector_t)discard_block << (sb->s_blocksize_bits - 9),  in ext4_issue_discard()
3138 			(sector_t)count << (sb->s_blocksize_bits - 9),  in ext4_issue_discard()
3152 		 entry->efd_count, entry->efd_group, entry);  in ext4_free_data_in_buddy()
3154 	err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b);  in ext4_free_data_in_buddy()
3158 	spin_lock(&EXT4_SB(sb)->s_md_lock);  in ext4_free_data_in_buddy()
3159 	EXT4_SB(sb)->s_mb_free_pending -= entry->efd_count;  in ext4_free_data_in_buddy()
3160 	spin_unlock(&EXT4_SB(sb)->s_md_lock);  in ext4_free_data_in_buddy()
3164 	count += entry->efd_count;  in ext4_free_data_in_buddy()
3166 	ext4_lock_group(sb, entry->efd_group);  in ext4_free_data_in_buddy()
3168 	rb_erase(&entry->efd_node, &(db->bb_free_root));  in ext4_free_data_in_buddy()
3169 	mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count);  in ext4_free_data_in_buddy()
3174 	 * If the volume is mounted with -o discard, online discard  in ext4_free_data_in_buddy()
3180 	if (!db->bb_free_root.rb_node) {  in ext4_free_data_in_buddy()
3187 	ext4_unlock_group(sb, entry->efd_group);  in ext4_free_data_in_buddy()
3210 	spin_lock(&sbi->s_md_lock);  in ext4_process_freed_data()
3211 	list_for_each_entry(entry, &sbi->s_freed_data_list, efd_list) {  in ext4_process_freed_data()
3212 		if (entry->efd_tid != commit_tid)  in ext4_process_freed_data()
3214 		cut_pos = &entry->efd_list;  in ext4_process_freed_data()
3217 		list_cut_position(&freed_data_list, &sbi->s_freed_data_list,  in ext4_process_freed_data()
3219 	spin_unlock(&sbi->s_md_lock);  in ext4_process_freed_data()
3223 			err = ext4_issue_discard(sb, entry->efd_group,  in ext4_process_freed_data()
3224 						 entry->efd_start_cluster,  in ext4_process_freed_data()
3225 						 entry->efd_count,  in ext4_process_freed_data()
3227 			if (err && err != -EOPNOTSUPP) {  in ext4_process_freed_data()
3230 					 " with %d", entry->efd_group,  in ext4_process_freed_data()
3231 					 entry->efd_start_cluster,  in ext4_process_freed_data()
3232 					 entry->efd_count, err);  in ext4_process_freed_data()
3233 			} else if (err == -EOPNOTSUPP)  in ext4_process_freed_data()
3271 	return -ENOMEM;  in ext4_init_mballoc()
3289  * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
3304 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_mark_diskspace_used()
3305 	BUG_ON(ac->ac_b_ex.fe_len <= 0);  in ext4_mb_mark_diskspace_used()
3307 	sb = ac->ac_sb;  in ext4_mb_mark_diskspace_used()
3310 	bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3322 	err = -EIO;  in ext4_mb_mark_diskspace_used()
3323 	gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh);  in ext4_mb_mark_diskspace_used()
3327 	ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group,  in ext4_mb_mark_diskspace_used()
3335 	block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_mark_diskspace_used()
3337 	len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3338 	if (!ext4_inode_block_valid(ac->ac_inode, block, len)) {  in ext4_mb_mark_diskspace_used()
3339 		ext4_error(sb, "Allocating blocks %llu-%llu which overlap "  in ext4_mb_mark_diskspace_used()
3345 		ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3346 		ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,  in ext4_mb_mark_diskspace_used()
3347 			      ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3348 		ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3351 			err = -EFSCORRUPTED;  in ext4_mb_mark_diskspace_used()
3355 	ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3359 		for (i = 0; i < ac->ac_b_ex.fe_len; i++) {  in ext4_mb_mark_diskspace_used()
3360 			BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i,  in ext4_mb_mark_diskspace_used()
3361 						bitmap_bh->b_data));  in ext4_mb_mark_diskspace_used()
3365 	ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,  in ext4_mb_mark_diskspace_used()
3366 		      ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3368 	    (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {  in ext4_mb_mark_diskspace_used()
3369 		gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);  in ext4_mb_mark_diskspace_used()
3372 						ac->ac_b_ex.fe_group, gdp));  in ext4_mb_mark_diskspace_used()
3374 	len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len;  in ext4_mb_mark_diskspace_used()
3376 	ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh);  in ext4_mb_mark_diskspace_used()
3377 	ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp);  in ext4_mb_mark_diskspace_used()
3379 	ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3380 	percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3384 	if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED))  in ext4_mb_mark_diskspace_used()
3386 		percpu_counter_sub(&sbi->s_dirtyclusters_counter,  in ext4_mb_mark_diskspace_used()
3389 	if (sbi->s_log_groups_per_flex) {  in ext4_mb_mark_diskspace_used()
3391 							  ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3392 		atomic64_sub(ac->ac_b_ex.fe_len,  in ext4_mb_mark_diskspace_used()
3394 						  flex_group)->free_clusters);  in ext4_mb_mark_diskspace_used()
3436 			EXT4_BLOCKS_PER_GROUP(sb) - EXT4_C2B(sbi, blkoff));  in ext4_mb_mark_bb()
3446 		err = -EIO;  in ext4_mb_mark_bb()
3454 			if (!mb_test_bit(blkoff + i, bitmap_bh->b_data) ==  in ext4_mb_mark_bb()
3458 		clen_changed = clen - already;  in ext4_mb_mark_bb()
3460 			ext4_set_bits(bitmap_bh->b_data, blkoff, clen);  in ext4_mb_mark_bb()
3462 			mb_test_and_clear_bits(bitmap_bh->b_data, blkoff, clen);  in ext4_mb_mark_bb()
3464 		    (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {  in ext4_mb_mark_bb()
3465 			gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);  in ext4_mb_mark_bb()
3470 			clen = ext4_free_group_clusters(sb, gdp) - clen_changed;  in ext4_mb_mark_bb()
3480 		if (sbi->s_log_groups_per_flex) {  in ext4_mb_mark_bb()
3486 				atomic64_sub(clen_changed, &fg->free_clusters);  in ext4_mb_mark_bb()
3488 				atomic64_add(clen_changed, &fg->free_clusters);  in ext4_mb_mark_bb()
3502 		len -= thisgrp_len;  in ext4_mb_mark_bb()
3512  * here we normalize request for locality group
3522 	struct super_block *sb = ac->ac_sb;  in ext4_mb_normalize_group_request()
3523 	struct ext4_locality_group *lg = ac->ac_lg;  in ext4_mb_normalize_group_request()
3526 	ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc;  in ext4_mb_normalize_group_request()
3527 	mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len);  in ext4_mb_normalize_group_request()
3538 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_normalize_request()
3539 	struct ext4_super_block *es = sbi->s_es;  in ext4_mb_normalize_request()
3544 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_normalize_request()
3549 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_normalize_request()
3553 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_normalize_request()
3558 	if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC)  in ext4_mb_normalize_request()
3561 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) {  in ext4_mb_normalize_request()
3566 	bsbits = ac->ac_sb->s_blocksize_bits;  in ext4_mb_normalize_request()
3570 	size = extent_logical_end(sbi, &ac->ac_o_ex);  in ext4_mb_normalize_request()
3572 	if (size < i_size_read(ac->ac_inode))  in ext4_mb_normalize_request()
3573 		size = i_size_read(ac->ac_inode);  in ext4_mb_normalize_request()
3600 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3601 						(21 - bsbits)) << 21;  in ext4_mb_normalize_request()
3604 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3605 							(22 - bsbits)) << 22;  in ext4_mb_normalize_request()
3607 	} else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len,  in ext4_mb_normalize_request()
3609 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3610 							(23 - bsbits)) << 23;  in ext4_mb_normalize_request()
3613 		start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits;  in ext4_mb_normalize_request()
3614 		size	  = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3615 					      ac->ac_o_ex.fe_len) << bsbits;  in ext4_mb_normalize_request()
3626 	start = max(start, rounddown(ac->ac_o_ex.fe_logical,  in ext4_mb_normalize_request()
3627 			(ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb)));  in ext4_mb_normalize_request()
3631 		size = EXT_MAX_BLOCKS - start;  in ext4_mb_normalize_request()
3634 	if (ar->pleft && start <= ar->lleft) {  in ext4_mb_normalize_request()
3635 		size -= ar->lleft + 1 - start;  in ext4_mb_normalize_request()
3636 		start = ar->lleft + 1;  in ext4_mb_normalize_request()
3638 	if (ar->pright && start + size - 1 >= ar->lright)  in ext4_mb_normalize_request()
3639 		size -= start + size - ar->lright;  in ext4_mb_normalize_request()
3645 	if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb))  in ext4_mb_normalize_request()
3646 		size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb);  in ext4_mb_normalize_request()
3652 	list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {  in ext4_mb_normalize_request()
3655 		if (pa->pa_deleted)  in ext4_mb_normalize_request()
3657 		spin_lock(&pa->pa_lock);  in ext4_mb_normalize_request()
3658 		if (pa->pa_deleted) {  in ext4_mb_normalize_request()
3659 			spin_unlock(&pa->pa_lock);  in ext4_mb_normalize_request()
3663 		pa_end = pa_logical_end(EXT4_SB(ac->ac_sb), pa);  in ext4_mb_normalize_request()
3666 		BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end ||  in ext4_mb_normalize_request()
3667 			ac->ac_o_ex.fe_logical < pa->pa_lstart));  in ext4_mb_normalize_request()
3670 		if (pa->pa_lstart >= end || pa_end <= start) {  in ext4_mb_normalize_request()
3671 			spin_unlock(&pa->pa_lock);  in ext4_mb_normalize_request()
3674 		BUG_ON(pa->pa_lstart <= start && pa_end >= end);  in ext4_mb_normalize_request()
3677 		if (pa_end <= ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3680 		} else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3681 			BUG_ON(pa->pa_lstart > end);  in ext4_mb_normalize_request()
3682 			end = pa->pa_lstart;  in ext4_mb_normalize_request()
3684 		spin_unlock(&pa->pa_lock);  in ext4_mb_normalize_request()
3687 	size = end - start;  in ext4_mb_normalize_request()
3691 	list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {  in ext4_mb_normalize_request()
3694 		spin_lock(&pa->pa_lock);  in ext4_mb_normalize_request()
3695 		if (pa->pa_deleted == 0) {  in ext4_mb_normalize_request()
3696 			pa_end = pa_logical_end(EXT4_SB(ac->ac_sb), pa);  in ext4_mb_normalize_request()
3697 			BUG_ON(!(start >= pa_end || end <= pa->pa_lstart));  in ext4_mb_normalize_request()
3699 		spin_unlock(&pa->pa_lock);  in ext4_mb_normalize_request()
3703 	if (start + size <= ac->ac_o_ex.fe_logical &&  in ext4_mb_normalize_request()
3704 			start > ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3705 		ext4_msg(ac->ac_sb, KERN_ERR,  in ext4_mb_normalize_request()
3708 			 (unsigned long) ac->ac_o_ex.fe_logical);  in ext4_mb_normalize_request()
3711 	BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb));  in ext4_mb_normalize_request()
3717 	ac->ac_g_ex.fe_logical = start;  in ext4_mb_normalize_request()
3718 	ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size);  in ext4_mb_normalize_request()
3721 	if (ar->pright && (ar->lright == (start + size)) &&  in ext4_mb_normalize_request()
3722 	    ar->pright >= size &&  in ext4_mb_normalize_request()
3723 	    ar->pright - size >= le32_to_cpu(es->s_first_data_block)) {  in ext4_mb_normalize_request()
3725 		ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size,  in ext4_mb_normalize_request()
3726 						&ac->ac_g_ex.fe_group,  in ext4_mb_normalize_request()
3727 						&ac->ac_g_ex.fe_start);  in ext4_mb_normalize_request()
3728 		ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;  in ext4_mb_normalize_request()
3730 	if (ar->pleft && (ar->lleft + 1 == start) &&  in ext4_mb_normalize_request()
3731 	    ar->pleft + 1 < ext4_blocks_count(es)) {  in ext4_mb_normalize_request()
3733 		ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1,  in ext4_mb_normalize_request()
3734 						&ac->ac_g_ex.fe_group,  in ext4_mb_normalize_request()
3735 						&ac->ac_g_ex.fe_start);  in ext4_mb_normalize_request()
3736 		ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;  in ext4_mb_normalize_request()
3739 	mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size,  in ext4_mb_normalize_request()
3745 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_collect_stats()
3747 	if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) {  in ext4_mb_collect_stats()
3748 		atomic_inc(&sbi->s_bal_reqs);  in ext4_mb_collect_stats()
3749 		atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);  in ext4_mb_collect_stats()
3750 		if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len)  in ext4_mb_collect_stats()
3751 			atomic_inc(&sbi->s_bal_success);  in ext4_mb_collect_stats()
3752 		atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);  in ext4_mb_collect_stats()
3753 		atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned);  in ext4_mb_collect_stats()
3754 		if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&  in ext4_mb_collect_stats()
3755 				ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)  in ext4_mb_collect_stats()
3756 			atomic_inc(&sbi->s_bal_goals);  in ext4_mb_collect_stats()
3757 		if (ac->ac_found > sbi->s_mb_max_to_scan)  in ext4_mb_collect_stats()
3758 			atomic_inc(&sbi->s_bal_breaks);  in ext4_mb_collect_stats()
3761 	if (ac->ac_op == EXT4_MB_HISTORY_ALLOC)  in ext4_mb_collect_stats()
3771  * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
3775 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_discard_allocated_blocks()
3780 		if (ac->ac_f_ex.fe_len == 0)  in ext4_discard_allocated_blocks()
3782 		err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b);  in ext4_discard_allocated_blocks()
3792 		ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group);  in ext4_discard_allocated_blocks()
3793 		mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start,  in ext4_discard_allocated_blocks()
3794 			       ac->ac_f_ex.fe_len);  in ext4_discard_allocated_blocks()
3795 		ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group);  in ext4_discard_allocated_blocks()
3799 	if (pa->pa_type == MB_INODE_PA)  in ext4_discard_allocated_blocks()
3800 		pa->pa_free += ac->ac_b_ex.fe_len;  in ext4_discard_allocated_blocks()
3809 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_inode_pa()
3815 	start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart);  in ext4_mb_use_inode_pa()
3816 	end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len),  in ext4_mb_use_inode_pa()
3817 		  start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len));  in ext4_mb_use_inode_pa()
3818 	len = EXT4_NUM_B2C(sbi, end - start);  in ext4_mb_use_inode_pa()
3819 	ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group,  in ext4_mb_use_inode_pa()
3820 					&ac->ac_b_ex.fe_start);  in ext4_mb_use_inode_pa()
3821 	ac->ac_b_ex.fe_len = len;  in ext4_mb_use_inode_pa()
3822 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_inode_pa()
3823 	ac->ac_pa = pa;  in ext4_mb_use_inode_pa()
3825 	BUG_ON(start < pa->pa_pstart);  in ext4_mb_use_inode_pa()
3826 	BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len));  in ext4_mb_use_inode_pa()
3827 	BUG_ON(pa->pa_free < len);  in ext4_mb_use_inode_pa()
3828 	BUG_ON(ac->ac_b_ex.fe_len <= 0);  in ext4_mb_use_inode_pa()
3829 	pa->pa_free -= len;  in ext4_mb_use_inode_pa()
3831 	mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa);  in ext4_mb_use_inode_pa()
3835  * use blocks preallocated to locality group
3840 	unsigned int len = ac->ac_o_ex.fe_len;  in ext4_mb_use_group_pa()
3842 	ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart,  in ext4_mb_use_group_pa()
3843 					&ac->ac_b_ex.fe_group,  in ext4_mb_use_group_pa()
3844 					&ac->ac_b_ex.fe_start);  in ext4_mb_use_group_pa()
3845 	ac->ac_b_ex.fe_len = len;  in ext4_mb_use_group_pa()
3846 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_group_pa()
3847 	ac->ac_pa = pa;  in ext4_mb_use_group_pa()
3852 	 * in on-disk bitmap -- see ext4_mb_release_context()  in ext4_mb_use_group_pa()
3855 	mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n",  in ext4_mb_use_group_pa()
3856 		 pa->pa_lstart-len, len, pa);  in ext4_mb_use_group_pa()
3873 		atomic_inc(&pa->pa_count);  in ext4_mb_check_group_pa()
3876 	cur_distance = abs(goal_block - cpa->pa_pstart);  in ext4_mb_check_group_pa()
3877 	new_distance = abs(goal_block - pa->pa_pstart);  in ext4_mb_check_group_pa()
3883 	atomic_dec(&cpa->pa_count);  in ext4_mb_check_group_pa()
3884 	atomic_inc(&pa->pa_count);  in ext4_mb_check_group_pa()
3894 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_preallocated()
3896 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_use_preallocated()
3902 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_use_preallocated()
3905 	/* first, try per-file preallocation */  in ext4_mb_use_preallocated()
3907 	list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {  in ext4_mb_use_preallocated()
3911 		if (ac->ac_o_ex.fe_logical < pa->pa_lstart ||  in ext4_mb_use_preallocated()
3912 		    ac->ac_o_ex.fe_logical >= pa_logical_end(sbi, pa))  in ext4_mb_use_preallocated()
3915 		/* non-extent files can't have physical blocks past 2^32 */  in ext4_mb_use_preallocated()
3916 		if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) &&  in ext4_mb_use_preallocated()
3917 		    (pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len) >  in ext4_mb_use_preallocated()
3922 		spin_lock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3923 		if (pa->pa_deleted == 0 && pa->pa_free) {  in ext4_mb_use_preallocated()
3924 			atomic_inc(&pa->pa_count);  in ext4_mb_use_preallocated()
3926 			spin_unlock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3927 			ac->ac_criteria = 10;  in ext4_mb_use_preallocated()
3931 		spin_unlock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3936 	if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC))  in ext4_mb_use_preallocated()
3939 	/* inode may have no locality group for some reason */  in ext4_mb_use_preallocated()
3940 	lg = ac->ac_lg;  in ext4_mb_use_preallocated()
3943 	order  = fls(ac->ac_o_ex.fe_len) - 1;  in ext4_mb_use_preallocated()
3944 	if (order > PREALLOC_TB_SIZE - 1)  in ext4_mb_use_preallocated()
3946 		order = PREALLOC_TB_SIZE - 1;  in ext4_mb_use_preallocated()
3948 	goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex);  in ext4_mb_use_preallocated()
3955 		list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[i],  in ext4_mb_use_preallocated()
3957 			spin_lock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3958 			if (pa->pa_deleted == 0 &&  in ext4_mb_use_preallocated()
3959 					pa->pa_free >= ac->ac_o_ex.fe_len) {  in ext4_mb_use_preallocated()
3964 			spin_unlock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3970 		ac->ac_criteria = 20;  in ext4_mb_use_preallocated()
3978  * but not yet committed and marks them used in in-core bitmap.
3992 	n = rb_first(&(grp->bb_free_root));  in ext4_mb_generate_from_freelist()
3996 		ext4_set_bits(bitmap, entry->efd_start_cluster, entry->efd_count);  in ext4_mb_generate_from_freelist()
4004  * used in in-core bitmap. buddy must be generated from this bitmap
4030 	list_for_each(cur, &grp->bb_prealloc_list) {  in ext4_mb_generate_from_pa()
4032 		spin_lock(&pa->pa_lock);  in ext4_mb_generate_from_pa()
4033 		ext4_get_group_no_and_offset(sb, pa->pa_pstart,  in ext4_mb_generate_from_pa()
4035 		len = pa->pa_len;  in ext4_mb_generate_from_pa()
4036 		spin_unlock(&pa->pa_lock);  in ext4_mb_generate_from_pa()
4051 	if (pa->pa_deleted) {  in ext4_mb_mark_pa_deleted()
4053 			     pa->pa_type, pa->pa_pstart, pa->pa_lstart,  in ext4_mb_mark_pa_deleted()
4054 			     pa->pa_len);  in ext4_mb_mark_pa_deleted()
4058 	pa->pa_deleted = 1;  in ext4_mb_mark_pa_deleted()
4060 	if (pa->pa_type == MB_INODE_PA) {  in ext4_mb_mark_pa_deleted()
4061 		ei = EXT4_I(pa->pa_inode);  in ext4_mb_mark_pa_deleted()
4062 		atomic_dec(&ei->i_prealloc_active);  in ext4_mb_mark_pa_deleted()
4071 	BUG_ON(atomic_read(&pa->pa_count));  in ext4_mb_pa_callback()
4072 	BUG_ON(pa->pa_deleted == 0);  in ext4_mb_pa_callback()
4087 	spin_lock(&pa->pa_lock);  in ext4_mb_put_pa()
4088 	if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) {  in ext4_mb_put_pa()
4089 		spin_unlock(&pa->pa_lock);  in ext4_mb_put_pa()
4093 	if (pa->pa_deleted == 1) {  in ext4_mb_put_pa()
4094 		spin_unlock(&pa->pa_lock);  in ext4_mb_put_pa()
4099 	spin_unlock(&pa->pa_lock);  in ext4_mb_put_pa()
4101 	grp_blk = pa->pa_pstart;  in ext4_mb_put_pa()
4103 	 * If doing group-based preallocation, pa_pstart may be in the  in ext4_mb_put_pa()
4106 	if (pa->pa_type == MB_GROUP_PA)  in ext4_mb_put_pa()
4107 		grp_blk--;  in ext4_mb_put_pa()
4116 	 *  copy on-disk bitmap to buddy  in ext4_mb_put_pa()
4117 	 *  					mark B in on-disk bitmap  in ext4_mb_put_pa()
4126 	list_del(&pa->pa_group_list);  in ext4_mb_put_pa()
4129 	spin_lock(pa->pa_obj_lock);  in ext4_mb_put_pa()
4130 	list_del_rcu(&pa->pa_inode_list);  in ext4_mb_put_pa()
4131 	spin_unlock(pa->pa_obj_lock);  in ext4_mb_put_pa()
4133 	call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_mb_put_pa()
4142 	struct super_block *sb = ac->ac_sb;  in ext4_mb_new_inode_pa()
4149 	BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
4150 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_new_inode_pa()
4151 	BUG_ON(!S_ISREG(ac->ac_inode->i_mode));  in ext4_mb_new_inode_pa()
4152 	BUG_ON(ac->ac_pa == NULL);  in ext4_mb_new_inode_pa()
4154 	pa = ac->ac_pa;  in ext4_mb_new_inode_pa()
4156 	if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) {  in ext4_mb_new_inode_pa()
4158 			.fe_logical = ac->ac_g_ex.fe_logical,  in ext4_mb_new_inode_pa()
4159 			.fe_len = ac->ac_g_ex.fe_len,  in ext4_mb_new_inode_pa()
4166 		BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical);  in ext4_mb_new_inode_pa()
4167 		BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len);  in ext4_mb_new_inode_pa()
4180 		ex.fe_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_inode_pa()
4182 		ex.fe_logical = orig_goal_end - EXT4_C2B(sbi, ex.fe_len);  in ext4_mb_new_inode_pa()
4183 		if (ac->ac_o_ex.fe_logical >= ex.fe_logical)  in ext4_mb_new_inode_pa()
4186 		ex.fe_logical = ac->ac_g_ex.fe_logical;  in ext4_mb_new_inode_pa()
4187 		if (ac->ac_o_ex.fe_logical < extent_logical_end(sbi, &ex))  in ext4_mb_new_inode_pa()
4190 		ex.fe_logical = ac->ac_o_ex.fe_logical;  in ext4_mb_new_inode_pa()
4192 		ac->ac_b_ex.fe_logical = ex.fe_logical;  in ext4_mb_new_inode_pa()
4194 		BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical);  in ext4_mb_new_inode_pa()
4195 		BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
4201 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_new_inode_pa()
4203 	pa->pa_lstart = ac->ac_b_ex.fe_logical;  in ext4_mb_new_inode_pa()
4204 	pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_inode_pa()
4205 	pa->pa_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_inode_pa()
4206 	pa->pa_free = pa->pa_len;  in ext4_mb_new_inode_pa()
4207 	spin_lock_init(&pa->pa_lock);  in ext4_mb_new_inode_pa()
4208 	INIT_LIST_HEAD(&pa->pa_inode_list);  in ext4_mb_new_inode_pa()
4209 	INIT_LIST_HEAD(&pa->pa_group_list);  in ext4_mb_new_inode_pa()
4210 	pa->pa_deleted = 0;  in ext4_mb_new_inode_pa()
4211 	pa->pa_type = MB_INODE_PA;  in ext4_mb_new_inode_pa()
4213 	mb_debug(sb, "new inode pa %p: %llu/%d for %u\n", pa, pa->pa_pstart,  in ext4_mb_new_inode_pa()
4214 		 pa->pa_len, pa->pa_lstart);  in ext4_mb_new_inode_pa()
4218 	atomic_add(pa->pa_free, &sbi->s_mb_preallocated);  in ext4_mb_new_inode_pa()
4220 	ei = EXT4_I(ac->ac_inode);  in ext4_mb_new_inode_pa()
4221 	grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_inode_pa()
4225 	pa->pa_obj_lock = &ei->i_prealloc_lock;  in ext4_mb_new_inode_pa()
4226 	pa->pa_inode = ac->ac_inode;  in ext4_mb_new_inode_pa()
4228 	list_add(&pa->pa_group_list, &grp->bb_prealloc_list);  in ext4_mb_new_inode_pa()
4230 	spin_lock(pa->pa_obj_lock);  in ext4_mb_new_inode_pa()
4231 	list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list);  in ext4_mb_new_inode_pa()
4232 	spin_unlock(pa->pa_obj_lock);  in ext4_mb_new_inode_pa()
4233 	atomic_inc(&ei->i_prealloc_active);  in ext4_mb_new_inode_pa()
4237  * creates new preallocated space for locality group inodes belongs to
4242 	struct super_block *sb = ac->ac_sb;  in ext4_mb_new_group_pa()
4248 	BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);  in ext4_mb_new_group_pa()
4249 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_new_group_pa()
4250 	BUG_ON(!S_ISREG(ac->ac_inode->i_mode));  in ext4_mb_new_group_pa()
4251 	BUG_ON(ac->ac_pa == NULL);  in ext4_mb_new_group_pa()
4253 	pa = ac->ac_pa;  in ext4_mb_new_group_pa()
4257 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_new_group_pa()
4259 	pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_group_pa()
4260 	pa->pa_lstart = pa->pa_pstart;  in ext4_mb_new_group_pa()
4261 	pa->pa_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_group_pa()
4262 	pa->pa_free = pa->pa_len;  in ext4_mb_new_group_pa()
4263 	spin_lock_init(&pa->pa_lock);  in ext4_mb_new_group_pa()
4264 	INIT_LIST_HEAD(&pa->pa_inode_list);  in ext4_mb_new_group_pa()
4265 	INIT_LIST_HEAD(&pa->pa_group_list);  in ext4_mb_new_group_pa()
4266 	pa->pa_deleted = 0;  in ext4_mb_new_group_pa()
4267 	pa->pa_type = MB_GROUP_PA;  in ext4_mb_new_group_pa()
4269 	mb_debug(sb, "new group pa %p: %llu/%d for %u\n", pa, pa->pa_pstart,  in ext4_mb_new_group_pa()
4270 		 pa->pa_len, pa->pa_lstart);  in ext4_mb_new_group_pa()
4274 	atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);  in ext4_mb_new_group_pa()
4276 	grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_group_pa()
4279 	lg = ac->ac_lg;  in ext4_mb_new_group_pa()
4282 	pa->pa_obj_lock = &lg->lg_prealloc_lock;  in ext4_mb_new_group_pa()
4283 	pa->pa_inode = NULL;  in ext4_mb_new_group_pa()
4285 	list_add(&pa->pa_group_list, &grp->bb_prealloc_list);  in ext4_mb_new_group_pa()
4295 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)  in ext4_mb_new_preallocation()
4302  * finds all unused blocks in on-disk bitmap, frees them in
4303  * in-core bitmap and buddy.
4307  * TODO: optimize the case when there are no in-core structures yet
4313 	struct super_block *sb = e4b->bd_sb;  in ext4_mb_release_inode_pa()
4322 	BUG_ON(pa->pa_deleted == 0);  in ext4_mb_release_inode_pa()
4323 	ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);  in ext4_mb_release_inode_pa()
4324 	grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit);  in ext4_mb_release_inode_pa()
4325 	BUG_ON(group != e4b->bd_group && pa->pa_len != 0);  in ext4_mb_release_inode_pa()
4326 	end = bit + pa->pa_len;  in ext4_mb_release_inode_pa()
4329 		bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit);  in ext4_mb_release_inode_pa()
4332 		next = mb_find_next_bit(bitmap_bh->b_data, end, bit);  in ext4_mb_release_inode_pa()
4335 			 (unsigned) next - bit, (unsigned) group);  in ext4_mb_release_inode_pa()
4336 		free += next - bit;  in ext4_mb_release_inode_pa()
4338 		trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit);  in ext4_mb_release_inode_pa()
4341 					       next - bit);  in ext4_mb_release_inode_pa()
4342 		mb_free_blocks(pa->pa_inode, e4b, bit, next - bit);  in ext4_mb_release_inode_pa()
4345 	if (free != pa->pa_free) {  in ext4_mb_release_inode_pa()
4346 		ext4_msg(e4b->bd_sb, KERN_CRIT,  in ext4_mb_release_inode_pa()
4348 			 pa, (unsigned long) pa->pa_lstart,  in ext4_mb_release_inode_pa()
4349 			 (unsigned long) pa->pa_pstart,  in ext4_mb_release_inode_pa()
4350 			 pa->pa_len);  in ext4_mb_release_inode_pa()
4352 					free, pa->pa_free);  in ext4_mb_release_inode_pa()
4358 	atomic_add(free, &sbi->s_mb_discarded);  in ext4_mb_release_inode_pa()
4367 	struct super_block *sb = e4b->bd_sb;  in ext4_mb_release_group_pa()
4372 	BUG_ON(pa->pa_deleted == 0);  in ext4_mb_release_group_pa()
4373 	ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);  in ext4_mb_release_group_pa()
4374 	if (unlikely(group != e4b->bd_group && pa->pa_len != 0)) {  in ext4_mb_release_group_pa()
4376 			     e4b->bd_group, group, pa->pa_pstart);  in ext4_mb_release_group_pa()
4379 	mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len);  in ext4_mb_release_group_pa()
4380 	atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded);  in ext4_mb_release_group_pa()
4381 	trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len);  in ext4_mb_release_group_pa()
4390  * - when do we discard
4392  * - how many do we discard
4410 	if (list_empty(&grp->bb_prealloc_list))  in ext4_mb_discard_group_preallocations()
4416 		ext4_error_err(sb, -err,  in ext4_mb_discard_group_preallocations()
4433 				&grp->bb_prealloc_list, pa_group_list) {  in ext4_mb_discard_group_preallocations()
4434 		spin_lock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
4435 		if (atomic_read(&pa->pa_count)) {  in ext4_mb_discard_group_preallocations()
4436 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
4440 		if (pa->pa_deleted) {  in ext4_mb_discard_group_preallocations()
4441 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
4452 		free += pa->pa_free;  in ext4_mb_discard_group_preallocations()
4454 		spin_unlock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
4456 		list_del(&pa->pa_group_list);  in ext4_mb_discard_group_preallocations()
4457 		list_add(&pa->u.pa_tmp_list, &list);  in ext4_mb_discard_group_preallocations()
4463 		/* remove from object (inode or locality group) */  in ext4_mb_discard_group_preallocations()
4464 		spin_lock(pa->pa_obj_lock);  in ext4_mb_discard_group_preallocations()
4465 		list_del_rcu(&pa->pa_inode_list);  in ext4_mb_discard_group_preallocations()
4466 		spin_unlock(pa->pa_obj_lock);  in ext4_mb_discard_group_preallocations()
4468 		if (pa->pa_type == MB_GROUP_PA)  in ext4_mb_discard_group_preallocations()
4473 		list_del(&pa->u.pa_tmp_list);  in ext4_mb_discard_group_preallocations()
4474 		call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_mb_discard_group_preallocations()
4482 		 free, group, grp->bb_free);  in ext4_mb_discard_group_preallocations()
4487  * releases all non-used preallocated blocks for given inode
4498 	struct super_block *sb = inode->i_sb;  in ext4_discard_preallocations()
4506 	if (!S_ISREG(inode->i_mode)) {  in ext4_discard_preallocations()
4507 		/*BUG_ON(!list_empty(&ei->i_prealloc_list));*/  in ext4_discard_preallocations()
4511 	if (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)  in ext4_discard_preallocations()
4515 		 inode->i_ino);  in ext4_discard_preallocations()
4517 			atomic_read(&ei->i_prealloc_active), needed);  in ext4_discard_preallocations()
4526 	spin_lock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
4527 	while (!list_empty(&ei->i_prealloc_list) && needed) {  in ext4_discard_preallocations()
4528 		pa = list_entry(ei->i_prealloc_list.prev,  in ext4_discard_preallocations()
4530 		BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock);  in ext4_discard_preallocations()
4531 		spin_lock(&pa->pa_lock);  in ext4_discard_preallocations()
4532 		if (atomic_read(&pa->pa_count)) {  in ext4_discard_preallocations()
4533 			/* this shouldn't happen often - nobody should  in ext4_discard_preallocations()
4535 			spin_unlock(&pa->pa_lock);  in ext4_discard_preallocations()
4536 			spin_unlock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
4538 				 "uh-oh! used pa while discarding");  in ext4_discard_preallocations()
4544 		if (pa->pa_deleted == 0) {  in ext4_discard_preallocations()
4546 			spin_unlock(&pa->pa_lock);  in ext4_discard_preallocations()
4547 			list_del_rcu(&pa->pa_inode_list);  in ext4_discard_preallocations()
4548 			list_add(&pa->u.pa_tmp_list, &list);  in ext4_discard_preallocations()
4549 			needed--;  in ext4_discard_preallocations()
4554 		spin_unlock(&pa->pa_lock);  in ext4_discard_preallocations()
4555 		spin_unlock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
4560 		 * ->clear_inode() the inode will get freed  in ext4_discard_preallocations()
4563 		 * freed memory, bad-bad-bad */  in ext4_discard_preallocations()
4567 		 * of ->clear_inode(), but not in case of  in ext4_discard_preallocations()
4572 	spin_unlock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
4575 		BUG_ON(pa->pa_type != MB_INODE_PA);  in ext4_discard_preallocations()
4576 		group = ext4_get_group_number(sb, pa->pa_pstart);  in ext4_discard_preallocations()
4581 			ext4_error_err(sb, -err, "Error %d loading buddy information for %u",  in ext4_discard_preallocations()
4589 			ext4_error_err(sb, -err, "Error %d reading block bitmap for %u",  in ext4_discard_preallocations()
4596 		list_del(&pa->pa_group_list);  in ext4_discard_preallocations()
4603 		list_del(&pa->u.pa_tmp_list);  in ext4_discard_preallocations()
4604 		call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_discard_preallocations()
4615 		return -ENOMEM;  in ext4_mb_pa_alloc()
4616 	atomic_set(&pa->pa_count, 1);  in ext4_mb_pa_alloc()
4617 	ac->ac_pa = pa;  in ext4_mb_pa_alloc()
4623 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_mb_pa_free()
4626 	ac->ac_pa = NULL;  in ext4_mb_pa_free()
4627 	WARN_ON(!atomic_dec_and_test(&pa->pa_count));  in ext4_mb_pa_free()
4650 		list_for_each(cur, &grp->bb_prealloc_list) {  in ext4_mb_show_pa()
4653 			spin_lock(&pa->pa_lock);  in ext4_mb_show_pa()
4654 			ext4_get_group_no_and_offset(sb, pa->pa_pstart,  in ext4_mb_show_pa()
4656 			spin_unlock(&pa->pa_lock);  in ext4_mb_show_pa()
4658 				 pa->pa_len);  in ext4_mb_show_pa()
4661 		mb_debug(sb, "%u: %d/%d\n", i, grp->bb_free,  in ext4_mb_show_pa()
4662 			 grp->bb_fragments);  in ext4_mb_show_pa()
4668 	struct super_block *sb = ac->ac_sb;  in ext4_mb_show_ac()
4676 			ac->ac_status, ac->ac_flags);  in ext4_mb_show_ac()
4680 			(unsigned long)ac->ac_o_ex.fe_group,  in ext4_mb_show_ac()
4681 			(unsigned long)ac->ac_o_ex.fe_start,  in ext4_mb_show_ac()
4682 			(unsigned long)ac->ac_o_ex.fe_len,  in ext4_mb_show_ac()
4683 			(unsigned long)ac->ac_o_ex.fe_logical,  in ext4_mb_show_ac()
4684 			(unsigned long)ac->ac_g_ex.fe_group,  in ext4_mb_show_ac()
4685 			(unsigned long)ac->ac_g_ex.fe_start,  in ext4_mb_show_ac()
4686 			(unsigned long)ac->ac_g_ex.fe_len,  in ext4_mb_show_ac()
4687 			(unsigned long)ac->ac_g_ex.fe_logical,  in ext4_mb_show_ac()
4688 			(unsigned long)ac->ac_b_ex.fe_group,  in ext4_mb_show_ac()
4689 			(unsigned long)ac->ac_b_ex.fe_start,  in ext4_mb_show_ac()
4690 			(unsigned long)ac->ac_b_ex.fe_len,  in ext4_mb_show_ac()
4691 			(unsigned long)ac->ac_b_ex.fe_logical,  in ext4_mb_show_ac()
4692 			(int)ac->ac_criteria);  in ext4_mb_show_ac()
4693 	mb_debug(sb, "%u found", ac->ac_found);  in ext4_mb_show_ac()
4703 	ext4_mb_show_pa(ac->ac_sb);  in ext4_mb_show_ac()
4709  * We use locality group preallocation for small size file. The size of the
4717 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_group_or_file()
4718 	int bsbits = ac->ac_sb->s_blocksize_bits;  in ext4_mb_group_or_file()
4721 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_group_or_file()
4724 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_group_or_file()
4727 	size = extent_logical_end(sbi, &ac->ac_o_ex);  in ext4_mb_group_or_file()
4728 	isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1)  in ext4_mb_group_or_file()
4732 	    !inode_is_open_for_write(ac->ac_inode)) {  in ext4_mb_group_or_file()
4733 		ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC;  in ext4_mb_group_or_file()
4737 	if (sbi->s_mb_group_prealloc <= 0) {  in ext4_mb_group_or_file()
4738 		ac->ac_flags |= EXT4_MB_STREAM_ALLOC;  in ext4_mb_group_or_file()
4744 	if (size > sbi->s_mb_stream_request) {  in ext4_mb_group_or_file()
4745 		ac->ac_flags |= EXT4_MB_STREAM_ALLOC;  in ext4_mb_group_or_file()
4749 	BUG_ON(ac->ac_lg != NULL);  in ext4_mb_group_or_file()
4751 	 * locality group prealloc space are per cpu. The reason for having  in ext4_mb_group_or_file()
4752 	 * per cpu locality group is to reduce the contention between block  in ext4_mb_group_or_file()
4755 	ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups);  in ext4_mb_group_or_file()
4758 	ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC;  in ext4_mb_group_or_file()
4761 	mutex_lock(&ac->ac_lg->lg_mutex);  in ext4_mb_group_or_file()
4768 	struct super_block *sb = ar->inode->i_sb;  in ext4_mb_initialize_context()
4770 	struct ext4_super_block *es = sbi->s_es;  in ext4_mb_initialize_context()
4777 	len = ar->len;  in ext4_mb_initialize_context()
4784 	goal = ar->goal;  in ext4_mb_initialize_context()
4785 	if (goal < le32_to_cpu(es->s_first_data_block) ||  in ext4_mb_initialize_context()
4787 		goal = le32_to_cpu(es->s_first_data_block);  in ext4_mb_initialize_context()
4791 	ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical);  in ext4_mb_initialize_context()
4792 	ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_initialize_context()
4793 	ac->ac_sb = sb;  in ext4_mb_initialize_context()
4794 	ac->ac_inode = ar->inode;  in ext4_mb_initialize_context()
4795 	ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical;  in ext4_mb_initialize_context()
4796 	ac->ac_o_ex.fe_group = group;  in ext4_mb_initialize_context()
4797 	ac->ac_o_ex.fe_start = block;  in ext4_mb_initialize_context()
4798 	ac->ac_o_ex.fe_len = len;  in ext4_mb_initialize_context()
4799 	ac->ac_g_ex = ac->ac_o_ex;  in ext4_mb_initialize_context()
4800 	ac->ac_flags = ar->flags;  in ext4_mb_initialize_context()
4803 	 * locality group. this is a policy, actually */  in ext4_mb_initialize_context()
4808 			(unsigned) ar->len, (unsigned) ar->logical,  in ext4_mb_initialize_context()
4809 			(unsigned) ar->goal, ac->ac_flags, ac->ac_2order,  in ext4_mb_initialize_context()
4810 			(unsigned) ar->lleft, (unsigned) ar->pleft,  in ext4_mb_initialize_context()
4811 			(unsigned) ar->lright, (unsigned) ar->pright,  in ext4_mb_initialize_context()
4812 			inode_is_open_for_write(ar->inode) ? "" : "non-");  in ext4_mb_initialize_context()
4827 	mb_debug(sb, "discard locality group preallocation\n");  in ext4_mb_discard_lg_preallocations()
4831 	spin_lock(&lg->lg_prealloc_lock);  in ext4_mb_discard_lg_preallocations()
4832 	list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order],  in ext4_mb_discard_lg_preallocations()
4834 				lockdep_is_held(&lg->lg_prealloc_lock)) {  in ext4_mb_discard_lg_preallocations()
4835 		spin_lock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
4836 		if (atomic_read(&pa->pa_count)) {  in ext4_mb_discard_lg_preallocations()
4842 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
4845 		if (pa->pa_deleted) {  in ext4_mb_discard_lg_preallocations()
4846 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
4850 		BUG_ON(pa->pa_type != MB_GROUP_PA);  in ext4_mb_discard_lg_preallocations()
4854 		spin_unlock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
4856 		list_del_rcu(&pa->pa_inode_list);  in ext4_mb_discard_lg_preallocations()
4857 		list_add(&pa->u.pa_tmp_list, &discard_list);  in ext4_mb_discard_lg_preallocations()
4859 		total_entries--;  in ext4_mb_discard_lg_preallocations()
4870 	spin_unlock(&lg->lg_prealloc_lock);  in ext4_mb_discard_lg_preallocations()
4875 		group = ext4_get_group_number(sb, pa->pa_pstart);  in ext4_mb_discard_lg_preallocations()
4879 			ext4_error_err(sb, -err, "Error %d loading buddy information for %u",  in ext4_mb_discard_lg_preallocations()
4884 		list_del(&pa->pa_group_list);  in ext4_mb_discard_lg_preallocations()
4889 		list_del(&pa->u.pa_tmp_list);  in ext4_mb_discard_lg_preallocations()
4890 		call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_mb_discard_lg_preallocations()
4906 	struct super_block *sb = ac->ac_sb;  in ext4_mb_add_n_trim()
4907 	struct ext4_locality_group *lg = ac->ac_lg;  in ext4_mb_add_n_trim()
4908 	struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa;  in ext4_mb_add_n_trim()
4910 	order = fls(pa->pa_free) - 1;  in ext4_mb_add_n_trim()
4911 	if (order > PREALLOC_TB_SIZE - 1)  in ext4_mb_add_n_trim()
4913 		order = PREALLOC_TB_SIZE - 1;  in ext4_mb_add_n_trim()
4915 	spin_lock(&lg->lg_prealloc_lock);  in ext4_mb_add_n_trim()
4916 	list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order],  in ext4_mb_add_n_trim()
4918 				lockdep_is_held(&lg->lg_prealloc_lock)) {  in ext4_mb_add_n_trim()
4919 		spin_lock(&tmp_pa->pa_lock);  in ext4_mb_add_n_trim()
4920 		if (tmp_pa->pa_deleted) {  in ext4_mb_add_n_trim()
4921 			spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_add_n_trim()
4924 		if (!added && pa->pa_free < tmp_pa->pa_free) {  in ext4_mb_add_n_trim()
4926 			list_add_tail_rcu(&pa->pa_inode_list,  in ext4_mb_add_n_trim()
4927 						&tmp_pa->pa_inode_list);  in ext4_mb_add_n_trim()
4934 		spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_add_n_trim()
4938 		list_add_tail_rcu(&pa->pa_inode_list,  in ext4_mb_add_n_trim()
4939 					&lg->lg_prealloc_list[order]);  in ext4_mb_add_n_trim()
4940 	spin_unlock(&lg->lg_prealloc_lock);  in ext4_mb_add_n_trim()
4952  * if per-inode prealloc list is too long, trim some PA
4957 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);  in ext4_mb_trim_inode_pa()
4960 	count = atomic_read(&ei->i_prealloc_active);  in ext4_mb_trim_inode_pa()
4961 	delta = (sbi->s_mb_max_inode_prealloc >> 2) + 1;  in ext4_mb_trim_inode_pa()
4962 	if (count > sbi->s_mb_max_inode_prealloc + delta) {  in ext4_mb_trim_inode_pa()
4963 		count -= sbi->s_mb_max_inode_prealloc;  in ext4_mb_trim_inode_pa()
4973 	struct inode *inode = ac->ac_inode;  in ext4_mb_release_context()
4975 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_release_context()
4976 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_mb_release_context()
4978 		if (pa->pa_type == MB_GROUP_PA) {  in ext4_mb_release_context()
4980 			spin_lock(&pa->pa_lock);  in ext4_mb_release_context()
4981 			pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
4982 			pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
4983 			pa->pa_free -= ac->ac_b_ex.fe_len;  in ext4_mb_release_context()
4984 			pa->pa_len -= ac->ac_b_ex.fe_len;  in ext4_mb_release_context()
4985 			spin_unlock(&pa->pa_lock);  in ext4_mb_release_context()
4993 			if (likely(pa->pa_free)) {  in ext4_mb_release_context()
4994 				spin_lock(pa->pa_obj_lock);  in ext4_mb_release_context()
4995 				list_del_rcu(&pa->pa_inode_list);  in ext4_mb_release_context()
4996 				spin_unlock(pa->pa_obj_lock);  in ext4_mb_release_context()
5001 		if (pa->pa_type == MB_INODE_PA) {  in ext4_mb_release_context()
5003 			 * treat per-inode prealloc list as a lru list, then try  in ext4_mb_release_context()
5006 			spin_lock(pa->pa_obj_lock);  in ext4_mb_release_context()
5007 			list_move(&pa->pa_inode_list, &ei->i_prealloc_list);  in ext4_mb_release_context()
5008 			spin_unlock(pa->pa_obj_lock);  in ext4_mb_release_context()
5011 		ext4_mb_put_pa(ac, ac->ac_sb, pa);  in ext4_mb_release_context()
5013 	if (ac->ac_bitmap_page)  in ext4_mb_release_context()
5014 		put_page(ac->ac_bitmap_page);  in ext4_mb_release_context()
5015 	if (ac->ac_buddy_page)  in ext4_mb_release_context()
5016 		put_page(ac->ac_buddy_page);  in ext4_mb_release_context()
5017 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)  in ext4_mb_release_context()
5018 		mutex_unlock(&ac->ac_lg->lg_mutex);  in ext4_mb_release_context()
5039 		needed -= ret;  in ext4_mb_discard_preallocations()
5058 	freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len);  in ext4_mb_discard_preallocations_should_retry()
5064 	if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) {  in ext4_mb_discard_preallocations_should_retry()
5065 		ac->ac_flags |= EXT4_MB_STRICT_CHECK;  in ext4_mb_discard_preallocations_should_retry()
5096 	sb = ar->inode->i_sb;  in ext4_mb_new_blocks()
5100 	if (sbi->s_mount_state & EXT4_FC_REPLAY)  in ext4_mb_new_blocks()
5104 	if (ext4_is_quota_file(ar->inode))  in ext4_mb_new_blocks()
5105 		ar->flags |= EXT4_MB_USE_ROOT_BLOCKS;  in ext4_mb_new_blocks()
5107 	if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) {  in ext4_mb_new_blocks()
5112 		while (ar->len &&  in ext4_mb_new_blocks()
5113 			ext4_claim_free_clusters(sbi, ar->len, ar->flags)) {  in ext4_mb_new_blocks()
5117 			ar->len = ar->len >> 1;  in ext4_mb_new_blocks()
5119 		if (!ar->len) {  in ext4_mb_new_blocks()
5121 			*errp = -ENOSPC;  in ext4_mb_new_blocks()
5124 		reserv_clstrs = ar->len;  in ext4_mb_new_blocks()
5125 		if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) {  in ext4_mb_new_blocks()
5126 			dquot_alloc_block_nofail(ar->inode,  in ext4_mb_new_blocks()
5127 						 EXT4_C2B(sbi, ar->len));  in ext4_mb_new_blocks()
5129 			while (ar->len &&  in ext4_mb_new_blocks()
5130 				dquot_alloc_block(ar->inode,  in ext4_mb_new_blocks()
5131 						  EXT4_C2B(sbi, ar->len))) {  in ext4_mb_new_blocks()
5133 				ar->flags |= EXT4_MB_HINT_NOPREALLOC;  in ext4_mb_new_blocks()
5134 				ar->len--;  in ext4_mb_new_blocks()
5137 		inquota = ar->len;  in ext4_mb_new_blocks()
5138 		if (ar->len == 0) {  in ext4_mb_new_blocks()
5139 			*errp = -EDQUOT;  in ext4_mb_new_blocks()
5146 		ar->len = 0;  in ext4_mb_new_blocks()
5147 		*errp = -ENOMEM;  in ext4_mb_new_blocks()
5153 		ar->len = 0;  in ext4_mb_new_blocks()
5157 	ac->ac_op = EXT4_MB_HISTORY_PREALLOC;  in ext4_mb_new_blocks()
5160 		ac->ac_op = EXT4_MB_HISTORY_ALLOC;  in ext4_mb_new_blocks()
5170 		 * pa allocated above is added to grp->bb_prealloc_list only  in ext4_mb_new_blocks()
5172 		 * ac->ac_status == AC_STATUS_FOUND.  in ext4_mb_new_blocks()
5173 		 * And error from above mean ac->ac_status != AC_STATUS_FOUND  in ext4_mb_new_blocks()
5181 		if (ac->ac_status == AC_STATUS_FOUND &&  in ext4_mb_new_blocks()
5182 			ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len)  in ext4_mb_new_blocks()
5185 	if (likely(ac->ac_status == AC_STATUS_FOUND)) {  in ext4_mb_new_blocks()
5191 			block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_blocks()
5192 			ar->len = ac->ac_b_ex.fe_len;  in ext4_mb_new_blocks()
5203 		*errp = -ENOSPC;  in ext4_mb_new_blocks()
5208 		ac->ac_b_ex.fe_len = 0;  in ext4_mb_new_blocks()
5209 		ar->len = 0;  in ext4_mb_new_blocks()
5216 	if (inquota && ar->len < inquota)  in ext4_mb_new_blocks()
5217 		dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len));  in ext4_mb_new_blocks()
5218 	if (!ar->len) {  in ext4_mb_new_blocks()
5219 		if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0)  in ext4_mb_new_blocks()
5221 			percpu_counter_sub(&sbi->s_dirtyclusters_counter,  in ext4_mb_new_blocks()
5240 	if ((entry->efd_tid != new_entry->efd_tid) ||  in ext4_try_merge_freed_extent()
5241 	    (entry->efd_group != new_entry->efd_group))  in ext4_try_merge_freed_extent()
5243 	if (entry->efd_start_cluster + entry->efd_count ==  in ext4_try_merge_freed_extent()
5244 	    new_entry->efd_start_cluster) {  in ext4_try_merge_freed_extent()
5245 		new_entry->efd_start_cluster = entry->efd_start_cluster;  in ext4_try_merge_freed_extent()
5246 		new_entry->efd_count += entry->efd_count;  in ext4_try_merge_freed_extent()
5247 	} else if (new_entry->efd_start_cluster + new_entry->efd_count ==  in ext4_try_merge_freed_extent()
5248 		   entry->efd_start_cluster) {  in ext4_try_merge_freed_extent()
5249 		new_entry->efd_count += entry->efd_count;  in ext4_try_merge_freed_extent()
5252 	spin_lock(&sbi->s_md_lock);  in ext4_try_merge_freed_extent()
5253 	list_del(&entry->efd_list);  in ext4_try_merge_freed_extent()
5254 	spin_unlock(&sbi->s_md_lock);  in ext4_try_merge_freed_extent()
5255 	rb_erase(&entry->efd_node, entry_rb_root);  in ext4_try_merge_freed_extent()
5263 	ext4_group_t group = e4b->bd_group;  in ext4_mb_free_metadata()
5265 	ext4_grpblk_t clusters = new_entry->efd_count;  in ext4_mb_free_metadata()
5267 	struct ext4_group_info *db = e4b->bd_info;  in ext4_mb_free_metadata()
5268 	struct super_block *sb = e4b->bd_sb;  in ext4_mb_free_metadata()
5270 	struct rb_node **n = &db->bb_free_root.rb_node, *node;  in ext4_mb_free_metadata()
5274 	BUG_ON(e4b->bd_bitmap_page == NULL);  in ext4_mb_free_metadata()
5275 	BUG_ON(e4b->bd_buddy_page == NULL);  in ext4_mb_free_metadata()
5277 	new_node = &new_entry->efd_node;  in ext4_mb_free_metadata()
5278 	cluster = new_entry->efd_start_cluster;  in ext4_mb_free_metadata()
5284 		 * on-disk bitmap and lose not-yet-available  in ext4_mb_free_metadata()
5286 		get_page(e4b->bd_buddy_page);  in ext4_mb_free_metadata()
5287 		get_page(e4b->bd_bitmap_page);  in ext4_mb_free_metadata()
5292 		if (cluster < entry->efd_start_cluster)  in ext4_mb_free_metadata()
5293 			n = &(*n)->rb_left;  in ext4_mb_free_metadata()
5294 		else if (cluster >= (entry->efd_start_cluster + entry->efd_count))  in ext4_mb_free_metadata()
5295 			n = &(*n)->rb_right;  in ext4_mb_free_metadata()
5300 				"Block already on to-be-freed list");  in ext4_mb_free_metadata()
5307 	rb_insert_color(new_node, &db->bb_free_root);  in ext4_mb_free_metadata()
5314 					    &(db->bb_free_root));  in ext4_mb_free_metadata()
5321 					    &(db->bb_free_root));  in ext4_mb_free_metadata()
5324 	spin_lock(&sbi->s_md_lock);  in ext4_mb_free_metadata()
5325 	list_add_tail(&new_entry->efd_list, &sbi->s_freed_data_list);  in ext4_mb_free_metadata()
5326 	sbi->s_mb_free_pending += clusters;  in ext4_mb_free_metadata()
5327 	spin_unlock(&sbi->s_md_lock);  in ext4_mb_free_metadata()
5340 	struct super_block *sb = ar->inode->i_sb;  in ext4_mb_new_blocks_simple()
5346 	struct ext4_super_block *es = EXT4_SB(sb)->s_es;  in ext4_mb_new_blocks_simple()
5348 	goal = ar->goal;  in ext4_mb_new_blocks_simple()
5349 	if (goal < le32_to_cpu(es->s_first_data_block) ||  in ext4_mb_new_blocks_simple()
5351 		goal = le32_to_cpu(es->s_first_data_block);  in ext4_mb_new_blocks_simple()
5353 	ar->len = 0;  in ext4_mb_new_blocks_simple()
5367 			i = mb_find_next_zero_bit(bitmap_bh->b_data, max,  in ext4_mb_new_blocks_simple()
5383 		*errp = -ENOSPC;  in ext4_mb_new_blocks_simple()
5389 	ar->len = 1;  in ext4_mb_new_blocks_simple()
5398 	struct super_block *sb = inode->i_sb;  in ext4_free_blocks_simple()
5417 		if (!mb_test_bit(blkoff + i, bitmap_bh->b_data))  in ext4_free_blocks_simple()
5420 	mb_clear_bits(bitmap_bh->b_data, blkoff, count);  in ext4_free_blocks_simple()
5426 		count - already_freed);  in ext4_free_blocks_simple()
5436  * ext4_mb_clear_bb() -- helper function for freeing blocks.
5450 	struct super_block *sb = inode->i_sb;  in ext4_mb_clear_bb()
5467 		ext4_error(sb, "Freeing blocks in system zone - "  in ext4_mb_clear_bb()
5487 		overflow = EXT4_C2B(sbi, bit) + count -  in ext4_mb_clear_bb()
5489 		count -= overflow;  in ext4_mb_clear_bb()
5502 		err = -EIO;  in ext4_mb_clear_bb()
5508 		ext4_error(sb, "Freeing blocks in system zone - "  in ext4_mb_clear_bb()
5521 	 * to unshare ->b_data if a currently-committing transaction is  in ext4_mb_clear_bb()
5532 			BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data));  in ext4_mb_clear_bb()
5559 		new_entry->efd_start_cluster = bit;  in ext4_mb_clear_bb()
5560 		new_entry->efd_group = block_group;  in ext4_mb_clear_bb()
5561 		new_entry->efd_count = count_clusters;  in ext4_mb_clear_bb()
5562 		new_entry->efd_tid = handle->h_transaction->t_tid;  in ext4_mb_clear_bb()
5565 		mb_clear_bits(bitmap_bh->b_data, bit, count_clusters);  in ext4_mb_clear_bb()
5568 		/* need to update group_info->bb_free and bitmap  in ext4_mb_clear_bb()
5575 			if (err && err != -EOPNOTSUPP)  in ext4_mb_clear_bb()
5584 		mb_clear_bits(bitmap_bh->b_data, bit, count_clusters);  in ext4_mb_clear_bb()
5594 	if (sbi->s_log_groups_per_flex) {  in ext4_mb_clear_bb()
5598 						  flex_group)->free_clusters);  in ext4_mb_clear_bb()
5609 		percpu_counter_add(&sbi->s_freeclusters_counter,  in ext4_mb_clear_bb()
5640  * ext4_free_blocks() -- Free given blocks and update quota
5652 	struct super_block *sb = inode->i_sb;  in ext4_free_blocks()
5660 			BUG_ON(block != bh->b_blocknr);  in ext4_free_blocks()
5662 			block = bh->b_blocknr;  in ext4_free_blocks()
5665 	if (sbi->s_mount_state & EXT4_FC_REPLAY) {  in ext4_free_blocks()
5674 		ext4_error(sb, "Freeing blocks not in datazone - "  in ext4_free_blocks()
5700 			overflow = sbi->s_cluster_ratio - overflow;  in ext4_free_blocks()
5703 				count -= overflow;  in ext4_free_blocks()
5707 			block -= overflow;  in ext4_free_blocks()
5717 				count -= overflow;  in ext4_free_blocks()
5721 			count += sbi->s_cluster_ratio - overflow;  in ext4_free_blocks()
5733 				bh = sb_find_get_block(inode->i_sb, block + i);  in ext4_free_blocks()
5743  * ext4_group_add_blocks() -- Add given blocks to an existing group
5765 	ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1);  in ext4_group_add_blocks()
5766 	unsigned long cluster_count = last_cluster - first_cluster + 1;  in ext4_group_add_blocks()
5768 	ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1);  in ext4_group_add_blocks()
5781 		err = -EINVAL;  in ext4_group_add_blocks()
5794 		err = -EIO;  in ext4_group_add_blocks()
5799 		ext4_error(sb, "Adding blocks in system zones - "  in ext4_group_add_blocks()
5802 		err = -EINVAL;  in ext4_group_add_blocks()
5813 	 * to unshare ->b_data if a currently-committing transaction is  in ext4_group_add_blocks()
5823 		if (!mb_test_bit(bit + i, bitmap_bh->b_data)) {  in ext4_group_add_blocks()
5837 	 * need to update group_info->bb_free and bitmap  in ext4_group_add_blocks()
5842 	mb_clear_bits(bitmap_bh->b_data, bit, cluster_count);  in ext4_group_add_blocks()
5850 	percpu_counter_add(&sbi->s_freeclusters_counter,  in ext4_group_add_blocks()
5853 	if (sbi->s_log_groups_per_flex) {  in ext4_group_add_blocks()
5857 						  flex_group)->free_clusters);  in ext4_group_add_blocks()
5879  * ext4_trim_extent -- function to TRIM one single free extent in the group
5895 	ext4_group_t group = e4b->bd_group;  in ext4_trim_extent()
5923 	if (grp < (ext4_get_groups_count(sb) - 1))  in ext4_last_grp_cluster()
5926 		nr_clusters_in_group = (ext4_blocks_count(EXT4_SB(sb)->s_es) -  in ext4_last_grp_cluster()
5930 	return nr_clusters_in_group - 1;  in ext4_last_grp_cluster()
5946 	last = ext4_last_grp_cluster(sb, e4b->bd_group);  in ext4_try_to_trim_range()
5947 	bitmap = e4b->bd_bitmap;  in ext4_try_to_trim_range()
5951 	start = max(e4b->bd_info->bb_first_free, start);  in ext4_try_to_trim_range()
5964 		if ((next - start) >= minblocks) {  in ext4_try_to_trim_range()
5965 			int ret = ext4_trim_extent(sb, start, next - start, e4b);  in ext4_try_to_trim_range()
5967 			if (ret && ret != -EOPNOTSUPP)  in ext4_try_to_trim_range()
5969 			count += next - start;  in ext4_try_to_trim_range()
5971 		free_count += next - start;  in ext4_try_to_trim_range()
5978 			ext4_unlock_group(sb, e4b->bd_group);  in ext4_try_to_trim_range()
5980 			ext4_lock_group(sb, e4b->bd_group);  in ext4_try_to_trim_range()
5983 		if ((e4b->bd_info->bb_free - free_count) < minblocks)  in ext4_try_to_trim_range()
5988 		EXT4_MB_GRP_SET_TRIMMED(e4b->bd_info);  in ext4_try_to_trim_range()
5994  * ext4_trim_all_free -- function to trim all free space in alloc. group
6031 	    minblocks < EXT4_SB(sb)->s_last_trim_minblks)  in ext4_trim_all_free()
6046  * ext4_trim_fs() -- trim ioctl handle function
6059 	struct request_queue *q = bdev_get_queue(sb->s_bdev);  in ext4_trim_fs()
6065 			le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);  in ext4_trim_fs()
6066 	ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es);  in ext4_trim_fs()
6069 	start = range->start >> sb->s_blocksize_bits;  in ext4_trim_fs()
6070 	end = start + (range->len >> sb->s_blocksize_bits) - 1;  in ext4_trim_fs()
6072 			      range->minlen >> sb->s_blocksize_bits);  in ext4_trim_fs()
6076 	    range->len < sb->s_blocksize)  in ext4_trim_fs()
6077 		return -EINVAL;  in ext4_trim_fs()
6079 	if (range->minlen < q->limits.discard_granularity) {  in ext4_trim_fs()
6081 			q->limits.discard_granularity >> sb->s_blocksize_bits);  in ext4_trim_fs()
6085 	if (end >= max_blks - 1)  in ext4_trim_fs()
6086 		end = max_blks - 1;  in ext4_trim_fs()
6099 	end = EXT4_CLUSTERS_PER_GROUP(sb) - 1;  in ext4_trim_fs()
6116 		 * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to  in ext4_trim_fs()
6122 		if (grp->bb_free >= minlen) {  in ext4_trim_fs()
6140 		EXT4_SB(sb)->s_last_trim_minblks = minlen;  in ext4_trim_fs()
6143 	range->len = EXT4_C2B(EXT4_SB(sb), trimmed) << sb->s_blocksize_bits;  in ext4_trim_fs()
6169 	start = max(e4b.bd_info->bb_first_free, start);  in ext4_mballoc_query_range()
6171 		end = EXT4_CLUSTERS_PER_GROUP(sb) - 1;  in ext4_mballoc_query_range()
6180 		error = formatter(sb, group, start, next - start, priv);  in ext4_mballoc_query_range()