1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
18 #include <linux/backing-dev.h>
30  *   - test ext4_ext_search_left() and ext4_ext_search_right()
31  *   - search for metadata in few groups
34  *   - normalization should take into account whether file is still open
35  *   - discard preallocations if no free space left (policy?)
36  *   - don't normalize tails
37  *   - quota
38  *   - reservation for superuser
41  *   - bitmap read-ahead (proposed by Oleg Drokin aka green)
42  *   - track min/max extents in each group for better group selection
43  *   - mb_mark_used() may allocate chunk right after splitting buddy
44  *   - tree of groups sorted by number of free blocks
45  *   - error handling
56  * is larger. If the size is less than sbi->s_mb_stream_request we
66  * ext4_inode_info->i_prealloc_list, which contains list of prealloc
70  * pa_lstart -> the logical start block for this prealloc space
71  * pa_pstart -> the physical start block for this prealloc space
72  * pa_len    -> length for this prealloc space (in clusters)
73  * pa_free   ->  free space available in this prealloc space (in clusters)
85  * have the group allocation flag set then we look at the locality group
90  * The reason for having a per cpu locality group is to reduce the contention
93  * The locality group prealloc space is used looking at whether we have
96  * If we can't allocate blocks via inode prealloc or/and locality group
128  * sbi->s_mb_group_prealloc.  The default value of s_mb_group_prealloc is
129  * dependent on the cluster size; for non-bigalloc file systems, it is
132  * terms of number of blocks. If we have mounted the file system with -O
134  * the smallest multiple of the stripe value (sbi->s_stripe) which is
144  * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The
147  * stripe size (sbi->s_stripe), we try to search for contiguous block in
149  * not, we search in the specific group using bitmap for best extents. The
167  *  - on-disk bitmap
168  *  - in-core buddy (actually includes buddy and bitmap)
169  *  - preallocation descriptors (PAs)
172  *  - inode
173  *    assiged to specific inode and can be used for this inode only.
174  *    it describes part of inode's space preallocated to specific
181  *  - locality group
182  *    assigned to specific locality group which does not translate to
188  *    in-core buddy = on-disk bitmap + preallocation descriptors
191  *  - allocated blocks (persistent)
192  *  - preallocated blocks (non-persistent)
196  * literally -- time is discrete and delimited by locks.
199  *  blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA.
202  *  - init buddy:			buddy = on-disk + PAs
203  *  - new PA:				buddy += N; PA = N
204  *  - use inode PA:			on-disk += N; PA -= N
205  *  - discard inode PA			buddy -= on-disk - PA; PA = 0
206  *  - use locality group PA		on-disk += N; PA -= N
207  *  - discard locality group PA		buddy -= PA; PA = 0
208  *  note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap
210  *        bits from PA, only from on-disk bitmap
214  * killing performance on high-end SMP hardware. let's try to relax it using
218  *     nobody can re-allocate that block
219  *  3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has
221  *     on-disk bitmap if buddy has same bit set or/and PA covers corresponded
225  *  - init buddy vs.
226  *    - new PA
229  *    - use inode PA
230  *      we need to make sure that either on-disk bitmap or PA has uptodate data
231  *      given (3) we care that PA-=N operation doesn't interfere with init
232  *    - discard inode PA
234  *    - use locality group PA
235  *      again PA-=N must be serialized with init
236  *    - discard locality group PA
238  *  - new PA vs.
239  *    - use inode PA
241  *    - discard inode PA
243  *    - use locality group PA
245  *    - discard locality group PA
247  *  - use inode PA
248  *    - use inode PA
250  *    - discard inode PA
252  *    - use locality group PA
253  *      nothing wrong here -- they're different PAs covering different blocks
254  *    - discard locality group PA
258  *  - PA is referenced and while it is no discard is possible
259  *  - PA is referenced until block isn't marked in on-disk bitmap
260  *  - PA changes only after on-disk bitmap
261  *  - discard must not compete with init. either init is done before
263  *  - buddy init as sum of on-disk bitmap and PAs is done atomically
273  *  - allocation:
276  *    mark bits in on-disk bitmap
279  *  - use preallocation:
280  *    find proper PA (per-inode or group)
282  *    mark bits in on-disk bitmap
286  *  - free:
288  *    mark bits in on-disk bitmap
291  *  - discard preallocations in group:
294  *    load on-disk bitmap
296  *    remove PA from object (inode or locality group)
297  *    mark free blocks in-core
299  *  - discard inode's preallocations:
306  *  - bitlock on a group	(group)
307  *  - object (inode/locality)	(object)
308  *  - per-pa lock		(pa)
311  *  - new pa
315  *  - find and use pa:
318  *  - release consumed pa:
323  *  - generate in-core bitmap:
327  *  - discard all for given object (inode, locality group):
332  *  - discard all for given group:
409 	ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix;  in mb_find_next_zero_bit()
422 	ret = ext4_find_next_bit(addr, tmpmax, start) - fix;  in mb_find_next_bit()
432 	BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);  in mb_find_buddy()
435 	if (order > e4b->bd_blkbits + 1) {  in mb_find_buddy()
442 		*max = 1 << (e4b->bd_blkbits + 3);  in mb_find_buddy()
443 		return e4b->bd_bitmap;  in mb_find_buddy()
446 	bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order];  in mb_find_buddy()
447 	*max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order];  in mb_find_buddy()
457 	struct super_block *sb = e4b->bd_sb;  in mb_free_blocks_double()
459 	if (unlikely(e4b->bd_info->bb_bitmap == NULL))  in mb_free_blocks_double()
461 	assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));  in mb_free_blocks_double()
463 		if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) {  in mb_free_blocks_double()
466 			blocknr = ext4_group_first_block_no(sb, e4b->bd_group);  in mb_free_blocks_double()
468 			ext4_grp_locked_error(sb, e4b->bd_group,  in mb_free_blocks_double()
469 					      inode ? inode->i_ino : 0,  in mb_free_blocks_double()
474 			ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in mb_free_blocks_double()
477 		mb_clear_bit(first + i, e4b->bd_info->bb_bitmap);  in mb_free_blocks_double()
485 	if (unlikely(e4b->bd_info->bb_bitmap == NULL))  in mb_mark_used_double()
487 	assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));  in mb_mark_used_double()
489 		BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap));  in mb_mark_used_double()
490 		mb_set_bit(first + i, e4b->bd_info->bb_bitmap);  in mb_mark_used_double()
496 	if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) {  in mb_cmp_bitmaps()
499 		b1 = (unsigned char *) e4b->bd_info->bb_bitmap;  in mb_cmp_bitmaps()
501 		for (i = 0; i < e4b->bd_sb->s_blocksize; i++) {  in mb_cmp_bitmaps()
503 				ext4_msg(e4b->bd_sb, KERN_ERR,  in mb_cmp_bitmaps()
507 					 e4b->bd_group, i, i * 8, b1[i], b2[i]);  in mb_cmp_bitmaps()
546 	struct super_block *sb = e4b->bd_sb;  in __mb_check_buddy()
547 	int order = e4b->bd_blkbits + 1;  in __mb_check_buddy()
570 		buddy2 = mb_find_buddy(e4b, order - 1, &max2);  in __mb_check_buddy()
597 					!mb_test_bit(k, e4b->bd_bitmap));  in __mb_check_buddy()
601 		MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count);  in __mb_check_buddy()
602 		order--;  in __mb_check_buddy()
605 	fstart = -1;  in __mb_check_buddy()
609 			MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free);  in __mb_check_buddy()
610 			if (fstart == -1) {  in __mb_check_buddy()
616 		fstart = -1;  in __mb_check_buddy()
618 		for (j = 0; j < e4b->bd_blkbits + 1; j++) {  in __mb_check_buddy()
625 	MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info));  in __mb_check_buddy()
626 	MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments);  in __mb_check_buddy()
628 	grp = ext4_get_group_info(sb, e4b->bd_group);  in __mb_check_buddy()
629 	list_for_each(cur, &grp->bb_prealloc_list) {  in __mb_check_buddy()
633 		ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k);  in __mb_check_buddy()
634 		MB_CHECK_ASSERT(groupnr == e4b->bd_group);  in __mb_check_buddy()
635 		for (i = 0; i < pa->pa_len; i++)  in __mb_check_buddy()
665 	border = 2 << sb->s_blocksize_bits;  in ext4_mb_mark_free_simple()
669 		max = ffs(first | border) - 1;  in ext4_mb_mark_free_simple()
672 		min = fls(len) - 1;  in ext4_mb_mark_free_simple()
679 		grp->bb_counters[min]++;  in ext4_mb_mark_free_simple()
682 				     buddy + sbi->s_mb_offsets[min]);  in ext4_mb_mark_free_simple()
684 		len -= chunk;  in ext4_mb_mark_free_simple()
699 	grp->bb_largest_free_order = -1; /* uninit */  in mb_set_largest_free_order()
701 	bits = sb->s_blocksize_bits + 1;  in mb_set_largest_free_order()
702 	for (i = bits; i >= 0; i--) {  in mb_set_largest_free_order()
703 		if (grp->bb_counters[i] > 0) {  in mb_set_largest_free_order()
704 			grp->bb_largest_free_order = i;  in mb_set_largest_free_order()
725 	 * of on-disk bitmap and preallocations */  in ext4_mb_generate_buddy()
727 	grp->bb_first_free = i;  in ext4_mb_generate_buddy()
732 		len = i - first;  in ext4_mb_generate_buddy()
737 			grp->bb_counters[0]++;  in ext4_mb_generate_buddy()
741 	grp->bb_fragments = fragments;  in ext4_mb_generate_buddy()
743 	if (free != grp->bb_free) {  in ext4_mb_generate_buddy()
747 				      free, grp->bb_free);  in ext4_mb_generate_buddy()
752 		grp->bb_free = free;  in ext4_mb_generate_buddy()
758 	clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state));  in ext4_mb_generate_buddy()
760 	period = get_cycles() - period;  in ext4_mb_generate_buddy()
761 	spin_lock(&sbi->s_bal_lock);  in ext4_mb_generate_buddy()
762 	sbi->s_mb_buddies_generated++;  in ext4_mb_generate_buddy()
763 	sbi->s_mb_generation_time += period;  in ext4_mb_generate_buddy()
764 	spin_unlock(&sbi->s_bal_lock);  in ext4_mb_generate_buddy()
776 	e4b->bd_info->bb_fragments = 0;  in mb_regenerate_buddy()
777 	memset(e4b->bd_info->bb_counters, 0,  in mb_regenerate_buddy()
778 		sizeof(*e4b->bd_info->bb_counters) *  in mb_regenerate_buddy()
779 		(e4b->bd_sb->s_blocksize_bits + 2));  in mb_regenerate_buddy()
781 	ext4_mb_generate_buddy(e4b->bd_sb, e4b->bd_buddy,  in mb_regenerate_buddy()
782 		e4b->bd_bitmap, e4b->bd_group);  in mb_regenerate_buddy()
823 	mb_debug(1, "init page %lu\n", page->index);  in ext4_mb_init_cache()
825 	inode = page->mapping->host;  in ext4_mb_init_cache()
826 	sb = inode->i_sb;  in ext4_mb_init_cache()
840 			err = -ENOMEM;  in ext4_mb_init_cache()
846 	first_group = page->index * blocks_per_page / 2;  in ext4_mb_init_cache()
884 	first_block = page->index * blocks_per_page;  in ext4_mb_init_cache()
890 		if (!bh[group - first_group])  in ext4_mb_init_cache()
894 		if (!buffer_verified(bh[group - first_group]))  in ext4_mb_init_cache()
906 		bitmap = bh[group - first_group]->b_data;  in ext4_mb_init_cache()
916 				group, page->index, i * blocksize);  in ext4_mb_init_cache()
919 			grinfo->bb_fragments = 0;  in ext4_mb_init_cache()
920 			memset(grinfo->bb_counters, 0,  in ext4_mb_init_cache()
921 			       sizeof(*grinfo->bb_counters) *  in ext4_mb_init_cache()
922 				(sb->s_blocksize_bits+2));  in ext4_mb_init_cache()
936 				group, page->index, i * blocksize);  in ext4_mb_init_cache()
943 			/* mark all preallocated blks used in in-core bitmap */  in ext4_mb_init_cache()
970  * are on the same page e4b->bd_buddy_page is NULL and return value is 0.
975 	struct inode *inode = EXT4_SB(sb)->s_buddy_cache;  in ext4_mb_get_buddy_page_lock()
980 	e4b->bd_buddy_page = NULL;  in ext4_mb_get_buddy_page_lock()
981 	e4b->bd_bitmap_page = NULL;  in ext4_mb_get_buddy_page_lock()
983 	blocks_per_page = PAGE_SIZE / sb->s_blocksize;  in ext4_mb_get_buddy_page_lock()
992 	page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_get_buddy_page_lock()
994 		return -ENOMEM;  in ext4_mb_get_buddy_page_lock()
995 	BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_get_buddy_page_lock()
996 	e4b->bd_bitmap_page = page;  in ext4_mb_get_buddy_page_lock()
997 	e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);  in ext4_mb_get_buddy_page_lock()
1006 	page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_get_buddy_page_lock()
1008 		return -ENOMEM;  in ext4_mb_get_buddy_page_lock()
1009 	BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_get_buddy_page_lock()
1010 	e4b->bd_buddy_page = page;  in ext4_mb_get_buddy_page_lock()
1016 	if (e4b->bd_bitmap_page) {  in ext4_mb_put_buddy_page_lock()
1017 		unlock_page(e4b->bd_bitmap_page);  in ext4_mb_put_buddy_page_lock()
1018 		put_page(e4b->bd_bitmap_page);  in ext4_mb_put_buddy_page_lock()
1020 	if (e4b->bd_buddy_page) {  in ext4_mb_put_buddy_page_lock()
1021 		unlock_page(e4b->bd_buddy_page);  in ext4_mb_put_buddy_page_lock()
1022 		put_page(e4b->bd_buddy_page);  in ext4_mb_put_buddy_page_lock()
1066 		ret = -EIO;  in ext4_mb_init_group()
1085 		ret = -EIO;  in ext4_mb_init_group()
1110 	struct inode *inode = sbi->s_buddy_cache;  in ext4_mb_load_buddy_gfp()
1115 	blocks_per_page = PAGE_SIZE / sb->s_blocksize;  in ext4_mb_load_buddy_gfp()
1118 	e4b->bd_blkbits = sb->s_blocksize_bits;  in ext4_mb_load_buddy_gfp()
1119 	e4b->bd_info = grp;  in ext4_mb_load_buddy_gfp()
1120 	e4b->bd_sb = sb;  in ext4_mb_load_buddy_gfp()
1121 	e4b->bd_group = group;  in ext4_mb_load_buddy_gfp()
1122 	e4b->bd_buddy_page = NULL;  in ext4_mb_load_buddy_gfp()
1123 	e4b->bd_bitmap_page = NULL;  in ext4_mb_load_buddy_gfp()
1146 	page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED);  in ext4_mb_load_buddy_gfp()
1158 		page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_load_buddy_gfp()
1160 			BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_load_buddy_gfp()
1168 					       (poff * sb->s_blocksize));  in ext4_mb_load_buddy_gfp()
1174 		ret = -ENOMEM;  in ext4_mb_load_buddy_gfp()
1178 		ret = -EIO;  in ext4_mb_load_buddy_gfp()
1183 	e4b->bd_bitmap_page = page;  in ext4_mb_load_buddy_gfp()
1184 	e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);  in ext4_mb_load_buddy_gfp()
1190 	page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED);  in ext4_mb_load_buddy_gfp()
1194 		page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_load_buddy_gfp()
1196 			BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_load_buddy_gfp()
1198 				ret = ext4_mb_init_cache(page, e4b->bd_bitmap,  in ext4_mb_load_buddy_gfp()
1209 		ret = -ENOMEM;  in ext4_mb_load_buddy_gfp()
1213 		ret = -EIO;  in ext4_mb_load_buddy_gfp()
1218 	e4b->bd_buddy_page = page;  in ext4_mb_load_buddy_gfp()
1219 	e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize);  in ext4_mb_load_buddy_gfp()
1221 	BUG_ON(e4b->bd_bitmap_page == NULL);  in ext4_mb_load_buddy_gfp()
1222 	BUG_ON(e4b->bd_buddy_page == NULL);  in ext4_mb_load_buddy_gfp()
1229 	if (e4b->bd_bitmap_page)  in ext4_mb_load_buddy_gfp()
1230 		put_page(e4b->bd_bitmap_page);  in ext4_mb_load_buddy_gfp()
1231 	if (e4b->bd_buddy_page)  in ext4_mb_load_buddy_gfp()
1232 		put_page(e4b->bd_buddy_page);  in ext4_mb_load_buddy_gfp()
1233 	e4b->bd_buddy = NULL;  in ext4_mb_load_buddy_gfp()
1234 	e4b->bd_bitmap = NULL;  in ext4_mb_load_buddy_gfp()
1246 	if (e4b->bd_bitmap_page)  in ext4_mb_unload_buddy()
1247 		put_page(e4b->bd_bitmap_page);  in ext4_mb_unload_buddy()
1248 	if (e4b->bd_buddy_page)  in ext4_mb_unload_buddy()
1249 		put_page(e4b->bd_buddy_page);  in ext4_mb_unload_buddy()
1256 	int bb_incr = 1 << (e4b->bd_blkbits - 1);  in mb_find_order_for_block()
1259 	BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);  in mb_find_order_for_block()
1260 	BUG_ON(block >= (1 << (e4b->bd_blkbits + 3)));  in mb_find_order_for_block()
1262 	bb = e4b->bd_buddy;  in mb_find_order_for_block()
1263 	while (order <= e4b->bd_blkbits + 1) {  in mb_find_order_for_block()
1282 		if ((cur & 31) == 0 && (len - cur) >= 32) {  in mb_clear_bits()
1295  * will return first found zero bit if any, -1 otherwise
1300 	int zero_bit = -1;  in mb_test_and_clear_bits()
1304 		if ((cur & 31) == 0 && (len - cur) >= 32) {  in mb_test_and_clear_bits()
1307 			if (*addr != (__u32)(-1) && zero_bit == -1)  in mb_test_and_clear_bits()
1313 		if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1)  in mb_test_and_clear_bits()
1327 		if ((cur & 31) == 0 && (len - cur) >= 32) {  in ext4_set_bits()
1346 		(*bit) -= side;  in mb_buddy_adjust_border()
1352 		return -1;  in mb_buddy_adjust_border()
1375 		 * ---------------------------------  in mb_buddy_mark_free()
1377 		 * ---------------------------------  in mb_buddy_mark_free()
1379 		 * ---------------------------------  in mb_buddy_mark_free()
1395 			e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1);  in mb_buddy_mark_free()
1397 			e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1);  in mb_buddy_mark_free()
1403 			mb_clear_bits(buddy, first, last - first + 1);  in mb_buddy_mark_free()
1404 			e4b->bd_info->bb_counters[order - 1] += last - first + 1;  in mb_buddy_mark_free()
1419 	int last = first + count - 1;  in mb_free_blocks()
1420 	struct super_block *sb = e4b->bd_sb;  in mb_free_blocks()
1424 	BUG_ON(last >= (sb->s_blocksize << 3));  in mb_free_blocks()
1425 	assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));  in mb_free_blocks()
1427 	if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))  in mb_free_blocks()
1433 	e4b->bd_info->bb_free += count;  in mb_free_blocks()
1434 	if (first < e4b->bd_info->bb_first_free)  in mb_free_blocks()
1435 		e4b->bd_info->bb_first_free = first;  in mb_free_blocks()
1441 		left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap);  in mb_free_blocks()
1442 	block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count);  in mb_free_blocks()
1443 	if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0])  in mb_free_blocks()
1444 		right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap);  in mb_free_blocks()
1446 	if (unlikely(block != -1)) {  in mb_free_blocks()
1450 		blocknr = ext4_group_first_block_no(sb, e4b->bd_group);  in mb_free_blocks()
1452 		ext4_grp_locked_error(sb, e4b->bd_group,  in mb_free_blocks()
1453 				      inode ? inode->i_ino : 0,  in mb_free_blocks()
1458 		ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in mb_free_blocks()
1466 		e4b->bd_info->bb_fragments--;  in mb_free_blocks()
1468 		e4b->bd_info->bb_fragments++;  in mb_free_blocks()
1478 		e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1;  in mb_free_blocks()
1481 		last -= !right_is_free;  in mb_free_blocks()
1482 		e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1;  in mb_free_blocks()
1489 	mb_set_largest_free_order(sb, e4b->bd_info);  in mb_free_blocks()
1500 	assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));  in mb_find_extent()
1507 		ex->fe_len = 0;  in mb_find_extent()
1508 		ex->fe_start = 0;  in mb_find_extent()
1509 		ex->fe_group = 0;  in mb_find_extent()
1517 	ex->fe_len = 1 << order;  in mb_find_extent()
1518 	ex->fe_start = block << order;  in mb_find_extent()
1519 	ex->fe_group = e4b->bd_group;  in mb_find_extent()
1522 	next = next - ex->fe_start;  in mb_find_extent()
1523 	ex->fe_len -= next;  in mb_find_extent()
1524 	ex->fe_start += next;  in mb_find_extent()
1526 	while (needed > ex->fe_len &&  in mb_find_extent()
1533 		if (mb_test_bit(next, e4b->bd_bitmap))  in mb_find_extent()
1539 		ex->fe_len += 1 << order;  in mb_find_extent()
1542 	if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) {  in mb_find_extent()
1545 		ext4_error(e4b->bd_sb, "corruption or bug in mb_find_extent "  in mb_find_extent()
1547 			   block, order, needed, ex->fe_group, ex->fe_start,  in mb_find_extent()
1548 			   ex->fe_len, ex->fe_logical);  in mb_find_extent()
1549 		ex->fe_len = 0;  in mb_find_extent()
1550 		ex->fe_start = 0;  in mb_find_extent()
1551 		ex->fe_group = 0;  in mb_find_extent()
1553 	return ex->fe_len;  in mb_find_extent()
1562 	int start = ex->fe_start;  in mb_mark_used()
1563 	int len = ex->fe_len;  in mb_mark_used()
1568 	BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3));  in mb_mark_used()
1569 	BUG_ON(e4b->bd_group != ex->fe_group);  in mb_mark_used()
1570 	assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));  in mb_mark_used()
1574 	e4b->bd_info->bb_free -= len;  in mb_mark_used()
1575 	if (e4b->bd_info->bb_first_free == start)  in mb_mark_used()
1576 		e4b->bd_info->bb_first_free += len;  in mb_mark_used()
1580 		mlen = !mb_test_bit(start - 1, e4b->bd_bitmap);  in mb_mark_used()
1581 	if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0])  in mb_mark_used()
1582 		max = !mb_test_bit(start + len, e4b->bd_bitmap);  in mb_mark_used()
1584 		e4b->bd_info->bb_fragments++;  in mb_mark_used()
1586 		e4b->bd_info->bb_fragments--;  in mb_mark_used()
1598 			e4b->bd_info->bb_counters[ord]--;  in mb_mark_used()
1600 			len -= mlen;  in mb_mark_used()
1613 		e4b->bd_info->bb_counters[ord]--;  in mb_mark_used()
1615 		ord--;  in mb_mark_used()
1620 		e4b->bd_info->bb_counters[ord]++;  in mb_mark_used()
1621 		e4b->bd_info->bb_counters[ord]++;  in mb_mark_used()
1623 	mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info);  in mb_mark_used()
1625 	ext4_set_bits(e4b->bd_bitmap, ex->fe_start, len0);  in mb_mark_used()
1637 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_best_found()
1640 	BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);  in ext4_mb_use_best_found()
1641 	BUG_ON(ac->ac_status == AC_STATUS_FOUND);  in ext4_mb_use_best_found()
1643 	ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len);  in ext4_mb_use_best_found()
1644 	ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical;  in ext4_mb_use_best_found()
1645 	ret = mb_mark_used(e4b, &ac->ac_b_ex);  in ext4_mb_use_best_found()
1649 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_use_best_found()
1651 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_best_found()
1652 	ac->ac_tail = ret & 0xffff;  in ext4_mb_use_best_found()
1653 	ac->ac_buddy = ret >> 16;  in ext4_mb_use_best_found()
1662 	ac->ac_bitmap_page = e4b->bd_bitmap_page;  in ext4_mb_use_best_found()
1663 	get_page(ac->ac_bitmap_page);  in ext4_mb_use_best_found()
1664 	ac->ac_buddy_page = e4b->bd_buddy_page;  in ext4_mb_use_best_found()
1665 	get_page(ac->ac_buddy_page);  in ext4_mb_use_best_found()
1667 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {  in ext4_mb_use_best_found()
1668 		spin_lock(&sbi->s_md_lock);  in ext4_mb_use_best_found()
1669 		sbi->s_mb_last_group = ac->ac_f_ex.fe_group;  in ext4_mb_use_best_found()
1670 		sbi->s_mb_last_start = ac->ac_f_ex.fe_start;  in ext4_mb_use_best_found()
1671 		spin_unlock(&sbi->s_md_lock);  in ext4_mb_use_best_found()
1683 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_check_limits()
1684 	struct ext4_free_extent *bex = &ac->ac_b_ex;  in ext4_mb_check_limits()
1685 	struct ext4_free_extent *gex = &ac->ac_g_ex;  in ext4_mb_check_limits()
1689 	if (ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_check_limits()
1694 	if (ac->ac_found > sbi->s_mb_max_to_scan &&  in ext4_mb_check_limits()
1695 			!(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_check_limits()
1696 		ac->ac_status = AC_STATUS_BREAK;  in ext4_mb_check_limits()
1703 	if (bex->fe_len < gex->fe_len)  in ext4_mb_check_limits()
1706 	if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan)  in ext4_mb_check_limits()
1707 			&& bex->fe_group == e4b->bd_group) {  in ext4_mb_check_limits()
1708 		/* recheck chunk's availability - we don't know  in ext4_mb_check_limits()
1709 		 * when it was found (within this lock-unlock  in ext4_mb_check_limits()
1711 		max = mb_find_extent(e4b, bex->fe_start, gex->fe_len, &ex);  in ext4_mb_check_limits()
1712 		if (max >= gex->fe_len) {  in ext4_mb_check_limits()
1733 	struct ext4_free_extent *bex = &ac->ac_b_ex;  in ext4_mb_measure_extent()
1734 	struct ext4_free_extent *gex = &ac->ac_g_ex;  in ext4_mb_measure_extent()
1736 	BUG_ON(ex->fe_len <= 0);  in ext4_mb_measure_extent()
1737 	BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));  in ext4_mb_measure_extent()
1738 	BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));  in ext4_mb_measure_extent()
1739 	BUG_ON(ac->ac_status != AC_STATUS_CONTINUE);  in ext4_mb_measure_extent()
1741 	ac->ac_found++;  in ext4_mb_measure_extent()
1744 	 * The special case - take what you catch first  in ext4_mb_measure_extent()
1746 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_measure_extent()
1755 	if (ex->fe_len == gex->fe_len) {  in ext4_mb_measure_extent()
1764 	if (bex->fe_len == 0) {  in ext4_mb_measure_extent()
1772 	if (bex->fe_len < gex->fe_len) {  in ext4_mb_measure_extent()
1775 		if (ex->fe_len > bex->fe_len)  in ext4_mb_measure_extent()
1777 	} else if (ex->fe_len > gex->fe_len) {  in ext4_mb_measure_extent()
1781 		if (ex->fe_len < bex->fe_len)  in ext4_mb_measure_extent()
1792 	struct ext4_free_extent ex = ac->ac_b_ex;  in ext4_mb_try_best_found()
1798 	err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);  in ext4_mb_try_best_found()
1802 	ext4_lock_group(ac->ac_sb, group);  in ext4_mb_try_best_found()
1806 		ac->ac_b_ex = ex;  in ext4_mb_try_best_found()
1810 	ext4_unlock_group(ac->ac_sb, group);  in ext4_mb_try_best_found()
1820 	ext4_group_t group = ac->ac_g_ex.fe_group;  in ext4_mb_find_by_goal()
1823 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_find_by_goal()
1824 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1827 	if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL))  in ext4_mb_find_by_goal()
1829 	if (grp->bb_free == 0)  in ext4_mb_find_by_goal()
1832 	err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);  in ext4_mb_find_by_goal()
1836 	if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) {  in ext4_mb_find_by_goal()
1841 	ext4_lock_group(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1842 	max = mb_find_extent(e4b, ac->ac_g_ex.fe_start,  in ext4_mb_find_by_goal()
1843 			     ac->ac_g_ex.fe_len, &ex);  in ext4_mb_find_by_goal()
1846 	if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) {  in ext4_mb_find_by_goal()
1849 		start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) +  in ext4_mb_find_by_goal()
1851 		/* use do_div to get remainder (would be 64-bit modulo) */  in ext4_mb_find_by_goal()
1852 		if (do_div(start, sbi->s_stripe) == 0) {  in ext4_mb_find_by_goal()
1853 			ac->ac_found++;  in ext4_mb_find_by_goal()
1854 			ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1857 	} else if (max >= ac->ac_g_ex.fe_len) {  in ext4_mb_find_by_goal()
1859 		BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);  in ext4_mb_find_by_goal()
1860 		BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);  in ext4_mb_find_by_goal()
1861 		ac->ac_found++;  in ext4_mb_find_by_goal()
1862 		ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1864 	} else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) {  in ext4_mb_find_by_goal()
1868 		BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);  in ext4_mb_find_by_goal()
1869 		BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);  in ext4_mb_find_by_goal()
1870 		ac->ac_found++;  in ext4_mb_find_by_goal()
1871 		ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1874 	ext4_unlock_group(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1888 	struct super_block *sb = ac->ac_sb;  in ext4_mb_simple_scan_group()
1889 	struct ext4_group_info *grp = e4b->bd_info;  in ext4_mb_simple_scan_group()
1895 	BUG_ON(ac->ac_2order <= 0);  in ext4_mb_simple_scan_group()
1896 	for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) {  in ext4_mb_simple_scan_group()
1897 		if (grp->bb_counters[i] == 0)  in ext4_mb_simple_scan_group()
1905 			ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0,  in ext4_mb_simple_scan_group()
1907 				grp->bb_counters[i], i);  in ext4_mb_simple_scan_group()
1908 			ext4_mark_group_bitmap_corrupted(ac->ac_sb,  in ext4_mb_simple_scan_group()
1909 					 e4b->bd_group,  in ext4_mb_simple_scan_group()
1913 		ac->ac_found++;  in ext4_mb_simple_scan_group()
1915 		ac->ac_b_ex.fe_len = 1 << i;  in ext4_mb_simple_scan_group()
1916 		ac->ac_b_ex.fe_start = k << i;  in ext4_mb_simple_scan_group()
1917 		ac->ac_b_ex.fe_group = e4b->bd_group;  in ext4_mb_simple_scan_group()
1921 		BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len);  in ext4_mb_simple_scan_group()
1923 		if (EXT4_SB(sb)->s_mb_stats)  in ext4_mb_simple_scan_group()
1924 			atomic_inc(&EXT4_SB(sb)->s_bal_2orders);  in ext4_mb_simple_scan_group()
1939 	struct super_block *sb = ac->ac_sb;  in ext4_mb_complex_scan_group()
1940 	void *bitmap = e4b->bd_bitmap;  in ext4_mb_complex_scan_group()
1945 	free = e4b->bd_info->bb_free;  in ext4_mb_complex_scan_group()
1949 	i = e4b->bd_info->bb_first_free;  in ext4_mb_complex_scan_group()
1951 	while (free && ac->ac_status == AC_STATUS_CONTINUE) {  in ext4_mb_complex_scan_group()
1960 			ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,  in ext4_mb_complex_scan_group()
1964 			ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in ext4_mb_complex_scan_group()
1969 		mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex);  in ext4_mb_complex_scan_group()
1973 			ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,  in ext4_mb_complex_scan_group()
1977 			ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in ext4_mb_complex_scan_group()
1990 		free -= ex.fe_len;  in ext4_mb_complex_scan_group()
1998  * we try to find stripe-aligned chunks for stripe-size-multiple requests
2004 	struct super_block *sb = ac->ac_sb;  in ext4_mb_scan_aligned()
2006 	void *bitmap = e4b->bd_bitmap;  in ext4_mb_scan_aligned()
2013 	BUG_ON(sbi->s_stripe == 0);  in ext4_mb_scan_aligned()
2015 	/* find first stripe-aligned block in group */  in ext4_mb_scan_aligned()
2016 	first_group_block = ext4_group_first_block_no(sb, e4b->bd_group);  in ext4_mb_scan_aligned()
2018 	a = first_group_block + sbi->s_stripe - 1;  in ext4_mb_scan_aligned()
2019 	do_div(a, sbi->s_stripe);  in ext4_mb_scan_aligned()
2020 	i = (a * sbi->s_stripe) - first_group_block;  in ext4_mb_scan_aligned()
2024 			max = mb_find_extent(e4b, i, sbi->s_stripe, &ex);  in ext4_mb_scan_aligned()
2025 			if (max >= sbi->s_stripe) {  in ext4_mb_scan_aligned()
2026 				ac->ac_found++;  in ext4_mb_scan_aligned()
2028 				ac->ac_b_ex = ex;  in ext4_mb_scan_aligned()
2033 		i += sbi->s_stripe;  in ext4_mb_scan_aligned()
2047 	int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb));  in ext4_mb_good_group()
2048 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_good_group()
2052 	free = grp->bb_free;  in ext4_mb_good_group()
2055 	if (cr <= 2 && free < ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2063 		int ret = ext4_mb_init_group(ac->ac_sb, group, GFP_NOFS);  in ext4_mb_good_group()
2068 	fragments = grp->bb_fragments;  in ext4_mb_good_group()
2074 		BUG_ON(ac->ac_2order == 0);  in ext4_mb_good_group()
2077 		if ((ac->ac_flags & EXT4_MB_HINT_DATA) &&  in ext4_mb_good_group()
2082 		if ((ac->ac_2order > ac->ac_sb->s_blocksize_bits+1) ||  in ext4_mb_good_group()
2083 		    (free / fragments) >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2086 		if (grp->bb_largest_free_order < ac->ac_2order)  in ext4_mb_good_group()
2091 		if ((free / fragments) >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2095 		if (free >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2117 	sb = ac->ac_sb;  in ext4_mb_regular_allocator()
2120 	/* non-extent files are limited to low blocks/groups */  in ext4_mb_regular_allocator()
2121 	if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)))  in ext4_mb_regular_allocator()
2122 		ngroups = sbi->s_blockfile_groups;  in ext4_mb_regular_allocator()
2124 	BUG_ON(ac->ac_status == AC_STATUS_FOUND);  in ext4_mb_regular_allocator()
2128 	if (err || ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_regular_allocator()
2131 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_regular_allocator()
2135 	 * ac->ac2_order is set only if the fe_len is a power of 2  in ext4_mb_regular_allocator()
2139 	i = fls(ac->ac_g_ex.fe_len);  in ext4_mb_regular_allocator()
2140 	ac->ac_2order = 0;  in ext4_mb_regular_allocator()
2145 	 * We also support searching for power-of-two requests only for  in ext4_mb_regular_allocator()
2148 	if (i >= sbi->s_mb_order2_reqs && i <= sb->s_blocksize_bits + 2) {  in ext4_mb_regular_allocator()
2152 		if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0)  in ext4_mb_regular_allocator()
2153 			ac->ac_2order = array_index_nospec(i - 1,  in ext4_mb_regular_allocator()
2154 							   sb->s_blocksize_bits + 2);  in ext4_mb_regular_allocator()
2158 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {  in ext4_mb_regular_allocator()
2160 		spin_lock(&sbi->s_md_lock);  in ext4_mb_regular_allocator()
2161 		ac->ac_g_ex.fe_group = sbi->s_mb_last_group;  in ext4_mb_regular_allocator()
2162 		ac->ac_g_ex.fe_start = sbi->s_mb_last_start;  in ext4_mb_regular_allocator()
2163 		spin_unlock(&sbi->s_md_lock);  in ext4_mb_regular_allocator()
2166 	/* Let's just scan groups to find more-less suitable blocks */  in ext4_mb_regular_allocator()
2167 	cr = ac->ac_2order ? 0 : 1;  in ext4_mb_regular_allocator()
2173 	for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) {  in ext4_mb_regular_allocator()
2174 		ac->ac_criteria = cr;  in ext4_mb_regular_allocator()
2179 		group = ac->ac_g_ex.fe_group;  in ext4_mb_regular_allocator()
2185 			 * Artificially restricted ngroups for non-extent  in ext4_mb_regular_allocator()
2218 			ac->ac_groups_scanned++;  in ext4_mb_regular_allocator()
2221 			else if (cr == 1 && sbi->s_stripe &&  in ext4_mb_regular_allocator()
2222 					!(ac->ac_g_ex.fe_len % sbi->s_stripe))  in ext4_mb_regular_allocator()
2230 			if (ac->ac_status != AC_STATUS_CONTINUE)  in ext4_mb_regular_allocator()
2235 	if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND &&  in ext4_mb_regular_allocator()
2236 	    !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_regular_allocator()
2243 		if (ac->ac_status != AC_STATUS_FOUND) {  in ext4_mb_regular_allocator()
2248 			printk(KERN_DEBUG "EXT4-fs: someone won our chunk\n");  in ext4_mb_regular_allocator()
2250 			ac->ac_b_ex.fe_group = 0;  in ext4_mb_regular_allocator()
2251 			ac->ac_b_ex.fe_start = 0;  in ext4_mb_regular_allocator()
2252 			ac->ac_b_ex.fe_len = 0;  in ext4_mb_regular_allocator()
2253 			ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_regular_allocator()
2254 			ac->ac_flags |= EXT4_MB_HINT_FIRST;  in ext4_mb_regular_allocator()
2256 			atomic_inc(&sbi->s_mb_lost_chunks);  in ext4_mb_regular_allocator()
2261 	if (!err && ac->ac_status != AC_STATUS_FOUND && first_err)  in ext4_mb_regular_allocator()
2268 	struct super_block *sb = PDE_DATA(file_inode(seq->file));  in ext4_mb_seq_groups_start()
2279 	struct super_block *sb = PDE_DATA(file_inode(seq->file));  in ext4_mb_seq_groups_next()
2291 	struct super_block *sb = PDE_DATA(file_inode(seq->file));  in ext4_mb_seq_groups_show()
2298 					     sb->s_blocksize_bits,  in ext4_mb_seq_groups_show()
2305 	group--;  in ext4_mb_seq_groups_show()
2319 			seq_printf(seq, "#%-5u: I/O error\n", group);  in ext4_mb_seq_groups_show()
2330 	seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free,  in ext4_mb_seq_groups_show()
2333 		seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ?  in ext4_mb_seq_groups_show()
2353 	int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE;  in get_groupinfo_cache()
2361  * Allocate the top-level s_group_info array for the specified number
2370 	size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >>  in ext4_mb_alloc_groupinfo()
2372 	if (size <= sbi->s_group_info_size)  in ext4_mb_alloc_groupinfo()
2375 	size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size);  in ext4_mb_alloc_groupinfo()
2379 		return -ENOMEM;  in ext4_mb_alloc_groupinfo()
2382 	old_groupinfo = rcu_dereference(sbi->s_group_info);  in ext4_mb_alloc_groupinfo()
2385 		       sbi->s_group_info_size * sizeof(*sbi->s_group_info));  in ext4_mb_alloc_groupinfo()
2387 	rcu_assign_pointer(sbi->s_group_info, new_groupinfo);  in ext4_mb_alloc_groupinfo()
2388 	sbi->s_group_info_size = size / sizeof(*sbi->s_group_info);  in ext4_mb_alloc_groupinfo()
2392 		   sbi->s_group_info_size);  in ext4_mb_alloc_groupinfo()
2405 	struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);  in ext4_mb_add_groupinfo()
2422 		rcu_dereference(sbi->s_group_info)[idx] = meta_group_info;  in ext4_mb_add_groupinfo()
2427 	i = group & (EXT4_DESC_PER_BLOCK(sb) - 1);  in ext4_mb_add_groupinfo()
2435 		&(meta_group_info[i]->bb_state));  in ext4_mb_add_groupinfo()
2442 	    (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {  in ext4_mb_add_groupinfo()
2443 		meta_group_info[i]->bb_free =  in ext4_mb_add_groupinfo()
2446 		meta_group_info[i]->bb_free =  in ext4_mb_add_groupinfo()
2450 	INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list);  in ext4_mb_add_groupinfo()
2451 	init_rwsem(&meta_group_info[i]->alloc_sem);  in ext4_mb_add_groupinfo()
2452 	meta_group_info[i]->bb_free_root = RB_ROOT;  in ext4_mb_add_groupinfo()
2453 	meta_group_info[i]->bb_largest_free_order = -1;  /* uninit */  in ext4_mb_add_groupinfo()
2458 		meta_group_info[i]->bb_bitmap =  in ext4_mb_add_groupinfo()
2459 			kmalloc(sb->s_blocksize, GFP_NOFS);  in ext4_mb_add_groupinfo()
2460 		BUG_ON(meta_group_info[i]->bb_bitmap == NULL);  in ext4_mb_add_groupinfo()
2463 		memcpy(meta_group_info[i]->bb_bitmap, bh->b_data,  in ext4_mb_add_groupinfo()
2464 			sb->s_blocksize);  in ext4_mb_add_groupinfo()
2477 		group_info = rcu_dereference(sbi->s_group_info);  in ext4_mb_add_groupinfo()
2483 	return -ENOMEM;  in ext4_mb_add_groupinfo()
2500 	sbi->s_buddy_cache = new_inode(sb);  in ext4_mb_init_backend()
2501 	if (sbi->s_buddy_cache == NULL) {  in ext4_mb_init_backend()
2505 	/* To avoid potentially colliding with an valid on-disk inode number,  in ext4_mb_init_backend()
2509 	sbi->s_buddy_cache->i_ino = EXT4_BAD_INO;  in ext4_mb_init_backend()
2510 	EXT4_I(sbi->s_buddy_cache)->i_disksize = 0;  in ext4_mb_init_backend()
2524 	cachep = get_groupinfo_cache(sb->s_blocksize_bits);  in ext4_mb_init_backend()
2525 	while (i-- > 0)  in ext4_mb_init_backend()
2527 	i = sbi->s_group_info_size;  in ext4_mb_init_backend()
2529 	group_info = rcu_dereference(sbi->s_group_info);  in ext4_mb_init_backend()
2530 	while (i-- > 0)  in ext4_mb_init_backend()
2533 	iput(sbi->s_buddy_cache);  in ext4_mb_init_backend()
2536 	kvfree(rcu_dereference(sbi->s_group_info));  in ext4_mb_init_backend()
2538 	return -ENOMEM;  in ext4_mb_init_backend()
2556 	int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE;  in ext4_groupinfo_create_slab()
2560 		return -EINVAL;  in ext4_groupinfo_create_slab()
2583 		       "EXT4-fs: no memory for groupinfo slab cache\n");  in ext4_groupinfo_create_slab()
2584 		return -ENOMEM;  in ext4_groupinfo_create_slab()
2598 	i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_offsets);  in ext4_mb_init()
2600 	sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL);  in ext4_mb_init()
2601 	if (sbi->s_mb_offsets == NULL) {  in ext4_mb_init()
2602 		ret = -ENOMEM;  in ext4_mb_init()
2606 	i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_maxs);  in ext4_mb_init()
2607 	sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL);  in ext4_mb_init()
2608 	if (sbi->s_mb_maxs == NULL) {  in ext4_mb_init()
2609 		ret = -ENOMEM;  in ext4_mb_init()
2613 	ret = ext4_groupinfo_create_slab(sb->s_blocksize);  in ext4_mb_init()
2618 	sbi->s_mb_maxs[0] = sb->s_blocksize << 3;  in ext4_mb_init()
2619 	sbi->s_mb_offsets[0] = 0;  in ext4_mb_init()
2623 	offset_incr = 1 << (sb->s_blocksize_bits - 1);  in ext4_mb_init()
2624 	max = sb->s_blocksize << 2;  in ext4_mb_init()
2626 		sbi->s_mb_offsets[i] = offset;  in ext4_mb_init()
2627 		sbi->s_mb_maxs[i] = max;  in ext4_mb_init()
2632 	} while (i <= sb->s_blocksize_bits + 1);  in ext4_mb_init()
2634 	spin_lock_init(&sbi->s_md_lock);  in ext4_mb_init()
2635 	spin_lock_init(&sbi->s_bal_lock);  in ext4_mb_init()
2636 	sbi->s_mb_free_pending = 0;  in ext4_mb_init()
2637 	INIT_LIST_HEAD(&sbi->s_freed_data_list);  in ext4_mb_init()
2639 	sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN;  in ext4_mb_init()
2640 	sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN;  in ext4_mb_init()
2641 	sbi->s_mb_stats = MB_DEFAULT_STATS;  in ext4_mb_init()
2642 	sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD;  in ext4_mb_init()
2643 	sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS;  in ext4_mb_init()
2656 	sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >>  in ext4_mb_init()
2657 				       sbi->s_cluster_bits, 32);  in ext4_mb_init()
2666 	if (sbi->s_stripe > 1) {  in ext4_mb_init()
2667 		sbi->s_mb_group_prealloc = roundup(  in ext4_mb_init()
2668 			sbi->s_mb_group_prealloc, sbi->s_stripe);  in ext4_mb_init()
2671 	sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group);  in ext4_mb_init()
2672 	if (sbi->s_locality_groups == NULL) {  in ext4_mb_init()
2673 		ret = -ENOMEM;  in ext4_mb_init()
2678 		lg = per_cpu_ptr(sbi->s_locality_groups, i);  in ext4_mb_init()
2679 		mutex_init(&lg->lg_mutex);  in ext4_mb_init()
2681 			INIT_LIST_HEAD(&lg->lg_prealloc_list[j]);  in ext4_mb_init()
2682 		spin_lock_init(&lg->lg_prealloc_lock);  in ext4_mb_init()
2693 	free_percpu(sbi->s_locality_groups);  in ext4_mb_init()
2694 	sbi->s_locality_groups = NULL;  in ext4_mb_init()
2696 	kfree(sbi->s_mb_offsets);  in ext4_mb_init()
2697 	sbi->s_mb_offsets = NULL;  in ext4_mb_init()
2698 	kfree(sbi->s_mb_maxs);  in ext4_mb_init()
2699 	sbi->s_mb_maxs = NULL;  in ext4_mb_init()
2710 	list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) {  in ext4_mb_cleanup_pa()
2712 		list_del(&pa->pa_group_list);  in ext4_mb_cleanup_pa()
2728 	struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);  in ext4_mb_release()
2730 	if (sbi->s_group_info) {  in ext4_mb_release()
2734 			kfree(grinfo->bb_bitmap);  in ext4_mb_release()
2742 				EXT4_DESC_PER_BLOCK(sb) - 1) >>  in ext4_mb_release()
2745 		group_info = rcu_dereference(sbi->s_group_info);  in ext4_mb_release()
2751 	kfree(sbi->s_mb_offsets);  in ext4_mb_release()
2752 	kfree(sbi->s_mb_maxs);  in ext4_mb_release()
2753 	iput(sbi->s_buddy_cache);  in ext4_mb_release()
2754 	if (sbi->s_mb_stats) {  in ext4_mb_release()
2757 				atomic_read(&sbi->s_bal_allocated),  in ext4_mb_release()
2758 				atomic_read(&sbi->s_bal_reqs),  in ext4_mb_release()
2759 				atomic_read(&sbi->s_bal_success));  in ext4_mb_release()
2763 				atomic_read(&sbi->s_bal_ex_scanned),  in ext4_mb_release()
2764 				atomic_read(&sbi->s_bal_goals),  in ext4_mb_release()
2765 				atomic_read(&sbi->s_bal_2orders),  in ext4_mb_release()
2766 				atomic_read(&sbi->s_bal_breaks),  in ext4_mb_release()
2767 				atomic_read(&sbi->s_mb_lost_chunks));  in ext4_mb_release()
2770 				sbi->s_mb_buddies_generated,  in ext4_mb_release()
2771 				sbi->s_mb_generation_time);  in ext4_mb_release()
2774 				atomic_read(&sbi->s_mb_preallocated),  in ext4_mb_release()
2775 				atomic_read(&sbi->s_mb_discarded));  in ext4_mb_release()
2778 	free_percpu(sbi->s_locality_groups);  in ext4_mb_release()
2795 		return __blkdev_issue_discard(sb->s_bdev,  in ext4_issue_discard()
2796 			(sector_t)discard_block << (sb->s_blocksize_bits - 9),  in ext4_issue_discard()
2797 			(sector_t)count << (sb->s_blocksize_bits - 9),  in ext4_issue_discard()
2811 		 entry->efd_count, entry->efd_group, entry);  in ext4_free_data_in_buddy()
2813 	err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b);  in ext4_free_data_in_buddy()
2817 	spin_lock(&EXT4_SB(sb)->s_md_lock);  in ext4_free_data_in_buddy()
2818 	EXT4_SB(sb)->s_mb_free_pending -= entry->efd_count;  in ext4_free_data_in_buddy()
2819 	spin_unlock(&EXT4_SB(sb)->s_md_lock);  in ext4_free_data_in_buddy()
2823 	count += entry->efd_count;  in ext4_free_data_in_buddy()
2825 	ext4_lock_group(sb, entry->efd_group);  in ext4_free_data_in_buddy()
2827 	rb_erase(&entry->efd_node, &(db->bb_free_root));  in ext4_free_data_in_buddy()
2828 	mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count);  in ext4_free_data_in_buddy()
2833 	 * If the volume is mounted with -o discard, online discard  in ext4_free_data_in_buddy()
2839 	if (!db->bb_free_root.rb_node) {  in ext4_free_data_in_buddy()
2846 	ext4_unlock_group(sb, entry->efd_group);  in ext4_free_data_in_buddy()
2868 	spin_lock(&sbi->s_md_lock);  in ext4_process_freed_data()
2869 	list_for_each_entry(entry, &sbi->s_freed_data_list, efd_list) {  in ext4_process_freed_data()
2870 		if (entry->efd_tid != commit_tid)  in ext4_process_freed_data()
2872 		cut_pos = &entry->efd_list;  in ext4_process_freed_data()
2875 		list_cut_position(&freed_data_list, &sbi->s_freed_data_list,  in ext4_process_freed_data()
2877 	spin_unlock(&sbi->s_md_lock);  in ext4_process_freed_data()
2881 			err = ext4_issue_discard(sb, entry->efd_group,  in ext4_process_freed_data()
2882 						 entry->efd_start_cluster,  in ext4_process_freed_data()
2883 						 entry->efd_count,  in ext4_process_freed_data()
2885 			if (err && err != -EOPNOTSUPP) {  in ext4_process_freed_data()
2888 					 " with %d", entry->efd_group,  in ext4_process_freed_data()
2889 					 entry->efd_start_cluster,  in ext4_process_freed_data()
2890 					 entry->efd_count, err);  in ext4_process_freed_data()
2891 			} else if (err == -EOPNOTSUPP)  in ext4_process_freed_data()
2910 		return -ENOMEM;  in ext4_init_mballoc()
2916 		return -ENOMEM;  in ext4_init_mballoc()
2924 		return -ENOMEM;  in ext4_init_mballoc()
2944  * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
2959 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_mark_diskspace_used()
2960 	BUG_ON(ac->ac_b_ex.fe_len <= 0);  in ext4_mb_mark_diskspace_used()
2962 	sb = ac->ac_sb;  in ext4_mb_mark_diskspace_used()
2965 	bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
2977 	err = -EIO;  in ext4_mb_mark_diskspace_used()
2978 	gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh);  in ext4_mb_mark_diskspace_used()
2982 	ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group,  in ext4_mb_mark_diskspace_used()
2990 	block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_mark_diskspace_used()
2992 	len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
2994 		ext4_error(sb, "Allocating blocks %llu-%llu which overlap "  in ext4_mb_mark_diskspace_used()
3000 		ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3001 		ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,  in ext4_mb_mark_diskspace_used()
3002 			      ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3003 		ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3006 			err = -EFSCORRUPTED;  in ext4_mb_mark_diskspace_used()
3010 	ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3014 		for (i = 0; i < ac->ac_b_ex.fe_len; i++) {  in ext4_mb_mark_diskspace_used()
3015 			BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i,  in ext4_mb_mark_diskspace_used()
3016 						bitmap_bh->b_data));  in ext4_mb_mark_diskspace_used()
3020 	ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,  in ext4_mb_mark_diskspace_used()
3021 		      ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3023 	    (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {  in ext4_mb_mark_diskspace_used()
3024 		gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);  in ext4_mb_mark_diskspace_used()
3027 						ac->ac_b_ex.fe_group, gdp));  in ext4_mb_mark_diskspace_used()
3029 	len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len;  in ext4_mb_mark_diskspace_used()
3031 	ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh);  in ext4_mb_mark_diskspace_used()
3032 	ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp);  in ext4_mb_mark_diskspace_used()
3034 	ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3035 	percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3039 	if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED))  in ext4_mb_mark_diskspace_used()
3041 		percpu_counter_sub(&sbi->s_dirtyclusters_counter,  in ext4_mb_mark_diskspace_used()
3044 	if (sbi->s_log_groups_per_flex) {  in ext4_mb_mark_diskspace_used()
3046 							  ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3047 		atomic64_sub(ac->ac_b_ex.fe_len,  in ext4_mb_mark_diskspace_used()
3049 						  flex_group)->free_clusters);  in ext4_mb_mark_diskspace_used()
3063  * here we normalize request for locality group
3073 	struct super_block *sb = ac->ac_sb;  in ext4_mb_normalize_group_request()
3074 	struct ext4_locality_group *lg = ac->ac_lg;  in ext4_mb_normalize_group_request()
3077 	ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc;  in ext4_mb_normalize_group_request()
3078 	mb_debug(1, "#%u: goal %u blocks for locality group\n",  in ext4_mb_normalize_group_request()
3079 		current->pid, ac->ac_g_ex.fe_len);  in ext4_mb_normalize_group_request()
3090 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_normalize_request()
3096 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_normalize_request()
3101 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_normalize_request()
3105 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_normalize_request()
3110 	if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC)  in ext4_mb_normalize_request()
3113 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) {  in ext4_mb_normalize_request()
3118 	bsbits = ac->ac_sb->s_blocksize_bits;  in ext4_mb_normalize_request()
3122 	size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len);  in ext4_mb_normalize_request()
3124 	if (size < i_size_read(ac->ac_inode))  in ext4_mb_normalize_request()
3125 		size = i_size_read(ac->ac_inode);  in ext4_mb_normalize_request()
3152 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3153 						(21 - bsbits)) << 21;  in ext4_mb_normalize_request()
3156 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3157 							(22 - bsbits)) << 22;  in ext4_mb_normalize_request()
3159 	} else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len,  in ext4_mb_normalize_request()
3161 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3162 							(23 - bsbits)) << 23;  in ext4_mb_normalize_request()
3165 		start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits;  in ext4_mb_normalize_request()
3166 		size	  = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3167 					      ac->ac_o_ex.fe_len) << bsbits;  in ext4_mb_normalize_request()
3173 	if (ar->pleft && start <= ar->lleft) {  in ext4_mb_normalize_request()
3174 		size -= ar->lleft + 1 - start;  in ext4_mb_normalize_request()
3175 		start = ar->lleft + 1;  in ext4_mb_normalize_request()
3177 	if (ar->pright && start + size - 1 >= ar->lright)  in ext4_mb_normalize_request()
3178 		size -= start + size - ar->lright;  in ext4_mb_normalize_request()
3184 	if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb))  in ext4_mb_normalize_request()
3185 		size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb);  in ext4_mb_normalize_request()
3191 	list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {  in ext4_mb_normalize_request()
3194 		if (pa->pa_deleted)  in ext4_mb_normalize_request()
3196 		spin_lock(&pa->pa_lock);  in ext4_mb_normalize_request()
3197 		if (pa->pa_deleted) {  in ext4_mb_normalize_request()
3198 			spin_unlock(&pa->pa_lock);  in ext4_mb_normalize_request()
3202 		pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3203 						  pa->pa_len);  in ext4_mb_normalize_request()
3206 		BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end ||  in ext4_mb_normalize_request()
3207 			ac->ac_o_ex.fe_logical < pa->pa_lstart));  in ext4_mb_normalize_request()
3210 		if (pa->pa_lstart >= end || pa_end <= start) {  in ext4_mb_normalize_request()
3211 			spin_unlock(&pa->pa_lock);  in ext4_mb_normalize_request()
3214 		BUG_ON(pa->pa_lstart <= start && pa_end >= end);  in ext4_mb_normalize_request()
3217 		if (pa_end <= ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3220 		} else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3221 			BUG_ON(pa->pa_lstart > end);  in ext4_mb_normalize_request()
3222 			end = pa->pa_lstart;  in ext4_mb_normalize_request()
3224 		spin_unlock(&pa->pa_lock);  in ext4_mb_normalize_request()
3227 	size = end - start;  in ext4_mb_normalize_request()
3231 	list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {  in ext4_mb_normalize_request()
3234 		spin_lock(&pa->pa_lock);  in ext4_mb_normalize_request()
3235 		if (pa->pa_deleted == 0) {  in ext4_mb_normalize_request()
3236 			pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3237 							  pa->pa_len);  in ext4_mb_normalize_request()
3238 			BUG_ON(!(start >= pa_end || end <= pa->pa_lstart));  in ext4_mb_normalize_request()
3240 		spin_unlock(&pa->pa_lock);  in ext4_mb_normalize_request()
3244 	if (start + size <= ac->ac_o_ex.fe_logical &&  in ext4_mb_normalize_request()
3245 			start > ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3246 		ext4_msg(ac->ac_sb, KERN_ERR,  in ext4_mb_normalize_request()
3249 			 (unsigned long) ac->ac_o_ex.fe_logical);  in ext4_mb_normalize_request()
3252 	BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb));  in ext4_mb_normalize_request()
3258 	ac->ac_g_ex.fe_logical = start;  in ext4_mb_normalize_request()
3259 	ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size);  in ext4_mb_normalize_request()
3262 	if (ar->pright && (ar->lright == (start + size))) {  in ext4_mb_normalize_request()
3264 		ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size,  in ext4_mb_normalize_request()
3265 						&ac->ac_f_ex.fe_group,  in ext4_mb_normalize_request()
3266 						&ac->ac_f_ex.fe_start);  in ext4_mb_normalize_request()
3267 		ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;  in ext4_mb_normalize_request()
3269 	if (ar->pleft && (ar->lleft + 1 == start)) {  in ext4_mb_normalize_request()
3271 		ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1,  in ext4_mb_normalize_request()
3272 						&ac->ac_f_ex.fe_group,  in ext4_mb_normalize_request()
3273 						&ac->ac_f_ex.fe_start);  in ext4_mb_normalize_request()
3274 		ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;  in ext4_mb_normalize_request()
3283 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_collect_stats()
3285 	if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) {  in ext4_mb_collect_stats()
3286 		atomic_inc(&sbi->s_bal_reqs);  in ext4_mb_collect_stats()
3287 		atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);  in ext4_mb_collect_stats()
3288 		if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len)  in ext4_mb_collect_stats()
3289 			atomic_inc(&sbi->s_bal_success);  in ext4_mb_collect_stats()
3290 		atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);  in ext4_mb_collect_stats()
3291 		if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&  in ext4_mb_collect_stats()
3292 				ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)  in ext4_mb_collect_stats()
3293 			atomic_inc(&sbi->s_bal_goals);  in ext4_mb_collect_stats()
3294 		if (ac->ac_found > sbi->s_mb_max_to_scan)  in ext4_mb_collect_stats()
3295 			atomic_inc(&sbi->s_bal_breaks);  in ext4_mb_collect_stats()
3298 	if (ac->ac_op == EXT4_MB_HISTORY_ALLOC)  in ext4_mb_collect_stats()
3308  * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
3312 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_discard_allocated_blocks()
3317 		if (ac->ac_f_ex.fe_len == 0)  in ext4_discard_allocated_blocks()
3319 		err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b);  in ext4_discard_allocated_blocks()
3329 		ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group);  in ext4_discard_allocated_blocks()
3330 		mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start,  in ext4_discard_allocated_blocks()
3331 			       ac->ac_f_ex.fe_len);  in ext4_discard_allocated_blocks()
3332 		ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group);  in ext4_discard_allocated_blocks()
3336 	if (pa->pa_type == MB_INODE_PA)  in ext4_discard_allocated_blocks()
3337 		pa->pa_free += ac->ac_b_ex.fe_len;  in ext4_discard_allocated_blocks()
3346 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_inode_pa()
3352 	start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart);  in ext4_mb_use_inode_pa()
3353 	end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len),  in ext4_mb_use_inode_pa()
3354 		  start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len));  in ext4_mb_use_inode_pa()
3355 	len = EXT4_NUM_B2C(sbi, end - start);  in ext4_mb_use_inode_pa()
3356 	ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group,  in ext4_mb_use_inode_pa()
3357 					&ac->ac_b_ex.fe_start);  in ext4_mb_use_inode_pa()
3358 	ac->ac_b_ex.fe_len = len;  in ext4_mb_use_inode_pa()
3359 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_inode_pa()
3360 	ac->ac_pa = pa;  in ext4_mb_use_inode_pa()
3362 	BUG_ON(start < pa->pa_pstart);  in ext4_mb_use_inode_pa()
3363 	BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len));  in ext4_mb_use_inode_pa()
3364 	BUG_ON(pa->pa_free < len);  in ext4_mb_use_inode_pa()
3365 	pa->pa_free -= len;  in ext4_mb_use_inode_pa()
3371  * use blocks preallocated to locality group
3376 	unsigned int len = ac->ac_o_ex.fe_len;  in ext4_mb_use_group_pa()
3378 	ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart,  in ext4_mb_use_group_pa()
3379 					&ac->ac_b_ex.fe_group,  in ext4_mb_use_group_pa()
3380 					&ac->ac_b_ex.fe_start);  in ext4_mb_use_group_pa()
3381 	ac->ac_b_ex.fe_len = len;  in ext4_mb_use_group_pa()
3382 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_group_pa()
3383 	ac->ac_pa = pa;  in ext4_mb_use_group_pa()
3388 	 * in on-disk bitmap -- see ext4_mb_release_context()  in ext4_mb_use_group_pa()
3391 	mb_debug(1, "use %u/%u from group pa %p\n", pa->pa_lstart-len, len, pa);  in ext4_mb_use_group_pa()
3408 		atomic_inc(&pa->pa_count);  in ext4_mb_check_group_pa()
3411 	cur_distance = abs(goal_block - cpa->pa_pstart);  in ext4_mb_check_group_pa()
3412 	new_distance = abs(goal_block - pa->pa_pstart);  in ext4_mb_check_group_pa()
3418 	atomic_dec(&cpa->pa_count);  in ext4_mb_check_group_pa()
3419 	atomic_inc(&pa->pa_count);  in ext4_mb_check_group_pa()
3429 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_preallocated()
3431 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_use_preallocated()
3437 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_use_preallocated()
3440 	/* first, try per-file preallocation */  in ext4_mb_use_preallocated()
3442 	list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {  in ext4_mb_use_preallocated()
3446 		if (ac->ac_o_ex.fe_logical < pa->pa_lstart ||  in ext4_mb_use_preallocated()
3447 		    ac->ac_o_ex.fe_logical >= (pa->pa_lstart +  in ext4_mb_use_preallocated()
3448 					       EXT4_C2B(sbi, pa->pa_len)))  in ext4_mb_use_preallocated()
3451 		/* non-extent files can't have physical blocks past 2^32 */  in ext4_mb_use_preallocated()
3452 		if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) &&  in ext4_mb_use_preallocated()
3453 		    (pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len) >  in ext4_mb_use_preallocated()
3458 		spin_lock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3459 		if (pa->pa_deleted == 0 && pa->pa_free) {  in ext4_mb_use_preallocated()
3460 			atomic_inc(&pa->pa_count);  in ext4_mb_use_preallocated()
3462 			spin_unlock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3463 			ac->ac_criteria = 10;  in ext4_mb_use_preallocated()
3467 		spin_unlock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3472 	if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC))  in ext4_mb_use_preallocated()
3475 	/* inode may have no locality group for some reason */  in ext4_mb_use_preallocated()
3476 	lg = ac->ac_lg;  in ext4_mb_use_preallocated()
3479 	order  = fls(ac->ac_o_ex.fe_len) - 1;  in ext4_mb_use_preallocated()
3480 	if (order > PREALLOC_TB_SIZE - 1)  in ext4_mb_use_preallocated()
3482 		order = PREALLOC_TB_SIZE - 1;  in ext4_mb_use_preallocated()
3484 	goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex);  in ext4_mb_use_preallocated()
3491 		list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[i],  in ext4_mb_use_preallocated()
3493 			spin_lock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3494 			if (pa->pa_deleted == 0 &&  in ext4_mb_use_preallocated()
3495 					pa->pa_free >= ac->ac_o_ex.fe_len) {  in ext4_mb_use_preallocated()
3500 			spin_unlock(&pa->pa_lock);  in ext4_mb_use_preallocated()
3506 		ac->ac_criteria = 20;  in ext4_mb_use_preallocated()
3514  * but not yet committed and marks them used in in-core bitmap.
3526 	n = rb_first(&(grp->bb_free_root));  in ext4_mb_generate_from_freelist()
3530 		ext4_set_bits(bitmap, entry->efd_start_cluster, entry->efd_count);  in ext4_mb_generate_from_freelist()
3538  * used in in-core bitmap. buddy must be generated from this bitmap
3561 	list_for_each(cur, &grp->bb_prealloc_list) {  in ext4_mb_generate_from_pa()
3563 		spin_lock(&pa->pa_lock);  in ext4_mb_generate_from_pa()
3564 		ext4_get_group_no_and_offset(sb, pa->pa_pstart,  in ext4_mb_generate_from_pa()
3566 		len = pa->pa_len;  in ext4_mb_generate_from_pa()
3567 		spin_unlock(&pa->pa_lock);  in ext4_mb_generate_from_pa()
3582 	BUG_ON(atomic_read(&pa->pa_count));  in ext4_mb_pa_callback()
3583 	BUG_ON(pa->pa_deleted == 0);  in ext4_mb_pa_callback()
3598 	spin_lock(&pa->pa_lock);  in ext4_mb_put_pa()
3599 	if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) {  in ext4_mb_put_pa()
3600 		spin_unlock(&pa->pa_lock);  in ext4_mb_put_pa()
3604 	if (pa->pa_deleted == 1) {  in ext4_mb_put_pa()
3605 		spin_unlock(&pa->pa_lock);  in ext4_mb_put_pa()
3609 	pa->pa_deleted = 1;  in ext4_mb_put_pa()
3610 	spin_unlock(&pa->pa_lock);  in ext4_mb_put_pa()
3612 	grp_blk = pa->pa_pstart;  in ext4_mb_put_pa()
3614 	 * If doing group-based preallocation, pa_pstart may be in the  in ext4_mb_put_pa()
3617 	if (pa->pa_type == MB_GROUP_PA)  in ext4_mb_put_pa()
3618 		grp_blk--;  in ext4_mb_put_pa()
3627 	 *  copy on-disk bitmap to buddy  in ext4_mb_put_pa()
3628 	 *  					mark B in on-disk bitmap  in ext4_mb_put_pa()
3637 	list_del(&pa->pa_group_list);  in ext4_mb_put_pa()
3640 	spin_lock(pa->pa_obj_lock);  in ext4_mb_put_pa()
3641 	list_del_rcu(&pa->pa_inode_list);  in ext4_mb_put_pa()
3642 	spin_unlock(pa->pa_obj_lock);  in ext4_mb_put_pa()
3644 	call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_mb_put_pa()
3653 	struct super_block *sb = ac->ac_sb;  in ext4_mb_new_inode_pa()
3660 	BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
3661 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_new_inode_pa()
3662 	BUG_ON(!S_ISREG(ac->ac_inode->i_mode));  in ext4_mb_new_inode_pa()
3666 		return -ENOMEM;  in ext4_mb_new_inode_pa()
3668 	if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) {  in ext4_mb_new_inode_pa()
3677 		BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical);  in ext4_mb_new_inode_pa()
3678 		BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len);  in ext4_mb_new_inode_pa()
3683 		winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical;  in ext4_mb_new_inode_pa()
3686 		wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len);  in ext4_mb_new_inode_pa()
3691 		offs = ac->ac_o_ex.fe_logical %  in ext4_mb_new_inode_pa()
3692 			EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
3696 		ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical -  in ext4_mb_new_inode_pa()
3698 		BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical);  in ext4_mb_new_inode_pa()
3699 		BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
3704 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_new_inode_pa()
3706 	pa->pa_lstart = ac->ac_b_ex.fe_logical;  in ext4_mb_new_inode_pa()
3707 	pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_inode_pa()
3708 	pa->pa_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_inode_pa()
3709 	pa->pa_free = pa->pa_len;  in ext4_mb_new_inode_pa()
3710 	atomic_set(&pa->pa_count, 1);  in ext4_mb_new_inode_pa()
3711 	spin_lock_init(&pa->pa_lock);  in ext4_mb_new_inode_pa()
3712 	INIT_LIST_HEAD(&pa->pa_inode_list);  in ext4_mb_new_inode_pa()
3713 	INIT_LIST_HEAD(&pa->pa_group_list);  in ext4_mb_new_inode_pa()
3714 	pa->pa_deleted = 0;  in ext4_mb_new_inode_pa()
3715 	pa->pa_type = MB_INODE_PA;  in ext4_mb_new_inode_pa()
3718 			pa->pa_pstart, pa->pa_len, pa->pa_lstart);  in ext4_mb_new_inode_pa()
3722 	atomic_add(pa->pa_free, &sbi->s_mb_preallocated);  in ext4_mb_new_inode_pa()
3724 	ei = EXT4_I(ac->ac_inode);  in ext4_mb_new_inode_pa()
3725 	grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_inode_pa()
3727 	pa->pa_obj_lock = &ei->i_prealloc_lock;  in ext4_mb_new_inode_pa()
3728 	pa->pa_inode = ac->ac_inode;  in ext4_mb_new_inode_pa()
3730 	ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_inode_pa()
3731 	list_add(&pa->pa_group_list, &grp->bb_prealloc_list);  in ext4_mb_new_inode_pa()
3732 	ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_inode_pa()
3734 	spin_lock(pa->pa_obj_lock);  in ext4_mb_new_inode_pa()
3735 	list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list);  in ext4_mb_new_inode_pa()
3736 	spin_unlock(pa->pa_obj_lock);  in ext4_mb_new_inode_pa()
3742  * creates new preallocated space for locality group inodes belongs to
3747 	struct super_block *sb = ac->ac_sb;  in ext4_mb_new_group_pa()
3753 	BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);  in ext4_mb_new_group_pa()
3754 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_new_group_pa()
3755 	BUG_ON(!S_ISREG(ac->ac_inode->i_mode));  in ext4_mb_new_group_pa()
3760 		return -ENOMEM;  in ext4_mb_new_group_pa()
3764 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_new_group_pa()
3766 	pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_group_pa()
3767 	pa->pa_lstart = pa->pa_pstart;  in ext4_mb_new_group_pa()
3768 	pa->pa_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_group_pa()
3769 	pa->pa_free = pa->pa_len;  in ext4_mb_new_group_pa()
3770 	atomic_set(&pa->pa_count, 1);  in ext4_mb_new_group_pa()
3771 	spin_lock_init(&pa->pa_lock);  in ext4_mb_new_group_pa()
3772 	INIT_LIST_HEAD(&pa->pa_inode_list);  in ext4_mb_new_group_pa()
3773 	INIT_LIST_HEAD(&pa->pa_group_list);  in ext4_mb_new_group_pa()
3774 	pa->pa_deleted = 0;  in ext4_mb_new_group_pa()
3775 	pa->pa_type = MB_GROUP_PA;  in ext4_mb_new_group_pa()
3778 			pa->pa_pstart, pa->pa_len, pa->pa_lstart);  in ext4_mb_new_group_pa()
3782 	atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);  in ext4_mb_new_group_pa()
3784 	grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_group_pa()
3785 	lg = ac->ac_lg;  in ext4_mb_new_group_pa()
3788 	pa->pa_obj_lock = &lg->lg_prealloc_lock;  in ext4_mb_new_group_pa()
3789 	pa->pa_inode = NULL;  in ext4_mb_new_group_pa()
3791 	ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_group_pa()
3792 	list_add(&pa->pa_group_list, &grp->bb_prealloc_list);  in ext4_mb_new_group_pa()
3793 	ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_group_pa()
3806 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)  in ext4_mb_new_preallocation()
3814  * finds all unused blocks in on-disk bitmap, frees them in
3815  * in-core bitmap and buddy.
3819  * TODO: optimize the case when there are no in-core structures yet
3825 	struct super_block *sb = e4b->bd_sb;  in ext4_mb_release_inode_pa()
3834 	BUG_ON(pa->pa_deleted == 0);  in ext4_mb_release_inode_pa()
3835 	ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);  in ext4_mb_release_inode_pa()
3836 	grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit);  in ext4_mb_release_inode_pa()
3837 	BUG_ON(group != e4b->bd_group && pa->pa_len != 0);  in ext4_mb_release_inode_pa()
3838 	end = bit + pa->pa_len;  in ext4_mb_release_inode_pa()
3841 		bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit);  in ext4_mb_release_inode_pa()
3844 		next = mb_find_next_bit(bitmap_bh->b_data, end, bit);  in ext4_mb_release_inode_pa()
3847 			 (unsigned) next - bit, (unsigned) group);  in ext4_mb_release_inode_pa()
3848 		free += next - bit;  in ext4_mb_release_inode_pa()
3850 		trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit);  in ext4_mb_release_inode_pa()
3853 					       next - bit);  in ext4_mb_release_inode_pa()
3854 		mb_free_blocks(pa->pa_inode, e4b, bit, next - bit);  in ext4_mb_release_inode_pa()
3857 	if (free != pa->pa_free) {  in ext4_mb_release_inode_pa()
3858 		ext4_msg(e4b->bd_sb, KERN_CRIT,  in ext4_mb_release_inode_pa()
3860 			 pa, (unsigned long) pa->pa_lstart,  in ext4_mb_release_inode_pa()
3861 			 (unsigned long) pa->pa_pstart,  in ext4_mb_release_inode_pa()
3862 			 (unsigned long) pa->pa_len);  in ext4_mb_release_inode_pa()
3864 					free, pa->pa_free);  in ext4_mb_release_inode_pa()
3870 	atomic_add(free, &sbi->s_mb_discarded);  in ext4_mb_release_inode_pa()
3879 	struct super_block *sb = e4b->bd_sb;  in ext4_mb_release_group_pa()
3884 	BUG_ON(pa->pa_deleted == 0);  in ext4_mb_release_group_pa()
3885 	ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);  in ext4_mb_release_group_pa()
3886 	BUG_ON(group != e4b->bd_group && pa->pa_len != 0);  in ext4_mb_release_group_pa()
3887 	mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len);  in ext4_mb_release_group_pa()
3888 	atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded);  in ext4_mb_release_group_pa()
3889 	trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len);  in ext4_mb_release_group_pa()
3898  * - when do we discard
3900  * - how many do we discard
3918 	if (list_empty(&grp->bb_prealloc_list))  in ext4_mb_discard_group_preallocations()
3944 				&grp->bb_prealloc_list, pa_group_list) {  in ext4_mb_discard_group_preallocations()
3945 		spin_lock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
3946 		if (atomic_read(&pa->pa_count)) {  in ext4_mb_discard_group_preallocations()
3947 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
3951 		if (pa->pa_deleted) {  in ext4_mb_discard_group_preallocations()
3952 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
3957 		pa->pa_deleted = 1;  in ext4_mb_discard_group_preallocations()
3960 		free += pa->pa_free;  in ext4_mb_discard_group_preallocations()
3962 		spin_unlock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
3964 		list_del(&pa->pa_group_list);  in ext4_mb_discard_group_preallocations()
3965 		list_add(&pa->u.pa_tmp_list, &list);  in ext4_mb_discard_group_preallocations()
3985 		/* remove from object (inode or locality group) */  in ext4_mb_discard_group_preallocations()
3986 		spin_lock(pa->pa_obj_lock);  in ext4_mb_discard_group_preallocations()
3987 		list_del_rcu(&pa->pa_inode_list);  in ext4_mb_discard_group_preallocations()
3988 		spin_unlock(pa->pa_obj_lock);  in ext4_mb_discard_group_preallocations()
3990 		if (pa->pa_type == MB_GROUP_PA)  in ext4_mb_discard_group_preallocations()
3995 		list_del(&pa->u.pa_tmp_list);  in ext4_mb_discard_group_preallocations()
3996 		call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_mb_discard_group_preallocations()
4007  * releases all non-used preallocated blocks for given inode
4018 	struct super_block *sb = inode->i_sb;  in ext4_discard_preallocations()
4026 	if (!S_ISREG(inode->i_mode)) {  in ext4_discard_preallocations()
4027 		/*BUG_ON(!list_empty(&ei->i_prealloc_list));*/  in ext4_discard_preallocations()
4031 	mb_debug(1, "discard preallocation for inode %lu\n", inode->i_ino);  in ext4_discard_preallocations()
4038 	spin_lock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
4039 	while (!list_empty(&ei->i_prealloc_list)) {  in ext4_discard_preallocations()
4040 		pa = list_entry(ei->i_prealloc_list.next,  in ext4_discard_preallocations()
4042 		BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock);  in ext4_discard_preallocations()
4043 		spin_lock(&pa->pa_lock);  in ext4_discard_preallocations()
4044 		if (atomic_read(&pa->pa_count)) {  in ext4_discard_preallocations()
4045 			/* this shouldn't happen often - nobody should  in ext4_discard_preallocations()
4047 			spin_unlock(&pa->pa_lock);  in ext4_discard_preallocations()
4048 			spin_unlock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
4050 				 "uh-oh! used pa while discarding");  in ext4_discard_preallocations()
4056 		if (pa->pa_deleted == 0) {  in ext4_discard_preallocations()
4057 			pa->pa_deleted = 1;  in ext4_discard_preallocations()
4058 			spin_unlock(&pa->pa_lock);  in ext4_discard_preallocations()
4059 			list_del_rcu(&pa->pa_inode_list);  in ext4_discard_preallocations()
4060 			list_add(&pa->u.pa_tmp_list, &list);  in ext4_discard_preallocations()
4065 		spin_unlock(&pa->pa_lock);  in ext4_discard_preallocations()
4066 		spin_unlock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
4071 		 * ->clear_inode() the inode will get freed  in ext4_discard_preallocations()
4074 		 * freed memory, bad-bad-bad */  in ext4_discard_preallocations()
4078 		 * of ->clear_inode(), but not in case of  in ext4_discard_preallocations()
4083 	spin_unlock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
4086 		BUG_ON(pa->pa_type != MB_INODE_PA);  in ext4_discard_preallocations()
4087 		group = ext4_get_group_number(sb, pa->pa_pstart);  in ext4_discard_preallocations()
4107 		list_del(&pa->pa_group_list);  in ext4_discard_preallocations()
4114 		list_del(&pa->u.pa_tmp_list);  in ext4_discard_preallocations()
4115 		call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_discard_preallocations()
4122 	struct super_block *sb = ac->ac_sb;  in ext4_mb_show_ac()
4126 	    (EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED))  in ext4_mb_show_ac()
4129 	ext4_msg(ac->ac_sb, KERN_ERR, "Can't allocate:"  in ext4_mb_show_ac()
4131 	ext4_msg(ac->ac_sb, KERN_ERR, "status %d flags %d",  in ext4_mb_show_ac()
4132 			ac->ac_status, ac->ac_flags);  in ext4_mb_show_ac()
4133 	ext4_msg(ac->ac_sb, KERN_ERR, "orig %lu/%lu/%lu@%lu, "  in ext4_mb_show_ac()
4136 			(unsigned long)ac->ac_o_ex.fe_group,  in ext4_mb_show_ac()
4137 			(unsigned long)ac->ac_o_ex.fe_start,  in ext4_mb_show_ac()
4138 			(unsigned long)ac->ac_o_ex.fe_len,  in ext4_mb_show_ac()
4139 			(unsigned long)ac->ac_o_ex.fe_logical,  in ext4_mb_show_ac()
4140 			(unsigned long)ac->ac_g_ex.fe_group,  in ext4_mb_show_ac()
4141 			(unsigned long)ac->ac_g_ex.fe_start,  in ext4_mb_show_ac()
4142 			(unsigned long)ac->ac_g_ex.fe_len,  in ext4_mb_show_ac()
4143 			(unsigned long)ac->ac_g_ex.fe_logical,  in ext4_mb_show_ac()
4144 			(unsigned long)ac->ac_b_ex.fe_group,  in ext4_mb_show_ac()
4145 			(unsigned long)ac->ac_b_ex.fe_start,  in ext4_mb_show_ac()
4146 			(unsigned long)ac->ac_b_ex.fe_len,  in ext4_mb_show_ac()
4147 			(unsigned long)ac->ac_b_ex.fe_logical,  in ext4_mb_show_ac()
4148 			(int)ac->ac_criteria);  in ext4_mb_show_ac()
4149 	ext4_msg(ac->ac_sb, KERN_ERR, "%d found", ac->ac_found);  in ext4_mb_show_ac()
4150 	ext4_msg(ac->ac_sb, KERN_ERR, "groups: ");  in ext4_mb_show_ac()
4158 		list_for_each(cur, &grp->bb_prealloc_list) {  in ext4_mb_show_ac()
4161 			spin_lock(&pa->pa_lock);  in ext4_mb_show_ac()
4162 			ext4_get_group_no_and_offset(sb, pa->pa_pstart,  in ext4_mb_show_ac()
4164 			spin_unlock(&pa->pa_lock);  in ext4_mb_show_ac()
4166 			       start, pa->pa_len);  in ext4_mb_show_ac()
4170 		if (grp->bb_free == 0)  in ext4_mb_show_ac()
4173 		       i, grp->bb_free, grp->bb_fragments);  in ext4_mb_show_ac()
4185  * We use locality group preallocation for small size file. The size of the
4193 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_group_or_file()
4194 	int bsbits = ac->ac_sb->s_blocksize_bits;  in ext4_mb_group_or_file()
4197 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_group_or_file()
4200 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_group_or_file()
4203 	size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len);  in ext4_mb_group_or_file()
4204 	isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1)  in ext4_mb_group_or_file()
4209 	    (atomic_read(&ac->ac_inode->i_writecount) == 0)) {  in ext4_mb_group_or_file()
4210 		ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC;  in ext4_mb_group_or_file()
4214 	if (sbi->s_mb_group_prealloc <= 0) {  in ext4_mb_group_or_file()
4215 		ac->ac_flags |= EXT4_MB_STREAM_ALLOC;  in ext4_mb_group_or_file()
4221 	if (size > sbi->s_mb_stream_request) {  in ext4_mb_group_or_file()
4222 		ac->ac_flags |= EXT4_MB_STREAM_ALLOC;  in ext4_mb_group_or_file()
4226 	BUG_ON(ac->ac_lg != NULL);  in ext4_mb_group_or_file()
4228 	 * locality group prealloc space are per cpu. The reason for having  in ext4_mb_group_or_file()
4229 	 * per cpu locality group is to reduce the contention between block  in ext4_mb_group_or_file()
4232 	ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups);  in ext4_mb_group_or_file()
4235 	ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC;  in ext4_mb_group_or_file()
4238 	mutex_lock(&ac->ac_lg->lg_mutex);  in ext4_mb_group_or_file()
4245 	struct super_block *sb = ar->inode->i_sb;  in ext4_mb_initialize_context()
4247 	struct ext4_super_block *es = sbi->s_es;  in ext4_mb_initialize_context()
4254 	len = ar->len;  in ext4_mb_initialize_context()
4261 	goal = ar->goal;  in ext4_mb_initialize_context()
4262 	if (goal < le32_to_cpu(es->s_first_data_block) ||  in ext4_mb_initialize_context()
4264 		goal = le32_to_cpu(es->s_first_data_block);  in ext4_mb_initialize_context()
4268 	ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical);  in ext4_mb_initialize_context()
4269 	ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_initialize_context()
4270 	ac->ac_sb = sb;  in ext4_mb_initialize_context()
4271 	ac->ac_inode = ar->inode;  in ext4_mb_initialize_context()
4272 	ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical;  in ext4_mb_initialize_context()
4273 	ac->ac_o_ex.fe_group = group;  in ext4_mb_initialize_context()
4274 	ac->ac_o_ex.fe_start = block;  in ext4_mb_initialize_context()
4275 	ac->ac_o_ex.fe_len = len;  in ext4_mb_initialize_context()
4276 	ac->ac_g_ex = ac->ac_o_ex;  in ext4_mb_initialize_context()
4277 	ac->ac_flags = ar->flags;  in ext4_mb_initialize_context()
4280 	 * locality group. this is a policy, actually */  in ext4_mb_initialize_context()
4285 			(unsigned) ar->len, (unsigned) ar->logical,  in ext4_mb_initialize_context()
4286 			(unsigned) ar->goal, ac->ac_flags, ac->ac_2order,  in ext4_mb_initialize_context()
4287 			(unsigned) ar->lleft, (unsigned) ar->pleft,  in ext4_mb_initialize_context()
4288 			(unsigned) ar->lright, (unsigned) ar->pright,  in ext4_mb_initialize_context()
4289 			atomic_read(&ar->inode->i_writecount) ? "" : "non-");  in ext4_mb_initialize_context()
4304 	mb_debug(1, "discard locality group preallocation\n");  in ext4_mb_discard_lg_preallocations()
4308 	spin_lock(&lg->lg_prealloc_lock);  in ext4_mb_discard_lg_preallocations()
4309 	list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order],  in ext4_mb_discard_lg_preallocations()
4311 		spin_lock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
4312 		if (atomic_read(&pa->pa_count)) {  in ext4_mb_discard_lg_preallocations()
4318 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
4321 		if (pa->pa_deleted) {  in ext4_mb_discard_lg_preallocations()
4322 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
4326 		BUG_ON(pa->pa_type != MB_GROUP_PA);  in ext4_mb_discard_lg_preallocations()
4329 		pa->pa_deleted = 1;  in ext4_mb_discard_lg_preallocations()
4330 		spin_unlock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
4332 		list_del_rcu(&pa->pa_inode_list);  in ext4_mb_discard_lg_preallocations()
4333 		list_add(&pa->u.pa_tmp_list, &discard_list);  in ext4_mb_discard_lg_preallocations()
4335 		total_entries--;  in ext4_mb_discard_lg_preallocations()
4346 	spin_unlock(&lg->lg_prealloc_lock);  in ext4_mb_discard_lg_preallocations()
4351 		group = ext4_get_group_number(sb, pa->pa_pstart);  in ext4_mb_discard_lg_preallocations()
4360 		list_del(&pa->pa_group_list);  in ext4_mb_discard_lg_preallocations()
4365 		list_del(&pa->u.pa_tmp_list);  in ext4_mb_discard_lg_preallocations()
4366 		call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_mb_discard_lg_preallocations()
4382 	struct super_block *sb = ac->ac_sb;  in ext4_mb_add_n_trim()
4383 	struct ext4_locality_group *lg = ac->ac_lg;  in ext4_mb_add_n_trim()
4384 	struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa;  in ext4_mb_add_n_trim()
4386 	order = fls(pa->pa_free) - 1;  in ext4_mb_add_n_trim()
4387 	if (order > PREALLOC_TB_SIZE - 1)  in ext4_mb_add_n_trim()
4389 		order = PREALLOC_TB_SIZE - 1;  in ext4_mb_add_n_trim()
4391 	spin_lock(&lg->lg_prealloc_lock);  in ext4_mb_add_n_trim()
4392 	list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order],  in ext4_mb_add_n_trim()
4394 		spin_lock(&tmp_pa->pa_lock);  in ext4_mb_add_n_trim()
4395 		if (tmp_pa->pa_deleted) {  in ext4_mb_add_n_trim()
4396 			spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_add_n_trim()
4399 		if (!added && pa->pa_free < tmp_pa->pa_free) {  in ext4_mb_add_n_trim()
4401 			list_add_tail_rcu(&pa->pa_inode_list,  in ext4_mb_add_n_trim()
4402 						&tmp_pa->pa_inode_list);  in ext4_mb_add_n_trim()
4409 		spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_add_n_trim()
4413 		list_add_tail_rcu(&pa->pa_inode_list,  in ext4_mb_add_n_trim()
4414 					&lg->lg_prealloc_list[order]);  in ext4_mb_add_n_trim()
4415 	spin_unlock(&lg->lg_prealloc_lock);  in ext4_mb_add_n_trim()
4431 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_release_context()
4432 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_mb_release_context()
4434 		if (pa->pa_type == MB_GROUP_PA) {  in ext4_mb_release_context()
4436 			spin_lock(&pa->pa_lock);  in ext4_mb_release_context()
4437 			pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
4438 			pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
4439 			pa->pa_free -= ac->ac_b_ex.fe_len;  in ext4_mb_release_context()
4440 			pa->pa_len -= ac->ac_b_ex.fe_len;  in ext4_mb_release_context()
4441 			spin_unlock(&pa->pa_lock);  in ext4_mb_release_context()
4451 		if ((pa->pa_type == MB_GROUP_PA) && likely(pa->pa_free)) {  in ext4_mb_release_context()
4452 			spin_lock(pa->pa_obj_lock);  in ext4_mb_release_context()
4453 			list_del_rcu(&pa->pa_inode_list);  in ext4_mb_release_context()
4454 			spin_unlock(pa->pa_obj_lock);  in ext4_mb_release_context()
4457 		ext4_mb_put_pa(ac, ac->ac_sb, pa);  in ext4_mb_release_context()
4459 	if (ac->ac_bitmap_page)  in ext4_mb_release_context()
4460 		put_page(ac->ac_bitmap_page);  in ext4_mb_release_context()
4461 	if (ac->ac_buddy_page)  in ext4_mb_release_context()
4462 		put_page(ac->ac_buddy_page);  in ext4_mb_release_context()
4463 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)  in ext4_mb_release_context()
4464 		mutex_unlock(&ac->ac_lg->lg_mutex);  in ext4_mb_release_context()
4479 		needed -= ret;  in ext4_mb_discard_preallocations()
4502 	sb = ar->inode->i_sb;  in ext4_mb_new_blocks()
4508 	if (ext4_is_quota_file(ar->inode))  in ext4_mb_new_blocks()
4509 		ar->flags |= EXT4_MB_USE_ROOT_BLOCKS;  in ext4_mb_new_blocks()
4511 	if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) {  in ext4_mb_new_blocks()
4516 		while (ar->len &&  in ext4_mb_new_blocks()
4517 			ext4_claim_free_clusters(sbi, ar->len, ar->flags)) {  in ext4_mb_new_blocks()
4521 			ar->len = ar->len >> 1;  in ext4_mb_new_blocks()
4523 		if (!ar->len) {  in ext4_mb_new_blocks()
4524 			*errp = -ENOSPC;  in ext4_mb_new_blocks()
4527 		reserv_clstrs = ar->len;  in ext4_mb_new_blocks()
4528 		if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) {  in ext4_mb_new_blocks()
4529 			dquot_alloc_block_nofail(ar->inode,  in ext4_mb_new_blocks()
4530 						 EXT4_C2B(sbi, ar->len));  in ext4_mb_new_blocks()
4532 			while (ar->len &&  in ext4_mb_new_blocks()
4533 				dquot_alloc_block(ar->inode,  in ext4_mb_new_blocks()
4534 						  EXT4_C2B(sbi, ar->len))) {  in ext4_mb_new_blocks()
4536 				ar->flags |= EXT4_MB_HINT_NOPREALLOC;  in ext4_mb_new_blocks()
4537 				ar->len--;  in ext4_mb_new_blocks()
4540 		inquota = ar->len;  in ext4_mb_new_blocks()
4541 		if (ar->len == 0) {  in ext4_mb_new_blocks()
4542 			*errp = -EDQUOT;  in ext4_mb_new_blocks()
4549 		ar->len = 0;  in ext4_mb_new_blocks()
4550 		*errp = -ENOMEM;  in ext4_mb_new_blocks()
4556 		ar->len = 0;  in ext4_mb_new_blocks()
4560 	ac->ac_op = EXT4_MB_HISTORY_PREALLOC;  in ext4_mb_new_blocks()
4562 		ac->ac_op = EXT4_MB_HISTORY_ALLOC;  in ext4_mb_new_blocks()
4573 		if (ac->ac_status == AC_STATUS_FOUND &&  in ext4_mb_new_blocks()
4574 		    ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len)  in ext4_mb_new_blocks()
4582 	if (likely(ac->ac_status == AC_STATUS_FOUND)) {  in ext4_mb_new_blocks()
4588 			block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_blocks()
4589 			ar->len = ac->ac_b_ex.fe_len;  in ext4_mb_new_blocks()
4592 		freed  = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len);  in ext4_mb_new_blocks()
4595 		*errp = -ENOSPC;  in ext4_mb_new_blocks()
4600 		ac->ac_b_ex.fe_len = 0;  in ext4_mb_new_blocks()
4601 		ar->len = 0;  in ext4_mb_new_blocks()
4608 	if (inquota && ar->len < inquota)  in ext4_mb_new_blocks()
4609 		dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len));  in ext4_mb_new_blocks()
4610 	if (!ar->len) {  in ext4_mb_new_blocks()
4611 		if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0)  in ext4_mb_new_blocks()
4613 			percpu_counter_sub(&sbi->s_dirtyclusters_counter,  in ext4_mb_new_blocks()
4632 	if ((entry->efd_tid != new_entry->efd_tid) ||  in ext4_try_merge_freed_extent()
4633 	    (entry->efd_group != new_entry->efd_group))  in ext4_try_merge_freed_extent()
4635 	if (entry->efd_start_cluster + entry->efd_count ==  in ext4_try_merge_freed_extent()
4636 	    new_entry->efd_start_cluster) {  in ext4_try_merge_freed_extent()
4637 		new_entry->efd_start_cluster = entry->efd_start_cluster;  in ext4_try_merge_freed_extent()
4638 		new_entry->efd_count += entry->efd_count;  in ext4_try_merge_freed_extent()
4639 	} else if (new_entry->efd_start_cluster + new_entry->efd_count ==  in ext4_try_merge_freed_extent()
4640 		   entry->efd_start_cluster) {  in ext4_try_merge_freed_extent()
4641 		new_entry->efd_count += entry->efd_count;  in ext4_try_merge_freed_extent()
4644 	spin_lock(&sbi->s_md_lock);  in ext4_try_merge_freed_extent()
4645 	list_del(&entry->efd_list);  in ext4_try_merge_freed_extent()
4646 	spin_unlock(&sbi->s_md_lock);  in ext4_try_merge_freed_extent()
4647 	rb_erase(&entry->efd_node, entry_rb_root);  in ext4_try_merge_freed_extent()
4655 	ext4_group_t group = e4b->bd_group;  in ext4_mb_free_metadata()
4657 	ext4_grpblk_t clusters = new_entry->efd_count;  in ext4_mb_free_metadata()
4659 	struct ext4_group_info *db = e4b->bd_info;  in ext4_mb_free_metadata()
4660 	struct super_block *sb = e4b->bd_sb;  in ext4_mb_free_metadata()
4662 	struct rb_node **n = &db->bb_free_root.rb_node, *node;  in ext4_mb_free_metadata()
4666 	BUG_ON(e4b->bd_bitmap_page == NULL);  in ext4_mb_free_metadata()
4667 	BUG_ON(e4b->bd_buddy_page == NULL);  in ext4_mb_free_metadata()
4669 	new_node = &new_entry->efd_node;  in ext4_mb_free_metadata()
4670 	cluster = new_entry->efd_start_cluster;  in ext4_mb_free_metadata()
4676 		 * on-disk bitmap and lose not-yet-available  in ext4_mb_free_metadata()
4678 		get_page(e4b->bd_buddy_page);  in ext4_mb_free_metadata()
4679 		get_page(e4b->bd_bitmap_page);  in ext4_mb_free_metadata()
4684 		if (cluster < entry->efd_start_cluster)  in ext4_mb_free_metadata()
4685 			n = &(*n)->rb_left;  in ext4_mb_free_metadata()
4686 		else if (cluster >= (entry->efd_start_cluster + entry->efd_count))  in ext4_mb_free_metadata()
4687 			n = &(*n)->rb_right;  in ext4_mb_free_metadata()
4692 				"Block already on to-be-freed list");  in ext4_mb_free_metadata()
4698 	rb_insert_color(new_node, &db->bb_free_root);  in ext4_mb_free_metadata()
4705 					    &(db->bb_free_root));  in ext4_mb_free_metadata()
4712 					    &(db->bb_free_root));  in ext4_mb_free_metadata()
4715 	spin_lock(&sbi->s_md_lock);  in ext4_mb_free_metadata()
4716 	list_add_tail(&new_entry->efd_list, &sbi->s_freed_data_list);  in ext4_mb_free_metadata()
4717 	sbi->s_mb_free_pending += clusters;  in ext4_mb_free_metadata()
4718 	spin_unlock(&sbi->s_md_lock);  in ext4_mb_free_metadata()
4723  * ext4_free_blocks() -- Free given blocks and update quota
4735 	struct super_block *sb = inode->i_sb;  in ext4_free_blocks()
4750 			BUG_ON(block != bh->b_blocknr);  in ext4_free_blocks()
4752 			block = bh->b_blocknr;  in ext4_free_blocks()
4758 		ext4_error(sb, "Freeing blocks not in datazone - "  in ext4_free_blocks()
4783 			overflow = sbi->s_cluster_ratio - overflow;  in ext4_free_blocks()
4786 				count -= overflow;  in ext4_free_blocks()
4790 			block -= overflow;  in ext4_free_blocks()
4798 				count -= overflow;  in ext4_free_blocks()
4802 			count += sbi->s_cluster_ratio - overflow;  in ext4_free_blocks()
4812 				bh = sb_find_get_block(inode->i_sb, block + i);  in ext4_free_blocks()
4830 		overflow = EXT4_C2B(sbi, bit) + count -  in ext4_free_blocks()
4832 		count -= overflow;  in ext4_free_blocks()
4843 		err = -EIO;  in ext4_free_blocks()
4850 		     sbi->s_itb_per_group) ||  in ext4_free_blocks()
4851 	    in_range(block + count - 1, ext4_inode_table(sb, gdp),  in ext4_free_blocks()
4852 		     sbi->s_itb_per_group)) {  in ext4_free_blocks()
4854 		ext4_error(sb, "Freeing blocks in system zone - "  in ext4_free_blocks()
4867 	 * to unshare ->b_data if a currently-committing transaction is  in ext4_free_blocks()
4878 			BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data));  in ext4_free_blocks()
4905 		new_entry->efd_start_cluster = bit;  in ext4_free_blocks()
4906 		new_entry->efd_group = block_group;  in ext4_free_blocks()
4907 		new_entry->efd_count = count_clusters;  in ext4_free_blocks()
4908 		new_entry->efd_tid = handle->h_transaction->t_tid;  in ext4_free_blocks()
4911 		mb_clear_bits(bitmap_bh->b_data, bit, count_clusters);  in ext4_free_blocks()
4914 		/* need to update group_info->bb_free and bitmap  in ext4_free_blocks()
4921 			if (err && err != -EOPNOTSUPP)  in ext4_free_blocks()
4930 		mb_clear_bits(bitmap_bh->b_data, bit, count_clusters);  in ext4_free_blocks()
4940 	if (sbi->s_log_groups_per_flex) {  in ext4_free_blocks()
4944 						  flex_group)->free_clusters);  in ext4_free_blocks()
4949 	percpu_counter_add(&sbi->s_freeclusters_counter, count_clusters);  in ext4_free_blocks()
4976  * ext4_group_add_blocks() -- Add given blocks to an existing group
4998 	ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1);  in ext4_group_add_blocks()
4999 	unsigned long cluster_count = last_cluster - first_cluster + 1;  in ext4_group_add_blocks()
5001 	ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1);  in ext4_group_add_blocks()
5014 		err = -EINVAL;  in ext4_group_add_blocks()
5027 		err = -EIO;  in ext4_group_add_blocks()
5033 	    in_range(block, ext4_inode_table(sb, desc), sbi->s_itb_per_group) ||  in ext4_group_add_blocks()
5034 	    in_range(block + count - 1, ext4_inode_table(sb, desc),  in ext4_group_add_blocks()
5035 		     sbi->s_itb_per_group)) {  in ext4_group_add_blocks()
5036 		ext4_error(sb, "Adding blocks in system zones - "  in ext4_group_add_blocks()
5039 		err = -EINVAL;  in ext4_group_add_blocks()
5050 	 * to unshare ->b_data if a currently-committing transaction is  in ext4_group_add_blocks()
5060 		if (!mb_test_bit(bit + i, bitmap_bh->b_data)) {  in ext4_group_add_blocks()
5074 	 * need to update group_info->bb_free and bitmap  in ext4_group_add_blocks()
5079 	mb_clear_bits(bitmap_bh->b_data, bit, cluster_count);  in ext4_group_add_blocks()
5087 	percpu_counter_add(&sbi->s_freeclusters_counter,  in ext4_group_add_blocks()
5090 	if (sbi->s_log_groups_per_flex) {  in ext4_group_add_blocks()
5094 						  flex_group)->free_clusters);  in ext4_group_add_blocks()
5116  * ext4_trim_extent -- function to TRIM one single free extent in the group
5156  * ext4_trim_all_free -- function to trim all free space in alloc. group
5195 	    minblocks >= atomic_read(&EXT4_SB(sb)->s_last_trim_minblks))  in ext4_trim_all_free()
5198 	start = (e4b.bd_info->bb_first_free > start) ?  in ext4_trim_all_free()
5199 		e4b.bd_info->bb_first_free : start;  in ext4_trim_all_free()
5207 		if ((next - start) >= minblocks) {  in ext4_trim_all_free()
5209 					       next - start, group, &e4b);  in ext4_trim_all_free()
5210 			if (ret && ret != -EOPNOTSUPP)  in ext4_trim_all_free()
5213 			count += next - start;  in ext4_trim_all_free()
5215 		free_count += next - start;  in ext4_trim_all_free()
5219 			count = -ERESTARTSYS;  in ext4_trim_all_free()
5229 		if ((e4b.bd_info->bb_free - free_count) < minblocks)  in ext4_trim_all_free()
5248  * ext4_trim_fs() -- trim ioctl handle function
5266 			le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);  in ext4_trim_fs()
5267 	ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es);  in ext4_trim_fs()
5270 	start = range->start >> sb->s_blocksize_bits;  in ext4_trim_fs()
5271 	end = start + (range->len >> sb->s_blocksize_bits) - 1;  in ext4_trim_fs()
5273 			      range->minlen >> sb->s_blocksize_bits);  in ext4_trim_fs()
5277 	    range->len < sb->s_blocksize)  in ext4_trim_fs()
5278 		return -EINVAL;  in ext4_trim_fs()
5280 		end = max_blks - 1;  in ext4_trim_fs()
5293 	end = EXT4_CLUSTERS_PER_GROUP(sb) - 1;  in ext4_trim_fs()
5306 		 * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to  in ext4_trim_fs()
5313 		if (grp->bb_free >= minlen) {  in ext4_trim_fs()
5331 		atomic_set(&EXT4_SB(sb)->s_last_trim_minblks, minlen);  in ext4_trim_fs()
5334 	range->len = EXT4_C2B(EXT4_SB(sb), trimmed) << sb->s_blocksize_bits;  in ext4_trim_fs()
5360 	start = (e4b.bd_info->bb_first_free > start) ?  in ext4_mballoc_query_range()
5361 		e4b.bd_info->bb_first_free : start;  in ext4_mballoc_query_range()
5363 		end = EXT4_CLUSTERS_PER_GROUP(sb) - 1;  in ext4_mballoc_query_range()
5372 		error = formatter(sb, group, start, next - start, priv);  in ext4_mballoc_query_range()