1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
18 #include <linux/backing-dev.h>
24  *   - test ext4_ext_search_left() and ext4_ext_search_right()
25  *   - search for metadata in few groups
28  *   - normalization should take into account whether file is still open
29  *   - discard preallocations if no free space left (policy?)
30  *   - don't normalize tails
31  *   - quota
32  *   - reservation for superuser
35  *   - bitmap read-ahead (proposed by Oleg Drokin aka green)
36  *   - track min/max extents in each group for better group selection
37  *   - mb_mark_used() may allocate chunk right after splitting buddy
38  *   - tree of groups sorted by number of free blocks
39  *   - error handling
50  * is larger. If the size is less than sbi->s_mb_stream_request we
60  * ext4_inode_info->i_prealloc_list, which contains list of prealloc
64  * pa_lstart -> the logical start block for this prealloc space
65  * pa_pstart -> the physical start block for this prealloc space
66  * pa_len    -> length for this prealloc space (in clusters)
67  * pa_free   ->  free space available in this prealloc space (in clusters)
79  * have the group allocation flag set then we look at the locality group
84  * The reason for having a per cpu locality group is to reduce the contention
87  * The locality group prealloc space is used looking at whether we have
90  * If we can't allocate blocks via inode prealloc or/and locality group
122  * sbi->s_mb_group_prealloc.  The default value of s_mb_group_prealloc is
123  * dependent on the cluster size; for non-bigalloc file systems, it is
126  * terms of number of blocks. If we have mounted the file system with -O
128  * smallest multiple of the stripe value (sbi->s_stripe) which is
134  * 1) Array of largest free order lists (sbi->s_mb_largest_free_orders)
136  *    Locking: sbi->s_mb_largest_free_orders_locks(array of rw locks)
141  *    number of buddy bitmap orders possible) number of lists. Group-infos are
144  * 2) Average fragment size lists (sbi->s_mb_avg_fragment_size)
146  *    Locking: sbi->s_mb_avg_fragment_size_locks(array of rw locks)
148  *    This is an array of lists where in the i-th list there are groups with
150  *    is computed as ext4_group_info->bb_free / ext4_group_info->bb_fragments.
192  * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The
195  * stripe size (sbi->s_stripe), we try to search for contiguous block in
197  * not, we search in the specific group using bitmap for best extents. The
210  * non-linear fashion. While that may not matter on non-rotational devices, for
225  *  - on-disk bitmap
226  *  - in-core buddy (actually includes buddy and bitmap)
227  *  - preallocation descriptors (PAs)
230  *  - inode
231  *    assiged to specific inode and can be used for this inode only.
232  *    it describes part of inode's space preallocated to specific
239  *  - locality group
240  *    assigned to specific locality group which does not translate to
246  *    in-core buddy = on-disk bitmap + preallocation descriptors
249  *  - allocated blocks (persistent)
250  *  - preallocated blocks (non-persistent)
254  * literally -- time is discrete and delimited by locks.
257  *  blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA.
260  *  - init buddy:			buddy = on-disk + PAs
261  *  - new PA:				buddy += N; PA = N
262  *  - use inode PA:			on-disk += N; PA -= N
263  *  - discard inode PA			buddy -= on-disk - PA; PA = 0
264  *  - use locality group PA		on-disk += N; PA -= N
265  *  - discard locality group PA		buddy -= PA; PA = 0
266  *  note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap
268  *        bits from PA, only from on-disk bitmap
272  * killing performance on high-end SMP hardware. let's try to relax it using
276  *     nobody can re-allocate that block
277  *  3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has
279  *     on-disk bitmap if buddy has same bit set or/and PA covers corresponded
283  *  - init buddy vs.
284  *    - new PA
287  *    - use inode PA
288  *      we need to make sure that either on-disk bitmap or PA has uptodate data
289  *      given (3) we care that PA-=N operation doesn't interfere with init
290  *    - discard inode PA
292  *    - use locality group PA
293  *      again PA-=N must be serialized with init
294  *    - discard locality group PA
296  *  - new PA vs.
297  *    - use inode PA
299  *    - discard inode PA
301  *    - use locality group PA
303  *    - discard locality group PA
305  *  - use inode PA
306  *    - use inode PA
308  *    - discard inode PA
310  *    - use locality group PA
311  *      nothing wrong here -- they're different PAs covering different blocks
312  *    - discard locality group PA
316  *  - PA is referenced and while it is no discard is possible
317  *  - PA is referenced until block isn't marked in on-disk bitmap
318  *  - PA changes only after on-disk bitmap
319  *  - discard must not compete with init. either init is done before
321  *  - buddy init as sum of on-disk bitmap and PAs is done atomically
331  *  - allocation:
334  *    mark bits in on-disk bitmap
337  *  - use preallocation:
338  *    find proper PA (per-inode or group)
340  *    mark bits in on-disk bitmap
344  *  - free:
346  *    mark bits in on-disk bitmap
349  *  - discard preallocations in group:
352  *    load on-disk bitmap
354  *    remove PA from object (inode or locality group)
355  *    mark free blocks in-core
357  *  - discard inode's preallocations:
364  *  - bitlock on a group	(group)
365  *  - object (inode/locality)	(object)
366  *  - per-pa lock		(pa)
367  *  - cr_power2_aligned lists lock	(cr_power2_aligned)
368  *  - cr_goal_len_fast lists lock	(cr_goal_len_fast)
371  *  - new pa
375  *  - find and use pa:
378  *  - release consumed pa:
383  *  - generate in-core bitmap:
387  *  - discard all for given object (inode, locality group):
392  *  - discard all for given group:
398  *  - allocation path (ext4_mb_regular_allocator)
445  * sure that all the PAs on grp->bb_prealloc_list got freed or if it's empty.
507 	ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix;  in mb_find_next_zero_bit()
520 	ret = ext4_find_next_bit(addr, tmpmax, start) - fix;  in mb_find_next_bit()
530 	BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);  in mb_find_buddy()
533 	if (order > e4b->bd_blkbits + 1) {  in mb_find_buddy()
540 		*max = 1 << (e4b->bd_blkbits + 3);  in mb_find_buddy()
541 		return e4b->bd_bitmap;  in mb_find_buddy()
544 	bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order];  in mb_find_buddy()
545 	*max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order];  in mb_find_buddy()
555 	struct super_block *sb = e4b->bd_sb;  in mb_free_blocks_double()
557 	if (unlikely(e4b->bd_info->bb_bitmap == NULL))  in mb_free_blocks_double()
559 	assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));  in mb_free_blocks_double()
561 		if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) {  in mb_free_blocks_double()
564 			blocknr = ext4_group_first_block_no(sb, e4b->bd_group);  in mb_free_blocks_double()
566 			ext4_grp_locked_error(sb, e4b->bd_group,  in mb_free_blocks_double()
567 					      inode ? inode->i_ino : 0,  in mb_free_blocks_double()
572 			ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in mb_free_blocks_double()
575 		mb_clear_bit(first + i, e4b->bd_info->bb_bitmap);  in mb_free_blocks_double()
583 	if (unlikely(e4b->bd_info->bb_bitmap == NULL))  in mb_mark_used_double()
585 	assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));  in mb_mark_used_double()
587 		BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap));  in mb_mark_used_double()
588 		mb_set_bit(first + i, e4b->bd_info->bb_bitmap);  in mb_mark_used_double()
594 	if (unlikely(e4b->bd_info->bb_bitmap == NULL))  in mb_cmp_bitmaps()
596 	if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) {  in mb_cmp_bitmaps()
599 		b1 = (unsigned char *) e4b->bd_info->bb_bitmap;  in mb_cmp_bitmaps()
601 		for (i = 0; i < e4b->bd_sb->s_blocksize; i++) {  in mb_cmp_bitmaps()
603 				ext4_msg(e4b->bd_sb, KERN_ERR,  in mb_cmp_bitmaps()
607 					 e4b->bd_group, i, i * 8, b1[i], b2[i]);  in mb_cmp_bitmaps()
619 	grp->bb_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS);  in mb_group_bb_bitmap_alloc()
620 	if (!grp->bb_bitmap)  in mb_group_bb_bitmap_alloc()
625 		kfree(grp->bb_bitmap);  in mb_group_bb_bitmap_alloc()
626 		grp->bb_bitmap = NULL;  in mb_group_bb_bitmap_alloc()
630 	memcpy(grp->bb_bitmap, bh->b_data, sb->s_blocksize);  in mb_group_bb_bitmap_alloc()
636 	kfree(grp->bb_bitmap);  in mb_group_bb_bitmap_free()
682 	struct super_block *sb = e4b->bd_sb;  in __mb_check_buddy()
683 	int order = e4b->bd_blkbits + 1;  in __mb_check_buddy()
697 	if (e4b->bd_info->bb_check_counter++ % 10)  in __mb_check_buddy()
703 		buddy2 = mb_find_buddy(e4b, order - 1, &max2);  in __mb_check_buddy()
727 					!mb_test_bit(k, e4b->bd_bitmap));  in __mb_check_buddy()
731 		MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count);  in __mb_check_buddy()
732 		order--;  in __mb_check_buddy()
735 	fstart = -1;  in __mb_check_buddy()
739 			MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free);  in __mb_check_buddy()
740 			if (fstart == -1) {  in __mb_check_buddy()
746 		fstart = -1;  in __mb_check_buddy()
748 		for (j = 0; j < e4b->bd_blkbits + 1; j++) {  in __mb_check_buddy()
755 	MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info));  in __mb_check_buddy()
756 	MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments);  in __mb_check_buddy()
758 	grp = ext4_get_group_info(sb, e4b->bd_group);  in __mb_check_buddy()
761 	list_for_each(cur, &grp->bb_prealloc_list) {  in __mb_check_buddy()
765 		ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k);  in __mb_check_buddy()
766 		MB_CHECK_ASSERT(groupnr == e4b->bd_group);  in __mb_check_buddy()
767 		for (i = 0; i < pa->pa_len; i++)  in __mb_check_buddy()
797 	border = 2 << sb->s_blocksize_bits;  in ext4_mb_mark_free_simple()
801 		max = ffs(first | border) - 1;  in ext4_mb_mark_free_simple()
804 		min = fls(len) - 1;  in ext4_mb_mark_free_simple()
811 		grp->bb_counters[min]++;  in ext4_mb_mark_free_simple()
814 				     buddy + sbi->s_mb_offsets[min]);  in ext4_mb_mark_free_simple()
816 		len -= chunk;  in ext4_mb_mark_free_simple()
829 	order = fls(len) - 2;  in mb_avg_fragment_size_order()
833 		order--;  in mb_avg_fragment_size_order()
835 		order = MB_NUM_ORDERS(sb) - 1;  in mb_avg_fragment_size_order()
846 	if (!test_opt2(sb, MB_OPTIMIZE_SCAN) || grp->bb_fragments == 0)  in mb_update_avg_fragment_size()
850 					grp->bb_free / grp->bb_fragments);  in mb_update_avg_fragment_size()
851 	if (new_order == grp->bb_avg_fragment_size_order)  in mb_update_avg_fragment_size()
854 	if (grp->bb_avg_fragment_size_order != -1) {  in mb_update_avg_fragment_size()
855 		write_lock(&sbi->s_mb_avg_fragment_size_locks[  in mb_update_avg_fragment_size()
856 					grp->bb_avg_fragment_size_order]);  in mb_update_avg_fragment_size()
857 		list_del(&grp->bb_avg_fragment_size_node);  in mb_update_avg_fragment_size()
858 		write_unlock(&sbi->s_mb_avg_fragment_size_locks[  in mb_update_avg_fragment_size()
859 					grp->bb_avg_fragment_size_order]);  in mb_update_avg_fragment_size()
861 	grp->bb_avg_fragment_size_order = new_order;  in mb_update_avg_fragment_size()
862 	write_lock(&sbi->s_mb_avg_fragment_size_locks[  in mb_update_avg_fragment_size()
863 					grp->bb_avg_fragment_size_order]);  in mb_update_avg_fragment_size()
864 	list_add_tail(&grp->bb_avg_fragment_size_node,  in mb_update_avg_fragment_size()
865 		&sbi->s_mb_avg_fragment_size[grp->bb_avg_fragment_size_order]);  in mb_update_avg_fragment_size()
866 	write_unlock(&sbi->s_mb_avg_fragment_size_locks[  in mb_update_avg_fragment_size()
867 					grp->bb_avg_fragment_size_order]);  in mb_update_avg_fragment_size()
877 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_choose_next_group_p2_aligned()
881 	if (ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_choose_next_group_p2_aligned()
884 	if (unlikely(sbi->s_mb_stats && ac->ac_flags & EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED))  in ext4_mb_choose_next_group_p2_aligned()
885 		atomic_inc(&sbi->s_bal_p2_aligned_bad_suggestions);  in ext4_mb_choose_next_group_p2_aligned()
887 	for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) {  in ext4_mb_choose_next_group_p2_aligned()
888 		if (list_empty(&sbi->s_mb_largest_free_orders[i]))  in ext4_mb_choose_next_group_p2_aligned()
890 		read_lock(&sbi->s_mb_largest_free_orders_locks[i]);  in ext4_mb_choose_next_group_p2_aligned()
891 		if (list_empty(&sbi->s_mb_largest_free_orders[i])) {  in ext4_mb_choose_next_group_p2_aligned()
892 			read_unlock(&sbi->s_mb_largest_free_orders_locks[i]);  in ext4_mb_choose_next_group_p2_aligned()
895 		list_for_each_entry(iter, &sbi->s_mb_largest_free_orders[i],  in ext4_mb_choose_next_group_p2_aligned()
897 			if (sbi->s_mb_stats)  in ext4_mb_choose_next_group_p2_aligned()
898 				atomic64_inc(&sbi->s_bal_cX_groups_considered[CR_POWER2_ALIGNED]);  in ext4_mb_choose_next_group_p2_aligned()
899 			if (likely(ext4_mb_good_group(ac, iter->bb_group, CR_POWER2_ALIGNED))) {  in ext4_mb_choose_next_group_p2_aligned()
900 				*group = iter->bb_group;  in ext4_mb_choose_next_group_p2_aligned()
901 				ac->ac_flags |= EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED;  in ext4_mb_choose_next_group_p2_aligned()
902 				read_unlock(&sbi->s_mb_largest_free_orders_locks[i]);  in ext4_mb_choose_next_group_p2_aligned()
906 		read_unlock(&sbi->s_mb_largest_free_orders_locks[i]);  in ext4_mb_choose_next_group_p2_aligned()
919 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_find_good_group_avg_frag_lists()
920 	struct list_head *frag_list = &sbi->s_mb_avg_fragment_size[order];  in ext4_mb_find_good_group_avg_frag_lists()
921 	rwlock_t *frag_list_lock = &sbi->s_mb_avg_fragment_size_locks[order];  in ext4_mb_find_good_group_avg_frag_lists()
923 	enum criteria cr = ac->ac_criteria;  in ext4_mb_find_good_group_avg_frag_lists()
933 		if (sbi->s_mb_stats)  in ext4_mb_find_good_group_avg_frag_lists()
934 			atomic64_inc(&sbi->s_bal_cX_groups_considered[cr]);  in ext4_mb_find_good_group_avg_frag_lists()
935 		if (likely(ext4_mb_good_group(ac, iter->bb_group, cr))) {  in ext4_mb_find_good_group_avg_frag_lists()
951 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_choose_next_group_goal_fast()
955 	if (unlikely(ac->ac_flags & EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED)) {  in ext4_mb_choose_next_group_goal_fast()
956 		if (sbi->s_mb_stats)  in ext4_mb_choose_next_group_goal_fast()
957 			atomic_inc(&sbi->s_bal_goal_fast_bad_suggestions);  in ext4_mb_choose_next_group_goal_fast()
960 	for (i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len);  in ext4_mb_choose_next_group_goal_fast()
961 	     i < MB_NUM_ORDERS(ac->ac_sb); i++) {  in ext4_mb_choose_next_group_goal_fast()
964 			*group = grp->bb_group;  in ext4_mb_choose_next_group_goal_fast()
965 			ac->ac_flags |= EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED;  in ext4_mb_choose_next_group_goal_fast()
974 	 * request len. However, allocation request for non-regular  in ext4_mb_choose_next_group_goal_fast()
978 	if (ac->ac_flags & EXT4_MB_HINT_DATA)  in ext4_mb_choose_next_group_goal_fast()
996 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_choose_next_group_best_avail()
1001 	if (unlikely(ac->ac_flags & EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED)) {  in ext4_mb_choose_next_group_best_avail()
1002 		if (sbi->s_mb_stats)  in ext4_mb_choose_next_group_best_avail()
1003 			atomic_inc(&sbi->s_bal_best_avail_bad_suggestions);  in ext4_mb_choose_next_group_best_avail()
1012 	order = fls(ac->ac_g_ex.fe_len) - 1;  in ext4_mb_choose_next_group_best_avail()
1013 	if (WARN_ON_ONCE(order - 1 > MB_NUM_ORDERS(ac->ac_sb)))  in ext4_mb_choose_next_group_best_avail()
1014 		order = MB_NUM_ORDERS(ac->ac_sb);  in ext4_mb_choose_next_group_best_avail()
1015 	min_order = order - sbi->s_mb_best_avail_max_trim_order;  in ext4_mb_choose_next_group_best_avail()
1019 	if (sbi->s_stripe > 0) {  in ext4_mb_choose_next_group_best_avail()
1024 		num_stripe_clusters = EXT4_NUM_B2C(sbi, sbi->s_stripe);  in ext4_mb_choose_next_group_best_avail()
1030 			min_order = fls(num_stripe_clusters) - 1;  in ext4_mb_choose_next_group_best_avail()
1033 	if (1 << min_order < ac->ac_o_ex.fe_len)  in ext4_mb_choose_next_group_best_avail()
1034 		min_order = fls(ac->ac_o_ex.fe_len);  in ext4_mb_choose_next_group_best_avail()
1036 	for (i = order; i >= min_order; i--) {  in ext4_mb_choose_next_group_best_avail()
1043 		ac->ac_g_ex.fe_len = 1 << i;  in ext4_mb_choose_next_group_best_avail()
1051 			ac->ac_g_ex.fe_len = roundup(ac->ac_g_ex.fe_len,  in ext4_mb_choose_next_group_best_avail()
1055 		frag_order = mb_avg_fragment_size_order(ac->ac_sb,  in ext4_mb_choose_next_group_best_avail()
1056 							ac->ac_g_ex.fe_len);  in ext4_mb_choose_next_group_best_avail()
1060 			*group = grp->bb_group;  in ext4_mb_choose_next_group_best_avail()
1061 			ac->ac_flags |= EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED;  in ext4_mb_choose_next_group_best_avail()
1067 	ac->ac_g_ex.fe_len = ac->ac_orig_goal_len;  in ext4_mb_choose_next_group_best_avail()
1073 	if (unlikely(!test_opt2(ac->ac_sb, MB_OPTIMIZE_SCAN)))  in should_optimize_scan()
1075 	if (ac->ac_criteria >= CR_GOAL_LEN_SLOW)  in should_optimize_scan()
1077 	if (!ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))  in should_optimize_scan()
1093 	if (ac->ac_groups_linear_remaining) {  in next_linear_group()
1094 		ac->ac_groups_linear_remaining--;  in next_linear_group()
1101 	 * Artificially restricted ngroups for non-extent  in next_linear_group()
1123 	*new_cr = ac->ac_criteria;  in ext4_mb_choose_next_group()
1125 	if (!should_optimize_scan(ac) || ac->ac_groups_linear_remaining) {  in ext4_mb_choose_next_group()
1155 	for (i = MB_NUM_ORDERS(sb) - 1; i >= 0; i--)  in mb_set_largest_free_order()
1156 		if (grp->bb_counters[i] > 0)  in mb_set_largest_free_order()
1160 	    i == grp->bb_largest_free_order) {  in mb_set_largest_free_order()
1161 		grp->bb_largest_free_order = i;  in mb_set_largest_free_order()
1165 	if (grp->bb_largest_free_order >= 0) {  in mb_set_largest_free_order()
1166 		write_lock(&sbi->s_mb_largest_free_orders_locks[  in mb_set_largest_free_order()
1167 					      grp->bb_largest_free_order]);  in mb_set_largest_free_order()
1168 		list_del_init(&grp->bb_largest_free_order_node);  in mb_set_largest_free_order()
1169 		write_unlock(&sbi->s_mb_largest_free_orders_locks[  in mb_set_largest_free_order()
1170 					      grp->bb_largest_free_order]);  in mb_set_largest_free_order()
1172 	grp->bb_largest_free_order = i;  in mb_set_largest_free_order()
1173 	if (grp->bb_largest_free_order >= 0 && grp->bb_free) {  in mb_set_largest_free_order()
1174 		write_lock(&sbi->s_mb_largest_free_orders_locks[  in mb_set_largest_free_order()
1175 					      grp->bb_largest_free_order]);  in mb_set_largest_free_order()
1176 		list_add_tail(&grp->bb_largest_free_order_node,  in mb_set_largest_free_order()
1177 		      &sbi->s_mb_largest_free_orders[grp->bb_largest_free_order]);  in mb_set_largest_free_order()
1178 		write_unlock(&sbi->s_mb_largest_free_orders_locks[  in mb_set_largest_free_order()
1179 					      grp->bb_largest_free_order]);  in mb_set_largest_free_order()
1198 	 * of on-disk bitmap and preallocations */  in ext4_mb_generate_buddy()
1200 	grp->bb_first_free = i;  in ext4_mb_generate_buddy()
1205 		len = i - first;  in ext4_mb_generate_buddy()
1210 			grp->bb_counters[0]++;  in ext4_mb_generate_buddy()
1214 	grp->bb_fragments = fragments;  in ext4_mb_generate_buddy()
1216 	if (free != grp->bb_free) {  in ext4_mb_generate_buddy()
1220 				      free, grp->bb_free);  in ext4_mb_generate_buddy()
1225 		grp->bb_free = free;  in ext4_mb_generate_buddy()
1232 	clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state));  in ext4_mb_generate_buddy()
1234 	period = get_cycles() - period;  in ext4_mb_generate_buddy()
1235 	atomic_inc(&sbi->s_mb_buddies_generated);  in ext4_mb_generate_buddy()
1236 	atomic64_add(period, &sbi->s_mb_generation_time);  in ext4_mb_generate_buddy()
1248 	e4b->bd_info->bb_fragments = 0;  in mb_regenerate_buddy()
1249 	memset(e4b->bd_info->bb_counters, 0,  in mb_regenerate_buddy()
1250 		sizeof(*e4b->bd_info->bb_counters) *  in mb_regenerate_buddy()
1251 		(e4b->bd_sb->s_blocksize_bits + 2));  in mb_regenerate_buddy()
1253 	ext4_mb_generate_buddy(e4b->bd_sb, e4b->bd_buddy,  in mb_regenerate_buddy()
1254 		e4b->bd_bitmap, e4b->bd_group, e4b->bd_info);  in mb_regenerate_buddy()
1295 	inode = page->mapping->host;  in ext4_mb_init_cache()
1296 	sb = inode->i_sb;  in ext4_mb_init_cache()
1301 	mb_debug(sb, "init page %lu\n", page->index);  in ext4_mb_init_cache()
1312 			return -ENOMEM;  in ext4_mb_init_cache()
1316 	first_group = page->index * blocks_per_page / 2;  in ext4_mb_init_cache()
1356 	first_block = page->index * blocks_per_page;  in ext4_mb_init_cache()
1362 		if (!bh[group - first_group])  in ext4_mb_init_cache()
1366 		if (!buffer_verified(bh[group - first_group]))  in ext4_mb_init_cache()
1378 		bitmap = bh[group - first_group]->b_data;  in ext4_mb_init_cache()
1386 			err = -EFSCORRUPTED;  in ext4_mb_init_cache()
1393 				group, page->index, i * blocksize);  in ext4_mb_init_cache()
1395 			grinfo->bb_fragments = 0;  in ext4_mb_init_cache()
1396 			memset(grinfo->bb_counters, 0,  in ext4_mb_init_cache()
1397 			       sizeof(*grinfo->bb_counters) *  in ext4_mb_init_cache()
1412 				group, page->index, i * blocksize);  in ext4_mb_init_cache()
1419 			/* mark all preallocated blks used in in-core bitmap */  in ext4_mb_init_cache()
1421 			WARN_ON_ONCE(!RB_EMPTY_ROOT(&grinfo->bb_free_root));  in ext4_mb_init_cache()
1446  * are on the same page e4b->bd_buddy_page is NULL and return value is 0.
1451 	struct inode *inode = EXT4_SB(sb)->s_buddy_cache;  in ext4_mb_get_buddy_page_lock()
1456 	e4b->bd_buddy_page = NULL;  in ext4_mb_get_buddy_page_lock()
1457 	e4b->bd_bitmap_page = NULL;  in ext4_mb_get_buddy_page_lock()
1459 	blocks_per_page = PAGE_SIZE / sb->s_blocksize;  in ext4_mb_get_buddy_page_lock()
1468 	page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_get_buddy_page_lock()
1470 		return -ENOMEM;  in ext4_mb_get_buddy_page_lock()
1471 	BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_get_buddy_page_lock()
1472 	e4b->bd_bitmap_page = page;  in ext4_mb_get_buddy_page_lock()
1473 	e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);  in ext4_mb_get_buddy_page_lock()
1482 	page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_get_buddy_page_lock()
1484 		return -ENOMEM;  in ext4_mb_get_buddy_page_lock()
1485 	BUG_ON(page->mapping != inode->i_mapping);  in ext4_mb_get_buddy_page_lock()
1486 	e4b->bd_buddy_page = page;  in ext4_mb_get_buddy_page_lock()
1492 	if (e4b->bd_bitmap_page) {  in ext4_mb_put_buddy_page_lock()
1493 		unlock_page(e4b->bd_bitmap_page);  in ext4_mb_put_buddy_page_lock()
1494 		put_page(e4b->bd_bitmap_page);  in ext4_mb_put_buddy_page_lock()
1496 	if (e4b->bd_buddy_page) {  in ext4_mb_put_buddy_page_lock()
1497 		unlock_page(e4b->bd_buddy_page);  in ext4_mb_put_buddy_page_lock()
1498 		put_page(e4b->bd_buddy_page);  in ext4_mb_put_buddy_page_lock()
1520 		return -EFSCORRUPTED;  in ext4_mb_init_group()
1545 		ret = -EIO;  in ext4_mb_init_group()
1564 		ret = -EIO;  in ext4_mb_init_group()
1589 	struct inode *inode = sbi->s_buddy_cache;  in ext4_mb_load_buddy_gfp()
1594 	blocks_per_page = PAGE_SIZE / sb->s_blocksize;  in ext4_mb_load_buddy_gfp()
1597 		return -EFSCORRUPTED;  in ext4_mb_load_buddy_gfp()
1599 	e4b->bd_blkbits = sb->s_blocksize_bits;  in ext4_mb_load_buddy_gfp()
1600 	e4b->bd_info = grp;  in ext4_mb_load_buddy_gfp()
1601 	e4b->bd_sb = sb;  in ext4_mb_load_buddy_gfp()
1602 	e4b->bd_group = group;  in ext4_mb_load_buddy_gfp()
1603 	e4b->bd_buddy_page = NULL;  in ext4_mb_load_buddy_gfp()
1604 	e4b->bd_bitmap_page = NULL;  in ext4_mb_load_buddy_gfp()
1627 	page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED);  in ext4_mb_load_buddy_gfp()
1639 		page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_load_buddy_gfp()
1641 			if (WARN_RATELIMIT(page->mapping != inode->i_mapping,  in ext4_mb_load_buddy_gfp()
1642 	"ext4: bitmap's paging->mapping != inode->i_mapping\n")) {  in ext4_mb_load_buddy_gfp()
1645 				ret = -EINVAL;  in ext4_mb_load_buddy_gfp()
1655 					       (poff * sb->s_blocksize));  in ext4_mb_load_buddy_gfp()
1661 		ret = -ENOMEM;  in ext4_mb_load_buddy_gfp()
1665 		ret = -EIO;  in ext4_mb_load_buddy_gfp()
1670 	e4b->bd_bitmap_page = page;  in ext4_mb_load_buddy_gfp()
1671 	e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);  in ext4_mb_load_buddy_gfp()
1677 	page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED);  in ext4_mb_load_buddy_gfp()
1681 		page = find_or_create_page(inode->i_mapping, pnum, gfp);  in ext4_mb_load_buddy_gfp()
1683 			if (WARN_RATELIMIT(page->mapping != inode->i_mapping,  in ext4_mb_load_buddy_gfp()
1684 	"ext4: buddy bitmap's page->mapping != inode->i_mapping\n")) {  in ext4_mb_load_buddy_gfp()
1687 				ret = -EINVAL;  in ext4_mb_load_buddy_gfp()
1691 				ret = ext4_mb_init_cache(page, e4b->bd_bitmap,  in ext4_mb_load_buddy_gfp()
1702 		ret = -ENOMEM;  in ext4_mb_load_buddy_gfp()
1706 		ret = -EIO;  in ext4_mb_load_buddy_gfp()
1711 	e4b->bd_buddy_page = page;  in ext4_mb_load_buddy_gfp()
1712 	e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize);  in ext4_mb_load_buddy_gfp()
1719 	if (e4b->bd_bitmap_page)  in ext4_mb_load_buddy_gfp()
1720 		put_page(e4b->bd_bitmap_page);  in ext4_mb_load_buddy_gfp()
1722 	e4b->bd_buddy = NULL;  in ext4_mb_load_buddy_gfp()
1723 	e4b->bd_bitmap = NULL;  in ext4_mb_load_buddy_gfp()
1735 	if (e4b->bd_bitmap_page)  in ext4_mb_unload_buddy()
1736 		put_page(e4b->bd_bitmap_page);  in ext4_mb_unload_buddy()
1737 	if (e4b->bd_buddy_page)  in ext4_mb_unload_buddy()
1738 		put_page(e4b->bd_buddy_page);  in ext4_mb_unload_buddy()
1747 	BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);  in mb_find_order_for_block()
1748 	BUG_ON(block >= (1 << (e4b->bd_blkbits + 3)));  in mb_find_order_for_block()
1750 	while (order <= e4b->bd_blkbits + 1) {  in mb_find_order_for_block()
1767 		if ((cur & 31) == 0 && (len - cur) >= 32) {  in mb_clear_bits()
1780  * will return first found zero bit if any, -1 otherwise
1785 	int zero_bit = -1;  in mb_test_and_clear_bits()
1789 		if ((cur & 31) == 0 && (len - cur) >= 32) {  in mb_test_and_clear_bits()
1792 			if (*addr != (__u32)(-1) && zero_bit == -1)  in mb_test_and_clear_bits()
1798 		if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1)  in mb_test_and_clear_bits()
1812 		if ((cur & 31) == 0 && (len - cur) >= 32) {  in mb_set_bits()
1828 		(*bit) -= side;  in mb_buddy_adjust_border()
1834 		return -1;  in mb_buddy_adjust_border()
1857 		 * ---------------------------------  in mb_buddy_mark_free()
1859 		 * ---------------------------------  in mb_buddy_mark_free()
1861 		 * ---------------------------------  in mb_buddy_mark_free()
1877 			e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1);  in mb_buddy_mark_free()
1879 			e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1);  in mb_buddy_mark_free()
1886 			mb_clear_bits(buddy, first, last - first + 1);  in mb_buddy_mark_free()
1887 			e4b->bd_info->bb_counters[order - 1] += last - first + 1;  in mb_buddy_mark_free()
1902 	int last = first + count - 1;  in mb_free_blocks()
1903 	struct super_block *sb = e4b->bd_sb;  in mb_free_blocks()
1907 	BUG_ON(last >= (sb->s_blocksize << 3));  in mb_free_blocks()
1908 	assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));  in mb_free_blocks()
1910 	if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))  in mb_free_blocks()
1920 		left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap);  in mb_free_blocks()
1921 	block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count);  in mb_free_blocks()
1922 	if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0])  in mb_free_blocks()
1923 		right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap);  in mb_free_blocks()
1925 	if (unlikely(block != -1)) {  in mb_free_blocks()
1933 		if (sbi->s_mount_state & EXT4_FC_REPLAY) {  in mb_free_blocks()
1938 		blocknr = ext4_group_first_block_no(sb, e4b->bd_group);  in mb_free_blocks()
1940 		ext4_grp_locked_error(sb, e4b->bd_group,  in mb_free_blocks()
1941 				      inode ? inode->i_ino : 0, blocknr,  in mb_free_blocks()
1944 		ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in mb_free_blocks()
1950 	e4b->bd_info->bb_free += count;  in mb_free_blocks()
1951 	if (first < e4b->bd_info->bb_first_free)  in mb_free_blocks()
1952 		e4b->bd_info->bb_first_free = first;  in mb_free_blocks()
1956 		e4b->bd_info->bb_fragments--;  in mb_free_blocks()
1958 		e4b->bd_info->bb_fragments++;  in mb_free_blocks()
1968 		e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1;  in mb_free_blocks()
1971 		last -= !right_is_free;  in mb_free_blocks()
1972 		e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1;  in mb_free_blocks()
1978 	mb_set_largest_free_order(sb, e4b->bd_info);  in mb_free_blocks()
1979 	mb_update_avg_fragment_size(sb, e4b->bd_info);  in mb_free_blocks()
1991 	assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));  in mb_find_extent()
1998 		ex->fe_len = 0;  in mb_find_extent()
1999 		ex->fe_start = 0;  in mb_find_extent()
2000 		ex->fe_group = 0;  in mb_find_extent()
2008 	ex->fe_len = 1 << order;  in mb_find_extent()
2009 	ex->fe_start = block << order;  in mb_find_extent()
2010 	ex->fe_group = e4b->bd_group;  in mb_find_extent()
2013 	next = next - ex->fe_start;  in mb_find_extent()
2014 	ex->fe_len -= next;  in mb_find_extent()
2015 	ex->fe_start += next;  in mb_find_extent()
2017 	while (needed > ex->fe_len &&  in mb_find_extent()
2024 		if (mb_test_bit(next, e4b->bd_bitmap))  in mb_find_extent()
2030 		ex->fe_len += 1 << order;  in mb_find_extent()
2033 	if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) {  in mb_find_extent()
2036 		ext4_grp_locked_error(e4b->bd_sb, e4b->bd_group, 0, 0,  in mb_find_extent()
2039 			block, order, needed, ex->fe_group, ex->fe_start,  in mb_find_extent()
2040 			ex->fe_len, ex->fe_logical);  in mb_find_extent()
2041 		ex->fe_len = 0;  in mb_find_extent()
2042 		ex->fe_start = 0;  in mb_find_extent()
2043 		ex->fe_group = 0;  in mb_find_extent()
2045 	return ex->fe_len;  in mb_find_extent()
2054 	int start = ex->fe_start;  in mb_mark_used()
2055 	int len = ex->fe_len;  in mb_mark_used()
2061 	BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3));  in mb_mark_used()
2062 	BUG_ON(e4b->bd_group != ex->fe_group);  in mb_mark_used()
2063 	assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));  in mb_mark_used()
2068 	e4b->bd_info->bb_free -= len;  in mb_mark_used()
2069 	if (e4b->bd_info->bb_first_free == start)  in mb_mark_used()
2070 		e4b->bd_info->bb_first_free += len;  in mb_mark_used()
2074 		mlen = !mb_test_bit(start - 1, e4b->bd_bitmap);  in mb_mark_used()
2075 	if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0])  in mb_mark_used()
2076 		max = !mb_test_bit(start + len, e4b->bd_bitmap);  in mb_mark_used()
2078 		e4b->bd_info->bb_fragments++;  in mb_mark_used()
2080 		e4b->bd_info->bb_fragments--;  in mb_mark_used()
2096 			e4b->bd_info->bb_counters[ord]--;  in mb_mark_used()
2098 			len -= mlen;  in mb_mark_used()
2111 		e4b->bd_info->bb_counters[ord]--;  in mb_mark_used()
2113 		ord--;  in mb_mark_used()
2118 		e4b->bd_info->bb_counters[ord]++;  in mb_mark_used()
2119 		e4b->bd_info->bb_counters[ord]++;  in mb_mark_used()
2122 	mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info);  in mb_mark_used()
2124 	mb_update_avg_fragment_size(e4b->bd_sb, e4b->bd_info);  in mb_mark_used()
2125 	mb_set_bits(e4b->bd_bitmap, ex->fe_start, len0);  in mb_mark_used()
2137 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_best_found()
2140 	BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);  in ext4_mb_use_best_found()
2141 	BUG_ON(ac->ac_status == AC_STATUS_FOUND);  in ext4_mb_use_best_found()
2143 	ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len);  in ext4_mb_use_best_found()
2144 	ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical;  in ext4_mb_use_best_found()
2145 	ret = mb_mark_used(e4b, &ac->ac_b_ex);  in ext4_mb_use_best_found()
2149 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_use_best_found()
2151 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_best_found()
2152 	ac->ac_tail = ret & 0xffff;  in ext4_mb_use_best_found()
2153 	ac->ac_buddy = ret >> 16;  in ext4_mb_use_best_found()
2162 	ac->ac_bitmap_page = e4b->bd_bitmap_page;  in ext4_mb_use_best_found()
2163 	get_page(ac->ac_bitmap_page);  in ext4_mb_use_best_found()
2164 	ac->ac_buddy_page = e4b->bd_buddy_page;  in ext4_mb_use_best_found()
2165 	get_page(ac->ac_buddy_page);  in ext4_mb_use_best_found()
2167 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {  in ext4_mb_use_best_found()
2168 		spin_lock(&sbi->s_md_lock);  in ext4_mb_use_best_found()
2169 		sbi->s_mb_last_group = ac->ac_f_ex.fe_group;  in ext4_mb_use_best_found()
2170 		sbi->s_mb_last_start = ac->ac_f_ex.fe_start;  in ext4_mb_use_best_found()
2171 		spin_unlock(&sbi->s_md_lock);  in ext4_mb_use_best_found()
2178 	if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len)  in ext4_mb_use_best_found()
2187 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_check_limits()
2188 	struct ext4_free_extent *bex = &ac->ac_b_ex;  in ext4_mb_check_limits()
2189 	struct ext4_free_extent *gex = &ac->ac_g_ex;  in ext4_mb_check_limits()
2191 	if (ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_check_limits()
2196 	if (ac->ac_found > sbi->s_mb_max_to_scan &&  in ext4_mb_check_limits()
2197 			!(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_check_limits()
2198 		ac->ac_status = AC_STATUS_BREAK;  in ext4_mb_check_limits()
2205 	if (bex->fe_len < gex->fe_len)  in ext4_mb_check_limits()
2208 	if (finish_group || ac->ac_found > sbi->s_mb_min_to_scan)  in ext4_mb_check_limits()
2226  *   upto a max of sbi->s_mb_max_to_scan times (default 200). After
2230  *   upto a max of sbi->s_mb_min_to_scan times (default 10) before
2240 	struct ext4_free_extent *bex = &ac->ac_b_ex;  in ext4_mb_measure_extent()
2241 	struct ext4_free_extent *gex = &ac->ac_g_ex;  in ext4_mb_measure_extent()
2243 	BUG_ON(ex->fe_len <= 0);  in ext4_mb_measure_extent()
2244 	BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));  in ext4_mb_measure_extent()
2245 	BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));  in ext4_mb_measure_extent()
2246 	BUG_ON(ac->ac_status != AC_STATUS_CONTINUE);  in ext4_mb_measure_extent()
2248 	ac->ac_found++;  in ext4_mb_measure_extent()
2249 	ac->ac_cX_found[ac->ac_criteria]++;  in ext4_mb_measure_extent()
2252 	 * The special case - take what you catch first  in ext4_mb_measure_extent()
2254 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_measure_extent()
2263 	if (ex->fe_len == gex->fe_len) {  in ext4_mb_measure_extent()
2272 	if (bex->fe_len == 0) {  in ext4_mb_measure_extent()
2280 	if (bex->fe_len < gex->fe_len) {  in ext4_mb_measure_extent()
2283 		if (ex->fe_len > bex->fe_len)  in ext4_mb_measure_extent()
2285 	} else if (ex->fe_len > gex->fe_len) {  in ext4_mb_measure_extent()
2289 		if (ex->fe_len < bex->fe_len)  in ext4_mb_measure_extent()
2300 	struct ext4_free_extent ex = ac->ac_b_ex;  in ext4_mb_try_best_found()
2306 	err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);  in ext4_mb_try_best_found()
2310 	ext4_lock_group(ac->ac_sb, group);  in ext4_mb_try_best_found()
2311 	if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))  in ext4_mb_try_best_found()
2317 		ac->ac_b_ex = ex;  in ext4_mb_try_best_found()
2322 	ext4_unlock_group(ac->ac_sb, group);  in ext4_mb_try_best_found()
2330 	ext4_group_t group = ac->ac_g_ex.fe_group;  in ext4_mb_find_by_goal()
2333 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_find_by_goal()
2334 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_find_by_goal()
2338 		return -EFSCORRUPTED;  in ext4_mb_find_by_goal()
2339 	if (!(ac->ac_flags & (EXT4_MB_HINT_TRY_GOAL | EXT4_MB_HINT_GOAL_ONLY)))  in ext4_mb_find_by_goal()
2341 	if (grp->bb_free == 0)  in ext4_mb_find_by_goal()
2344 	err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);  in ext4_mb_find_by_goal()
2348 	ext4_lock_group(ac->ac_sb, group);  in ext4_mb_find_by_goal()
2349 	if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))  in ext4_mb_find_by_goal()
2352 	max = mb_find_extent(e4b, ac->ac_g_ex.fe_start,  in ext4_mb_find_by_goal()
2353 			     ac->ac_g_ex.fe_len, &ex);  in ext4_mb_find_by_goal()
2356 	if (max >= ac->ac_g_ex.fe_len &&  in ext4_mb_find_by_goal()
2357 	    ac->ac_g_ex.fe_len == EXT4_B2C(sbi, sbi->s_stripe)) {  in ext4_mb_find_by_goal()
2360 		start = ext4_grp_offs_to_block(ac->ac_sb, &ex);  in ext4_mb_find_by_goal()
2361 		/* use do_div to get remainder (would be 64-bit modulo) */  in ext4_mb_find_by_goal()
2362 		if (do_div(start, sbi->s_stripe) == 0) {  in ext4_mb_find_by_goal()
2363 			ac->ac_found++;  in ext4_mb_find_by_goal()
2364 			ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
2367 	} else if (max >= ac->ac_g_ex.fe_len) {  in ext4_mb_find_by_goal()
2369 		BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);  in ext4_mb_find_by_goal()
2370 		BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);  in ext4_mb_find_by_goal()
2371 		ac->ac_found++;  in ext4_mb_find_by_goal()
2372 		ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
2374 	} else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) {  in ext4_mb_find_by_goal()
2378 		BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);  in ext4_mb_find_by_goal()
2379 		BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);  in ext4_mb_find_by_goal()
2380 		ac->ac_found++;  in ext4_mb_find_by_goal()
2381 		ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
2385 	ext4_unlock_group(ac->ac_sb, group);  in ext4_mb_find_by_goal()
2399 	struct super_block *sb = ac->ac_sb;  in ext4_mb_simple_scan_group()
2400 	struct ext4_group_info *grp = e4b->bd_info;  in ext4_mb_simple_scan_group()
2406 	BUG_ON(ac->ac_2order <= 0);  in ext4_mb_simple_scan_group()
2407 	for (i = ac->ac_2order; i < MB_NUM_ORDERS(sb); i++) {  in ext4_mb_simple_scan_group()
2408 		if (grp->bb_counters[i] == 0)  in ext4_mb_simple_scan_group()
2418 			ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0,  in ext4_mb_simple_scan_group()
2420 				grp->bb_counters[i], i);  in ext4_mb_simple_scan_group()
2421 			ext4_mark_group_bitmap_corrupted(ac->ac_sb,  in ext4_mb_simple_scan_group()
2422 					 e4b->bd_group,  in ext4_mb_simple_scan_group()
2426 		ac->ac_found++;  in ext4_mb_simple_scan_group()
2427 		ac->ac_cX_found[ac->ac_criteria]++;  in ext4_mb_simple_scan_group()
2429 		ac->ac_b_ex.fe_len = 1 << i;  in ext4_mb_simple_scan_group()
2430 		ac->ac_b_ex.fe_start = k << i;  in ext4_mb_simple_scan_group()
2431 		ac->ac_b_ex.fe_group = e4b->bd_group;  in ext4_mb_simple_scan_group()
2435 		BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len);  in ext4_mb_simple_scan_group()
2437 		if (EXT4_SB(sb)->s_mb_stats)  in ext4_mb_simple_scan_group()
2438 			atomic_inc(&EXT4_SB(sb)->s_bal_2orders);  in ext4_mb_simple_scan_group()
2453 	struct super_block *sb = ac->ac_sb;  in ext4_mb_complex_scan_group()
2454 	void *bitmap = e4b->bd_bitmap;  in ext4_mb_complex_scan_group()
2459 	free = e4b->bd_info->bb_free;  in ext4_mb_complex_scan_group()
2463 	i = e4b->bd_info->bb_first_free;  in ext4_mb_complex_scan_group()
2465 	while (free && ac->ac_status == AC_STATUS_CONTINUE) {  in ext4_mb_complex_scan_group()
2474 			ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,  in ext4_mb_complex_scan_group()
2478 			ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in ext4_mb_complex_scan_group()
2483 		if (!ext4_mb_cr_expensive(ac->ac_criteria)) {  in ext4_mb_complex_scan_group()
2492 			freelen = j - i;  in ext4_mb_complex_scan_group()
2494 			if (freelen < ac->ac_g_ex.fe_len) {  in ext4_mb_complex_scan_group()
2496 				free -= freelen;  in ext4_mb_complex_scan_group()
2501 		mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex);  in ext4_mb_complex_scan_group()
2505 			ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,  in ext4_mb_complex_scan_group()
2509 			ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,  in ext4_mb_complex_scan_group()
2522 		free -= ex.fe_len;  in ext4_mb_complex_scan_group()
2530  * we try to find stripe-aligned chunks for stripe-size-multiple requests
2536 	struct super_block *sb = ac->ac_sb;  in ext4_mb_scan_aligned()
2538 	void *bitmap = e4b->bd_bitmap;  in ext4_mb_scan_aligned()
2545 	BUG_ON(sbi->s_stripe == 0);  in ext4_mb_scan_aligned()
2547 	/* find first stripe-aligned block in group */  in ext4_mb_scan_aligned()
2548 	first_group_block = ext4_group_first_block_no(sb, e4b->bd_group);  in ext4_mb_scan_aligned()
2550 	a = first_group_block + sbi->s_stripe - 1;  in ext4_mb_scan_aligned()
2551 	do_div(a, sbi->s_stripe);  in ext4_mb_scan_aligned()
2552 	i = (a * sbi->s_stripe) - first_group_block;  in ext4_mb_scan_aligned()
2554 	stripe = EXT4_B2C(sbi, sbi->s_stripe);  in ext4_mb_scan_aligned()
2560 				ac->ac_found++;  in ext4_mb_scan_aligned()
2561 				ac->ac_cX_found[ac->ac_criteria]++;  in ext4_mb_scan_aligned()
2563 				ac->ac_b_ex = ex;  in ext4_mb_scan_aligned()
2581 	int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb));  in ext4_mb_good_group()
2582 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_good_group()
2589 	free = grp->bb_free;  in ext4_mb_good_group()
2593 	fragments = grp->bb_fragments;  in ext4_mb_good_group()
2599 		BUG_ON(ac->ac_2order == 0);  in ext4_mb_good_group()
2602 		if ((ac->ac_flags & EXT4_MB_HINT_DATA) &&  in ext4_mb_good_group()
2607 		if (free < ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2610 		if (ac->ac_2order >= MB_NUM_ORDERS(ac->ac_sb))  in ext4_mb_good_group()
2613 		if (grp->bb_largest_free_order < ac->ac_2order)  in ext4_mb_good_group()
2619 		if ((free / fragments) >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2623 		if (free >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2649 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_good_group_nolock()
2650 	struct super_block *sb = ac->ac_sb;  in ext4_mb_good_group_nolock()
2652 	bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK;  in ext4_mb_good_group_nolock()
2657 		return -EFSCORRUPTED;  in ext4_mb_good_group_nolock()
2658 	if (sbi->s_mb_stats)  in ext4_mb_good_group_nolock()
2659 		atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]);  in ext4_mb_good_group_nolock()
2664 	free = grp->bb_free;  in ext4_mb_good_group_nolock()
2672 	if (cr < CR_ANY_FREE && free < ac->ac_g_ex.fe_len)  in ext4_mb_good_group_nolock()
2697 		    (!sbi->s_log_groups_per_flex ||  in ext4_mb_good_group_nolock()
2698 		     ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) &&  in ext4_mb_good_group_nolock()
2700 		      (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))))  in ext4_mb_good_group_nolock()
2732 	while (nr-- > 0) {  in ext4_mb_prefetch()
2779 	while (nr-- > 0) {  in ext4_mb_prefetch_fini()
2782 		group--;  in ext4_mb_prefetch_fini()
2806 	sb = ac->ac_sb;  in ext4_mb_regular_allocator()
2809 	/* non-extent files are limited to low blocks/groups */  in ext4_mb_regular_allocator()
2810 	if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)))  in ext4_mb_regular_allocator()
2811 		ngroups = sbi->s_blockfile_groups;  in ext4_mb_regular_allocator()
2813 	BUG_ON(ac->ac_status == AC_STATUS_FOUND);  in ext4_mb_regular_allocator()
2817 	if (err || ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_regular_allocator()
2820 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_regular_allocator()
2824 	 * ac->ac_2order is set only if the fe_len is a power of 2  in ext4_mb_regular_allocator()
2825 	 * if ac->ac_2order is set we also set criteria to CR_POWER2_ALIGNED  in ext4_mb_regular_allocator()
2828 	i = fls(ac->ac_g_ex.fe_len);  in ext4_mb_regular_allocator()
2829 	ac->ac_2order = 0;  in ext4_mb_regular_allocator()
2834 	 * We also support searching for power-of-two requests only for  in ext4_mb_regular_allocator()
2837 	if (i >= sbi->s_mb_order2_reqs && i <= MB_NUM_ORDERS(sb)) {  in ext4_mb_regular_allocator()
2838 		if (is_power_of_2(ac->ac_g_ex.fe_len))  in ext4_mb_regular_allocator()
2839 			ac->ac_2order = array_index_nospec(i - 1,  in ext4_mb_regular_allocator()
2844 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {  in ext4_mb_regular_allocator()
2846 		spin_lock(&sbi->s_md_lock);  in ext4_mb_regular_allocator()
2847 		ac->ac_g_ex.fe_group = sbi->s_mb_last_group;  in ext4_mb_regular_allocator()
2848 		ac->ac_g_ex.fe_start = sbi->s_mb_last_start;  in ext4_mb_regular_allocator()
2849 		spin_unlock(&sbi->s_md_lock);  in ext4_mb_regular_allocator()
2853 	 * Let's just scan groups to find more-less suitable blocks We  in ext4_mb_regular_allocator()
2857 	if (ac->ac_2order)  in ext4_mb_regular_allocator()
2860 	for (; cr < EXT4_MB_NUM_CRS && ac->ac_status == AC_STATUS_CONTINUE; cr++) {  in ext4_mb_regular_allocator()
2861 		ac->ac_criteria = cr;  in ext4_mb_regular_allocator()
2866 		group = ac->ac_g_ex.fe_group;  in ext4_mb_regular_allocator()
2867 		ac->ac_groups_linear_remaining = sbi->s_mb_max_linear_groups;  in ext4_mb_regular_allocator()
2888 			     prefetch_ios < sbi->s_mb_prefetch_limit)) {  in ext4_mb_regular_allocator()
2889 				nr = sbi->s_mb_prefetch;  in ext4_mb_regular_allocator()
2891 					nr = 1 << sbi->s_log_groups_per_flex;  in ext4_mb_regular_allocator()
2892 					nr -= group & (nr - 1);  in ext4_mb_regular_allocator()
2893 					nr = min(nr, sbi->s_mb_prefetch);  in ext4_mb_regular_allocator()
2924 			ac->ac_groups_scanned++;  in ext4_mb_regular_allocator()
2929 				 sbi->s_stripe &&  in ext4_mb_regular_allocator()
2930 				 !(ac->ac_g_ex.fe_len %  in ext4_mb_regular_allocator()
2931 				 EXT4_B2C(sbi, sbi->s_stripe)))  in ext4_mb_regular_allocator()
2939 			if (ac->ac_status != AC_STATUS_CONTINUE)  in ext4_mb_regular_allocator()
2943 		if (sbi->s_mb_stats && i == ngroups)  in ext4_mb_regular_allocator()
2944 			atomic64_inc(&sbi->s_bal_cX_failed[cr]);  in ext4_mb_regular_allocator()
2946 		if (i == ngroups && ac->ac_criteria == CR_BEST_AVAIL_LEN)  in ext4_mb_regular_allocator()
2949 			ac->ac_g_ex.fe_len = ac->ac_orig_goal_len;  in ext4_mb_regular_allocator()
2952 	if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND &&  in ext4_mb_regular_allocator()
2953 	    !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_regular_allocator()
2959 		if (ac->ac_status != AC_STATUS_FOUND) {  in ext4_mb_regular_allocator()
2965 			lost = atomic_inc_return(&sbi->s_mb_lost_chunks);  in ext4_mb_regular_allocator()
2967 				 ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start,  in ext4_mb_regular_allocator()
2968 				 ac->ac_b_ex.fe_len, lost);  in ext4_mb_regular_allocator()
2970 			ac->ac_b_ex.fe_group = 0;  in ext4_mb_regular_allocator()
2971 			ac->ac_b_ex.fe_start = 0;  in ext4_mb_regular_allocator()
2972 			ac->ac_b_ex.fe_len = 0;  in ext4_mb_regular_allocator()
2973 			ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_regular_allocator()
2974 			ac->ac_flags |= EXT4_MB_HINT_FIRST;  in ext4_mb_regular_allocator()
2980 	if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_regular_allocator()
2981 		atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]);  in ext4_mb_regular_allocator()
2983 	if (!err && ac->ac_status != AC_STATUS_FOUND && first_err)  in ext4_mb_regular_allocator()
2987 		 ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status,  in ext4_mb_regular_allocator()
2988 		 ac->ac_flags, cr, err);  in ext4_mb_regular_allocator()
2998 	struct super_block *sb = pde_data(file_inode(seq->file));  in ext4_mb_seq_groups_start()
3009 	struct super_block *sb = pde_data(file_inode(seq->file));  in ext4_mb_seq_groups_next()
3021 	struct super_block *sb = pde_data(file_inode(seq->file));  in ext4_mb_seq_groups_show()
3028 					     sb->s_blocksize_bits,  in ext4_mb_seq_groups_show()
3035 	group--;  in ext4_mb_seq_groups_show()
3051 			seq_printf(seq, "#%-5u: I/O error\n", group);  in ext4_mb_seq_groups_show()
3062 	seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free,  in ext4_mb_seq_groups_show()
3065 		seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ?  in ext4_mb_seq_groups_show()
3088 	struct super_block *sb = seq->private;  in ext4_seq_mb_stats_show()
3092 	if (!sbi->s_mb_stats) {  in ext4_seq_mb_stats_show()
3099 	seq_printf(seq, "\treqs: %u\n", atomic_read(&sbi->s_bal_reqs));  in ext4_seq_mb_stats_show()
3100 	seq_printf(seq, "\tsuccess: %u\n", atomic_read(&sbi->s_bal_success));  in ext4_seq_mb_stats_show()
3103 		   atomic_read(&sbi->s_bal_groups_scanned));  in ext4_seq_mb_stats_show()
3108 		   atomic64_read(&sbi->s_bal_cX_hits[CR_POWER2_ALIGNED]));  in ext4_seq_mb_stats_show()
3112 			&sbi->s_bal_cX_groups_considered[CR_POWER2_ALIGNED]));  in ext4_seq_mb_stats_show()
3114 		   atomic_read(&sbi->s_bal_cX_ex_scanned[CR_POWER2_ALIGNED]));  in ext4_seq_mb_stats_show()
3116 		   atomic64_read(&sbi->s_bal_cX_failed[CR_POWER2_ALIGNED]));  in ext4_seq_mb_stats_show()
3118 		   atomic_read(&sbi->s_bal_p2_aligned_bad_suggestions));  in ext4_seq_mb_stats_show()
3123 		   atomic64_read(&sbi->s_bal_cX_hits[CR_GOAL_LEN_FAST]));  in ext4_seq_mb_stats_show()
3126 			   &sbi->s_bal_cX_groups_considered[CR_GOAL_LEN_FAST]));  in ext4_seq_mb_stats_show()
3128 		   atomic_read(&sbi->s_bal_cX_ex_scanned[CR_GOAL_LEN_FAST]));  in ext4_seq_mb_stats_show()
3130 		   atomic64_read(&sbi->s_bal_cX_failed[CR_GOAL_LEN_FAST]));  in ext4_seq_mb_stats_show()
3132 		   atomic_read(&sbi->s_bal_goal_fast_bad_suggestions));  in ext4_seq_mb_stats_show()
3137 		   atomic64_read(&sbi->s_bal_cX_hits[CR_BEST_AVAIL_LEN]));  in ext4_seq_mb_stats_show()
3141 			&sbi->s_bal_cX_groups_considered[CR_BEST_AVAIL_LEN]));  in ext4_seq_mb_stats_show()
3143 		   atomic_read(&sbi->s_bal_cX_ex_scanned[CR_BEST_AVAIL_LEN]));  in ext4_seq_mb_stats_show()
3145 		   atomic64_read(&sbi->s_bal_cX_failed[CR_BEST_AVAIL_LEN]));  in ext4_seq_mb_stats_show()
3147 		   atomic_read(&sbi->s_bal_best_avail_bad_suggestions));  in ext4_seq_mb_stats_show()
3152 		   atomic64_read(&sbi->s_bal_cX_hits[CR_GOAL_LEN_SLOW]));  in ext4_seq_mb_stats_show()
3155 			   &sbi->s_bal_cX_groups_considered[CR_GOAL_LEN_SLOW]));  in ext4_seq_mb_stats_show()
3157 		   atomic_read(&sbi->s_bal_cX_ex_scanned[CR_GOAL_LEN_SLOW]));  in ext4_seq_mb_stats_show()
3159 		   atomic64_read(&sbi->s_bal_cX_failed[CR_GOAL_LEN_SLOW]));  in ext4_seq_mb_stats_show()
3164 		   atomic64_read(&sbi->s_bal_cX_hits[CR_ANY_FREE]));  in ext4_seq_mb_stats_show()
3167 		atomic64_read(&sbi->s_bal_cX_groups_considered[CR_ANY_FREE]));  in ext4_seq_mb_stats_show()
3169 		   atomic_read(&sbi->s_bal_cX_ex_scanned[CR_ANY_FREE]));  in ext4_seq_mb_stats_show()
3171 		   atomic64_read(&sbi->s_bal_cX_failed[CR_ANY_FREE]));  in ext4_seq_mb_stats_show()
3175 		   atomic_read(&sbi->s_bal_ex_scanned));  in ext4_seq_mb_stats_show()
3176 	seq_printf(seq, "\t\tgoal_hits: %u\n", atomic_read(&sbi->s_bal_goals));  in ext4_seq_mb_stats_show()
3178 		   atomic_read(&sbi->s_bal_len_goals));  in ext4_seq_mb_stats_show()
3179 	seq_printf(seq, "\t\t2^n_hits: %u\n", atomic_read(&sbi->s_bal_2orders));  in ext4_seq_mb_stats_show()
3180 	seq_printf(seq, "\t\tbreaks: %u\n", atomic_read(&sbi->s_bal_breaks));  in ext4_seq_mb_stats_show()
3181 	seq_printf(seq, "\t\tlost: %u\n", atomic_read(&sbi->s_mb_lost_chunks));  in ext4_seq_mb_stats_show()
3183 		   atomic_read(&sbi->s_mb_buddies_generated),  in ext4_seq_mb_stats_show()
3186 		   atomic64_read(&sbi->s_mb_generation_time));  in ext4_seq_mb_stats_show()
3188 		   atomic_read(&sbi->s_mb_preallocated));  in ext4_seq_mb_stats_show()
3189 	seq_printf(seq, "\tdiscarded: %u\n", atomic_read(&sbi->s_mb_discarded));  in ext4_seq_mb_stats_show()
3194 __acquires(&EXT4_SB(sb)->s_mb_rb_lock)  in ext4_mb_seq_structs_summary_start()
3196 	struct super_block *sb = pde_data(file_inode(seq->file));  in ext4_mb_seq_structs_summary_start()
3207 	struct super_block *sb = pde_data(file_inode(seq->file));  in ext4_mb_seq_structs_summary_next()
3219 	struct super_block *sb = pde_data(file_inode(seq->file));  in ext4_mb_seq_structs_summary_show()
3225 	position--;  in ext4_mb_seq_structs_summary_show()
3227 		position -= MB_NUM_ORDERS(sb);  in ext4_mb_seq_structs_summary_show()
3232 		read_lock(&sbi->s_mb_avg_fragment_size_locks[position]);  in ext4_mb_seq_structs_summary_show()
3233 		list_for_each_entry(grp, &sbi->s_mb_avg_fragment_size[position],  in ext4_mb_seq_structs_summary_show()
3236 		read_unlock(&sbi->s_mb_avg_fragment_size_locks[position]);  in ext4_mb_seq_structs_summary_show()
3248 	read_lock(&sbi->s_mb_largest_free_orders_locks[position]);  in ext4_mb_seq_structs_summary_show()
3249 	list_for_each_entry(grp, &sbi->s_mb_largest_free_orders[position],  in ext4_mb_seq_structs_summary_show()
3252 	read_unlock(&sbi->s_mb_largest_free_orders_locks[position]);  in ext4_mb_seq_structs_summary_show()
3272 	int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE;  in get_groupinfo_cache()
3280  * Allocate the top-level s_group_info array for the specified number
3289 	size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >>  in ext4_mb_alloc_groupinfo()
3291 	if (size <= sbi->s_group_info_size)  in ext4_mb_alloc_groupinfo()
3294 	size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size);  in ext4_mb_alloc_groupinfo()
3298 		return -ENOMEM;  in ext4_mb_alloc_groupinfo()
3301 	old_groupinfo = rcu_dereference(sbi->s_group_info);  in ext4_mb_alloc_groupinfo()
3304 		       sbi->s_group_info_size * sizeof(*sbi->s_group_info));  in ext4_mb_alloc_groupinfo()
3306 	rcu_assign_pointer(sbi->s_group_info, new_groupinfo);  in ext4_mb_alloc_groupinfo()
3307 	sbi->s_group_info_size = size / sizeof(*sbi->s_group_info);  in ext4_mb_alloc_groupinfo()
3311 		   sbi->s_group_info_size);  in ext4_mb_alloc_groupinfo()
3324 	struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);  in ext4_mb_add_groupinfo()
3338 			return -ENOMEM;  in ext4_mb_add_groupinfo()
3341 		rcu_dereference(sbi->s_group_info)[idx] = meta_group_info;  in ext4_mb_add_groupinfo()
3346 	i = group & (EXT4_DESC_PER_BLOCK(sb) - 1);  in ext4_mb_add_groupinfo()
3354 		&(meta_group_info[i]->bb_state));  in ext4_mb_add_groupinfo()
3361 	    (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {  in ext4_mb_add_groupinfo()
3362 		meta_group_info[i]->bb_free =  in ext4_mb_add_groupinfo()
3365 		meta_group_info[i]->bb_free =  in ext4_mb_add_groupinfo()
3369 	INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list);  in ext4_mb_add_groupinfo()
3370 	init_rwsem(&meta_group_info[i]->alloc_sem);  in ext4_mb_add_groupinfo()
3371 	meta_group_info[i]->bb_free_root = RB_ROOT;  in ext4_mb_add_groupinfo()
3372 	INIT_LIST_HEAD(&meta_group_info[i]->bb_largest_free_order_node);  in ext4_mb_add_groupinfo()
3373 	INIT_LIST_HEAD(&meta_group_info[i]->bb_avg_fragment_size_node);  in ext4_mb_add_groupinfo()
3374 	meta_group_info[i]->bb_largest_free_order = -1;  /* uninit */  in ext4_mb_add_groupinfo()
3375 	meta_group_info[i]->bb_avg_fragment_size_order = -1;  /* uninit */  in ext4_mb_add_groupinfo()
3376 	meta_group_info[i]->bb_group = group;  in ext4_mb_add_groupinfo()
3387 		group_info = rcu_dereference(sbi->s_group_info);  in ext4_mb_add_groupinfo()
3392 	return -ENOMEM;  in ext4_mb_add_groupinfo()
3409 	sbi->s_buddy_cache = new_inode(sb);  in ext4_mb_init_backend()
3410 	if (sbi->s_buddy_cache == NULL) {  in ext4_mb_init_backend()
3414 	/* To avoid potentially colliding with an valid on-disk inode number,  in ext4_mb_init_backend()
3418 	sbi->s_buddy_cache->i_ino = EXT4_BAD_INO;  in ext4_mb_init_backend()
3419 	EXT4_I(sbi->s_buddy_cache)->i_disksize = 0;  in ext4_mb_init_backend()
3436 		if (sbi->s_es->s_log_groups_per_flex >= 32) {  in ext4_mb_init_backend()
3440 		sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex,  in ext4_mb_init_backend()
3441 			BLK_MAX_SEGMENT_SIZE >> (sb->s_blocksize_bits - 9));  in ext4_mb_init_backend()
3442 		sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */  in ext4_mb_init_backend()
3444 		sbi->s_mb_prefetch = 32;  in ext4_mb_init_backend()
3446 	if (sbi->s_mb_prefetch > ext4_get_groups_count(sb))  in ext4_mb_init_backend()
3447 		sbi->s_mb_prefetch = ext4_get_groups_count(sb);  in ext4_mb_init_backend()
3449 	 * given cr=0 is an CPU-related optimization we shouldn't try to  in ext4_mb_init_backend()
3455 	sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4;  in ext4_mb_init_backend()
3456 	if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb))  in ext4_mb_init_backend()
3457 		sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb);  in ext4_mb_init_backend()
3462 	cachep = get_groupinfo_cache(sb->s_blocksize_bits);  in ext4_mb_init_backend()
3463 	while (i-- > 0) {  in ext4_mb_init_backend()
3469 	i = sbi->s_group_info_size;  in ext4_mb_init_backend()
3471 	group_info = rcu_dereference(sbi->s_group_info);  in ext4_mb_init_backend()
3472 	while (i-- > 0)  in ext4_mb_init_backend()
3475 	iput(sbi->s_buddy_cache);  in ext4_mb_init_backend()
3478 	kvfree(rcu_dereference(sbi->s_group_info));  in ext4_mb_init_backend()
3480 	return -ENOMEM;  in ext4_mb_init_backend()
3498 	int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE;  in ext4_groupinfo_create_slab()
3502 		return -EINVAL;  in ext4_groupinfo_create_slab()
3525 		       "EXT4-fs: no memory for groupinfo slab cache\n");  in ext4_groupinfo_create_slab()
3526 		return -ENOMEM;  in ext4_groupinfo_create_slab()
3536 	struct super_block *sb = sbi->s_sb;  in ext4_discard_work()
3543 	spin_lock(&sbi->s_md_lock);  in ext4_discard_work()
3544 	list_splice_init(&sbi->s_discard_list, &discard_list);  in ext4_discard_work()
3545 	spin_unlock(&sbi->s_md_lock);  in ext4_discard_work()
3553 		if ((sb->s_flags & SB_ACTIVE) && !err &&  in ext4_discard_work()
3554 		    !atomic_read(&sbi->s_retry_alloc_pending)) {  in ext4_discard_work()
3555 			grp = fd->efd_group;  in ext4_discard_work()
3571 			ext4_try_to_trim_range(sb, &e4b, fd->efd_start_cluster,  in ext4_discard_work()
3572 						fd->efd_start_cluster + fd->efd_count - 1, 1);  in ext4_discard_work()
3590 	i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_offsets);  in ext4_mb_init()
3592 	sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL);  in ext4_mb_init()
3593 	if (sbi->s_mb_offsets == NULL) {  in ext4_mb_init()
3594 		ret = -ENOMEM;  in ext4_mb_init()
3598 	i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_maxs);  in ext4_mb_init()
3599 	sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL);  in ext4_mb_init()
3600 	if (sbi->s_mb_maxs == NULL) {  in ext4_mb_init()
3601 		ret = -ENOMEM;  in ext4_mb_init()
3605 	ret = ext4_groupinfo_create_slab(sb->s_blocksize);  in ext4_mb_init()
3610 	sbi->s_mb_maxs[0] = sb->s_blocksize << 3;  in ext4_mb_init()
3611 	sbi->s_mb_offsets[0] = 0;  in ext4_mb_init()
3615 	offset_incr = 1 << (sb->s_blocksize_bits - 1);  in ext4_mb_init()
3616 	max = sb->s_blocksize << 2;  in ext4_mb_init()
3618 		sbi->s_mb_offsets[i] = offset;  in ext4_mb_init()
3619 		sbi->s_mb_maxs[i] = max;  in ext4_mb_init()
3626 	sbi->s_mb_avg_fragment_size =  in ext4_mb_init()
3629 	if (!sbi->s_mb_avg_fragment_size) {  in ext4_mb_init()
3630 		ret = -ENOMEM;  in ext4_mb_init()
3633 	sbi->s_mb_avg_fragment_size_locks =  in ext4_mb_init()
3636 	if (!sbi->s_mb_avg_fragment_size_locks) {  in ext4_mb_init()
3637 		ret = -ENOMEM;  in ext4_mb_init()
3641 		INIT_LIST_HEAD(&sbi->s_mb_avg_fragment_size[i]);  in ext4_mb_init()
3642 		rwlock_init(&sbi->s_mb_avg_fragment_size_locks[i]);  in ext4_mb_init()
3644 	sbi->s_mb_largest_free_orders =  in ext4_mb_init()
3647 	if (!sbi->s_mb_largest_free_orders) {  in ext4_mb_init()
3648 		ret = -ENOMEM;  in ext4_mb_init()
3651 	sbi->s_mb_largest_free_orders_locks =  in ext4_mb_init()
3654 	if (!sbi->s_mb_largest_free_orders_locks) {  in ext4_mb_init()
3655 		ret = -ENOMEM;  in ext4_mb_init()
3659 		INIT_LIST_HEAD(&sbi->s_mb_largest_free_orders[i]);  in ext4_mb_init()
3660 		rwlock_init(&sbi->s_mb_largest_free_orders_locks[i]);  in ext4_mb_init()
3663 	spin_lock_init(&sbi->s_md_lock);  in ext4_mb_init()
3664 	sbi->s_mb_free_pending = 0;  in ext4_mb_init()
3665 	INIT_LIST_HEAD(&sbi->s_freed_data_list);  in ext4_mb_init()
3666 	INIT_LIST_HEAD(&sbi->s_discard_list);  in ext4_mb_init()
3667 	INIT_WORK(&sbi->s_discard_work, ext4_discard_work);  in ext4_mb_init()
3668 	atomic_set(&sbi->s_retry_alloc_pending, 0);  in ext4_mb_init()
3670 	sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN;  in ext4_mb_init()
3671 	sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN;  in ext4_mb_init()
3672 	sbi->s_mb_stats = MB_DEFAULT_STATS;  in ext4_mb_init()
3673 	sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD;  in ext4_mb_init()
3674 	sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS;  in ext4_mb_init()
3675 	sbi->s_mb_best_avail_max_trim_order = MB_DEFAULT_BEST_AVAIL_TRIM_ORDER;  in ext4_mb_init()
3689 	sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >>  in ext4_mb_init()
3690 				       sbi->s_cluster_bits, 32);  in ext4_mb_init()
3699 	if (sbi->s_stripe > 1) {  in ext4_mb_init()
3700 		sbi->s_mb_group_prealloc = roundup(  in ext4_mb_init()
3701 			sbi->s_mb_group_prealloc, EXT4_B2C(sbi, sbi->s_stripe));  in ext4_mb_init()
3704 	sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group);  in ext4_mb_init()
3705 	if (sbi->s_locality_groups == NULL) {  in ext4_mb_init()
3706 		ret = -ENOMEM;  in ext4_mb_init()
3711 		lg = per_cpu_ptr(sbi->s_locality_groups, i);  in ext4_mb_init()
3712 		mutex_init(&lg->lg_mutex);  in ext4_mb_init()
3714 			INIT_LIST_HEAD(&lg->lg_prealloc_list[j]);  in ext4_mb_init()
3715 		spin_lock_init(&lg->lg_prealloc_lock);  in ext4_mb_init()
3718 	if (bdev_nonrot(sb->s_bdev))  in ext4_mb_init()
3719 		sbi->s_mb_max_linear_groups = 0;  in ext4_mb_init()
3721 		sbi->s_mb_max_linear_groups = MB_DEFAULT_LINEAR_LIMIT;  in ext4_mb_init()
3730 	free_percpu(sbi->s_locality_groups);  in ext4_mb_init()
3731 	sbi->s_locality_groups = NULL;  in ext4_mb_init()
3733 	kfree(sbi->s_mb_avg_fragment_size);  in ext4_mb_init()
3734 	kfree(sbi->s_mb_avg_fragment_size_locks);  in ext4_mb_init()
3735 	kfree(sbi->s_mb_largest_free_orders);  in ext4_mb_init()
3736 	kfree(sbi->s_mb_largest_free_orders_locks);  in ext4_mb_init()
3737 	kfree(sbi->s_mb_offsets);  in ext4_mb_init()
3738 	sbi->s_mb_offsets = NULL;  in ext4_mb_init()
3739 	kfree(sbi->s_mb_maxs);  in ext4_mb_init()
3740 	sbi->s_mb_maxs = NULL;  in ext4_mb_init()
3751 	list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) {  in ext4_mb_cleanup_pa()
3753 		list_del(&pa->pa_group_list);  in ext4_mb_cleanup_pa()
3767 	struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);  in ext4_mb_release()
3774 		flush_work(&sbi->s_discard_work);  in ext4_mb_release()
3775 		WARN_ON_ONCE(!list_empty(&sbi->s_discard_list));  in ext4_mb_release()
3778 	if (sbi->s_group_info) {  in ext4_mb_release()
3794 				EXT4_DESC_PER_BLOCK(sb) - 1) >>  in ext4_mb_release()
3797 		group_info = rcu_dereference(sbi->s_group_info);  in ext4_mb_release()
3803 	kfree(sbi->s_mb_avg_fragment_size);  in ext4_mb_release()
3804 	kfree(sbi->s_mb_avg_fragment_size_locks);  in ext4_mb_release()
3805 	kfree(sbi->s_mb_largest_free_orders);  in ext4_mb_release()
3806 	kfree(sbi->s_mb_largest_free_orders_locks);  in ext4_mb_release()
3807 	kfree(sbi->s_mb_offsets);  in ext4_mb_release()
3808 	kfree(sbi->s_mb_maxs);  in ext4_mb_release()
3809 	iput(sbi->s_buddy_cache);  in ext4_mb_release()
3810 	if (sbi->s_mb_stats) {  in ext4_mb_release()
3813 				atomic_read(&sbi->s_bal_allocated),  in ext4_mb_release()
3814 				atomic_read(&sbi->s_bal_reqs),  in ext4_mb_release()
3815 				atomic_read(&sbi->s_bal_success));  in ext4_mb_release()
3819 				atomic_read(&sbi->s_bal_ex_scanned),  in ext4_mb_release()
3820 				atomic_read(&sbi->s_bal_groups_scanned),  in ext4_mb_release()
3821 				atomic_read(&sbi->s_bal_goals),  in ext4_mb_release()
3822 				atomic_read(&sbi->s_bal_2orders),  in ext4_mb_release()
3823 				atomic_read(&sbi->s_bal_breaks),  in ext4_mb_release()
3824 				atomic_read(&sbi->s_mb_lost_chunks));  in ext4_mb_release()
3827 				atomic_read(&sbi->s_mb_buddies_generated),  in ext4_mb_release()
3828 				atomic64_read(&sbi->s_mb_generation_time));  in ext4_mb_release()
3831 				atomic_read(&sbi->s_mb_preallocated),  in ext4_mb_release()
3832 				atomic_read(&sbi->s_mb_discarded));  in ext4_mb_release()
3835 	free_percpu(sbi->s_locality_groups);  in ext4_mb_release()
3852 		return __blkdev_issue_discard(sb->s_bdev,  in ext4_issue_discard()
3853 			(sector_t)discard_block << (sb->s_blocksize_bits - 9),  in ext4_issue_discard()
3854 			(sector_t)count << (sb->s_blocksize_bits - 9),  in ext4_issue_discard()
3868 		 entry->efd_count, entry->efd_group, entry);  in ext4_free_data_in_buddy()
3870 	err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b);  in ext4_free_data_in_buddy()
3874 	spin_lock(&EXT4_SB(sb)->s_md_lock);  in ext4_free_data_in_buddy()
3875 	EXT4_SB(sb)->s_mb_free_pending -= entry->efd_count;  in ext4_free_data_in_buddy()
3876 	spin_unlock(&EXT4_SB(sb)->s_md_lock);  in ext4_free_data_in_buddy()
3880 	count += entry->efd_count;  in ext4_free_data_in_buddy()
3881 	ext4_lock_group(sb, entry->efd_group);  in ext4_free_data_in_buddy()
3883 	rb_erase(&entry->efd_node, &(db->bb_free_root));  in ext4_free_data_in_buddy()
3884 	mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count);  in ext4_free_data_in_buddy()
3892 	if (!db->bb_free_root.rb_node) {  in ext4_free_data_in_buddy()
3899 	ext4_unlock_group(sb, entry->efd_group);  in ext4_free_data_in_buddy()
3917 	spin_lock(&sbi->s_md_lock);  in ext4_process_freed_data()
3918 	list_for_each_entry(entry, &sbi->s_freed_data_list, efd_list) {  in ext4_process_freed_data()
3919 		if (entry->efd_tid != commit_tid)  in ext4_process_freed_data()
3921 		cut_pos = &entry->efd_list;  in ext4_process_freed_data()
3924 		list_cut_position(&freed_data_list, &sbi->s_freed_data_list,  in ext4_process_freed_data()
3926 	spin_unlock(&sbi->s_md_lock);  in ext4_process_freed_data()
3932 		spin_lock(&sbi->s_md_lock);  in ext4_process_freed_data()
3933 		wake = list_empty(&sbi->s_discard_list);  in ext4_process_freed_data()
3934 		list_splice_tail(&freed_data_list, &sbi->s_discard_list);  in ext4_process_freed_data()
3935 		spin_unlock(&sbi->s_md_lock);  in ext4_process_freed_data()
3937 			queue_work(system_unbound_wq, &sbi->s_discard_work);  in ext4_process_freed_data()
3968 	return -ENOMEM;  in ext4_init_mballoc()
3986  * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
4001 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_mark_diskspace_used()
4002 	BUG_ON(ac->ac_b_ex.fe_len <= 0);  in ext4_mb_mark_diskspace_used()
4004 	sb = ac->ac_sb;  in ext4_mb_mark_diskspace_used()
4007 	bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
4018 	err = -EIO;  in ext4_mb_mark_diskspace_used()
4019 	gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh);  in ext4_mb_mark_diskspace_used()
4023 	ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group,  in ext4_mb_mark_diskspace_used()
4031 	block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_mark_diskspace_used()
4033 	len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
4034 	if (!ext4_inode_block_valid(ac->ac_inode, block, len)) {  in ext4_mb_mark_diskspace_used()
4035 		ext4_error(sb, "Allocating blocks %llu-%llu which overlap "  in ext4_mb_mark_diskspace_used()
4041 		ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
4042 		mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,  in ext4_mb_mark_diskspace_used()
4043 			      ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
4044 		ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
4047 			err = -EFSCORRUPTED;  in ext4_mb_mark_diskspace_used()
4051 	ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
4055 		for (i = 0; i < ac->ac_b_ex.fe_len; i++) {  in ext4_mb_mark_diskspace_used()
4056 			BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i,  in ext4_mb_mark_diskspace_used()
4057 						bitmap_bh->b_data));  in ext4_mb_mark_diskspace_used()
4061 	mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,  in ext4_mb_mark_diskspace_used()
4062 		      ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
4064 	    (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {  in ext4_mb_mark_diskspace_used()
4065 		gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);  in ext4_mb_mark_diskspace_used()
4068 						ac->ac_b_ex.fe_group, gdp));  in ext4_mb_mark_diskspace_used()
4070 	len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len;  in ext4_mb_mark_diskspace_used()
4073 	ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp);  in ext4_mb_mark_diskspace_used()
4075 	ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
4076 	percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
4080 	if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED))  in ext4_mb_mark_diskspace_used()
4082 		percpu_counter_sub(&sbi->s_dirtyclusters_counter,  in ext4_mb_mark_diskspace_used()
4085 	if (sbi->s_log_groups_per_flex) {  in ext4_mb_mark_diskspace_used()
4087 							  ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
4088 		atomic64_sub(ac->ac_b_ex.fe_len,  in ext4_mb_mark_diskspace_used()
4090 						  flex_group)->free_clusters);  in ext4_mb_mark_diskspace_used()
4132 			EXT4_BLOCKS_PER_GROUP(sb) - EXT4_C2B(sbi, blkoff));  in ext4_mb_mark_bb()
4136 			ext4_error(sb, "Marking blocks in system zone - "  in ext4_mb_mark_bb()
4150 		err = -EIO;  in ext4_mb_mark_bb()
4158 			if (!mb_test_bit(blkoff + i, bitmap_bh->b_data) ==  in ext4_mb_mark_bb()
4162 		clen_changed = clen - already;  in ext4_mb_mark_bb()
4164 			mb_set_bits(bitmap_bh->b_data, blkoff, clen);  in ext4_mb_mark_bb()
4166 			mb_clear_bits(bitmap_bh->b_data, blkoff, clen);  in ext4_mb_mark_bb()
4168 		    (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {  in ext4_mb_mark_bb()
4169 			gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);  in ext4_mb_mark_bb()
4174 			clen = ext4_free_group_clusters(sb, gdp) - clen_changed;  in ext4_mb_mark_bb()
4184 		if (sbi->s_log_groups_per_flex) {  in ext4_mb_mark_bb()
4190 				atomic64_sub(clen_changed, &fg->free_clusters);  in ext4_mb_mark_bb()
4192 				atomic64_add(clen_changed, &fg->free_clusters);  in ext4_mb_mark_bb()
4206 		len -= thisgrp_len;  in ext4_mb_mark_bb()
4216  * here we normalize request for locality group
4226 	struct super_block *sb = ac->ac_sb;  in ext4_mb_normalize_group_request()
4227 	struct ext4_locality_group *lg = ac->ac_lg;  in ext4_mb_normalize_group_request()
4230 	ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc;  in ext4_mb_normalize_group_request()
4231 	mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len);  in ext4_mb_normalize_group_request()
4237  * (ei->i_prealloc_lock)
4247 		return node->rb_left;  in ext4_mb_pa_rb_next_iter()
4249 		return node->rb_right;  in ext4_mb_pa_rb_next_iter()
4256 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_pa_assert_overlap()
4257 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_pa_assert_overlap()
4263 	read_lock(&ei->i_prealloc_lock);  in ext4_mb_pa_assert_overlap()
4264 	for (iter = ei->i_prealloc_node.rb_node; iter;  in ext4_mb_pa_assert_overlap()
4268 		tmp_pa_start = tmp_pa->pa_lstart;  in ext4_mb_pa_assert_overlap()
4271 		spin_lock(&tmp_pa->pa_lock);  in ext4_mb_pa_assert_overlap()
4272 		if (tmp_pa->pa_deleted == 0)  in ext4_mb_pa_assert_overlap()
4274 		spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_pa_assert_overlap()
4276 	read_unlock(&ei->i_prealloc_lock);  in ext4_mb_pa_assert_overlap()
4293 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_pa_adjust_overlap()
4294 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_pa_adjust_overlap()
4297 	ext4_lblk_t new_start, tmp_pa_start, right_pa_start = -1;  in ext4_mb_pa_adjust_overlap()
4298 	loff_t new_end, tmp_pa_end, left_pa_end = -1;  in ext4_mb_pa_adjust_overlap()
4308 	read_lock(&ei->i_prealloc_lock);  in ext4_mb_pa_adjust_overlap()
4311 	for (iter = ei->i_prealloc_node.rb_node; iter;  in ext4_mb_pa_adjust_overlap()
4312 	     iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical,  in ext4_mb_pa_adjust_overlap()
4316 		tmp_pa_start = tmp_pa->pa_lstart;  in ext4_mb_pa_adjust_overlap()
4320 		spin_lock(&tmp_pa->pa_lock);  in ext4_mb_pa_adjust_overlap()
4321 		if (tmp_pa->pa_deleted == 0)  in ext4_mb_pa_adjust_overlap()
4322 			BUG_ON(!(ac->ac_o_ex.fe_logical >= tmp_pa_end ||  in ext4_mb_pa_adjust_overlap()
4323 				 ac->ac_o_ex.fe_logical < tmp_pa_start));  in ext4_mb_pa_adjust_overlap()
4324 		spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_pa_adjust_overlap()
4332 		if (tmp_pa->pa_lstart < ac->ac_o_ex.fe_logical) {  in ext4_mb_pa_adjust_overlap()
4336 			tmp = rb_next(&left_pa->pa_node.inode_node);  in ext4_mb_pa_adjust_overlap()
4346 			tmp = rb_prev(&right_pa->pa_node.inode_node);  in ext4_mb_pa_adjust_overlap()
4357 		for (iter = &left_pa->pa_node.inode_node;;  in ext4_mb_pa_adjust_overlap()
4367 			spin_lock(&tmp_pa->pa_lock);  in ext4_mb_pa_adjust_overlap()
4368 			if (tmp_pa->pa_deleted == 0) {  in ext4_mb_pa_adjust_overlap()
4369 				spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_pa_adjust_overlap()
4372 			spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_pa_adjust_overlap()
4377 		for (iter = &right_pa->pa_node.inode_node;;  in ext4_mb_pa_adjust_overlap()
4387 			spin_lock(&tmp_pa->pa_lock);  in ext4_mb_pa_adjust_overlap()
4388 			if (tmp_pa->pa_deleted == 0) {  in ext4_mb_pa_adjust_overlap()
4389 				spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_pa_adjust_overlap()
4392 			spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_pa_adjust_overlap()
4398 		BUG_ON(left_pa_end > ac->ac_o_ex.fe_logical);  in ext4_mb_pa_adjust_overlap()
4402 		right_pa_start = right_pa->pa_lstart;  in ext4_mb_pa_adjust_overlap()
4403 		BUG_ON(right_pa_start <= ac->ac_o_ex.fe_logical);  in ext4_mb_pa_adjust_overlap()
4416 	read_unlock(&ei->i_prealloc_lock);  in ext4_mb_pa_adjust_overlap()
4433 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_normalize_request()
4434 	struct ext4_super_block *es = sbi->s_es;  in ext4_mb_normalize_request()
4442 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_normalize_request()
4446 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_normalize_request()
4451 	if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC)  in ext4_mb_normalize_request()
4454 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) {  in ext4_mb_normalize_request()
4459 	bsbits = ac->ac_sb->s_blocksize_bits;  in ext4_mb_normalize_request()
4463 	size = extent_logical_end(sbi, &ac->ac_o_ex);  in ext4_mb_normalize_request()
4465 	if (size < i_size_read(ac->ac_inode))  in ext4_mb_normalize_request()
4466 		size = i_size_read(ac->ac_inode);  in ext4_mb_normalize_request()
4493 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
4494 						(21 - bsbits)) << 21;  in ext4_mb_normalize_request()
4497 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
4498 							(22 - bsbits)) << 22;  in ext4_mb_normalize_request()
4500 	} else if (NRL_CHECK_SIZE(EXT4_C2B(sbi, ac->ac_o_ex.fe_len),  in ext4_mb_normalize_request()
4502 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
4503 							(23 - bsbits)) << 23;  in ext4_mb_normalize_request()
4506 		start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits;  in ext4_mb_normalize_request()
4508 					      ac->ac_o_ex.fe_len) << bsbits;  in ext4_mb_normalize_request()
4519 	start = max(start, rounddown(ac->ac_o_ex.fe_logical,  in ext4_mb_normalize_request()
4520 			(ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb)));  in ext4_mb_normalize_request()
4524 		size = EXT_MAX_BLOCKS - start;  in ext4_mb_normalize_request()
4527 	if (ar->pleft && start <= ar->lleft) {  in ext4_mb_normalize_request()
4528 		size -= ar->lleft + 1 - start;  in ext4_mb_normalize_request()
4529 		start = ar->lleft + 1;  in ext4_mb_normalize_request()
4531 	if (ar->pright && start + size - 1 >= ar->lright)  in ext4_mb_normalize_request()
4532 		size -= start + size - ar->lright;  in ext4_mb_normalize_request()
4538 	if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb))  in ext4_mb_normalize_request()
4539 		size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb);  in ext4_mb_normalize_request()
4545 	size = end - start;  in ext4_mb_normalize_request()
4551 	 * ac->ac_o_ex.fe_logical & fe_len should always lie within "start" and  in ext4_mb_normalize_request()
4557 	 * [pa_pstart + fe_logical - pa_lstart, fe_len/size] from the preallocated  in ext4_mb_normalize_request()
4562 	if (start + size <= ac->ac_o_ex.fe_logical ||  in ext4_mb_normalize_request()
4563 			start > ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
4564 		ext4_msg(ac->ac_sb, KERN_ERR,  in ext4_mb_normalize_request()
4567 			 (unsigned long) ac->ac_o_ex.fe_logical);  in ext4_mb_normalize_request()
4570 	BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb));  in ext4_mb_normalize_request()
4576 	ac->ac_g_ex.fe_logical = start;  in ext4_mb_normalize_request()
4577 	ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size);  in ext4_mb_normalize_request()
4578 	ac->ac_orig_goal_len = ac->ac_g_ex.fe_len;  in ext4_mb_normalize_request()
4581 	if (ar->pright && (ar->lright == (start + size)) &&  in ext4_mb_normalize_request()
4582 	    ar->pright >= size &&  in ext4_mb_normalize_request()
4583 	    ar->pright - size >= le32_to_cpu(es->s_first_data_block)) {  in ext4_mb_normalize_request()
4585 		ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size,  in ext4_mb_normalize_request()
4586 						&ac->ac_g_ex.fe_group,  in ext4_mb_normalize_request()
4587 						&ac->ac_g_ex.fe_start);  in ext4_mb_normalize_request()
4588 		ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;  in ext4_mb_normalize_request()
4590 	if (ar->pleft && (ar->lleft + 1 == start) &&  in ext4_mb_normalize_request()
4591 	    ar->pleft + 1 < ext4_blocks_count(es)) {  in ext4_mb_normalize_request()
4593 		ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1,  in ext4_mb_normalize_request()
4594 						&ac->ac_g_ex.fe_group,  in ext4_mb_normalize_request()
4595 						&ac->ac_g_ex.fe_start);  in ext4_mb_normalize_request()
4596 		ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;  in ext4_mb_normalize_request()
4599 	mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size,  in ext4_mb_normalize_request()
4605 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_collect_stats()
4607 	if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) {  in ext4_mb_collect_stats()
4608 		atomic_inc(&sbi->s_bal_reqs);  in ext4_mb_collect_stats()
4609 		atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);  in ext4_mb_collect_stats()
4610 		if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len)  in ext4_mb_collect_stats()
4611 			atomic_inc(&sbi->s_bal_success);  in ext4_mb_collect_stats()
4613 		atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);  in ext4_mb_collect_stats()
4615 			atomic_add(ac->ac_cX_found[i], &sbi->s_bal_cX_ex_scanned[i]);  in ext4_mb_collect_stats()
4618 		atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned);  in ext4_mb_collect_stats()
4619 		if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&  in ext4_mb_collect_stats()
4620 				ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)  in ext4_mb_collect_stats()
4621 			atomic_inc(&sbi->s_bal_goals);  in ext4_mb_collect_stats()
4623 		if (ac->ac_f_ex.fe_len == ac->ac_orig_goal_len)  in ext4_mb_collect_stats()
4624 			atomic_inc(&sbi->s_bal_len_goals);  in ext4_mb_collect_stats()
4626 		if (ac->ac_found > sbi->s_mb_max_to_scan)  in ext4_mb_collect_stats()
4627 			atomic_inc(&sbi->s_bal_breaks);  in ext4_mb_collect_stats()
4630 	if (ac->ac_op == EXT4_MB_HISTORY_ALLOC)  in ext4_mb_collect_stats()
4640  * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
4644 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_discard_allocated_blocks()
4649 		if (ac->ac_f_ex.fe_len == 0)  in ext4_discard_allocated_blocks()
4651 		err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b);  in ext4_discard_allocated_blocks()
4660 		ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group);  in ext4_discard_allocated_blocks()
4661 		mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start,  in ext4_discard_allocated_blocks()
4662 			       ac->ac_f_ex.fe_len);  in ext4_discard_allocated_blocks()
4663 		ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group);  in ext4_discard_allocated_blocks()
4667 	if (pa->pa_type == MB_INODE_PA) {  in ext4_discard_allocated_blocks()
4668 		spin_lock(&pa->pa_lock);  in ext4_discard_allocated_blocks()
4669 		pa->pa_free += ac->ac_b_ex.fe_len;  in ext4_discard_allocated_blocks()
4670 		spin_unlock(&pa->pa_lock);  in ext4_discard_allocated_blocks()
4680 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_inode_pa()
4686 	start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart);  in ext4_mb_use_inode_pa()
4687 	end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len),  in ext4_mb_use_inode_pa()
4688 		  start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len));  in ext4_mb_use_inode_pa()
4689 	len = EXT4_NUM_B2C(sbi, end - start);  in ext4_mb_use_inode_pa()
4690 	ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group,  in ext4_mb_use_inode_pa()
4691 					&ac->ac_b_ex.fe_start);  in ext4_mb_use_inode_pa()
4692 	ac->ac_b_ex.fe_len = len;  in ext4_mb_use_inode_pa()
4693 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_inode_pa()
4694 	ac->ac_pa = pa;  in ext4_mb_use_inode_pa()
4696 	BUG_ON(start < pa->pa_pstart);  in ext4_mb_use_inode_pa()
4697 	BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len));  in ext4_mb_use_inode_pa()
4698 	BUG_ON(pa->pa_free < len);  in ext4_mb_use_inode_pa()
4699 	BUG_ON(ac->ac_b_ex.fe_len <= 0);  in ext4_mb_use_inode_pa()
4700 	pa->pa_free -= len;  in ext4_mb_use_inode_pa()
4702 	mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa);  in ext4_mb_use_inode_pa()
4706  * use blocks preallocated to locality group
4711 	unsigned int len = ac->ac_o_ex.fe_len;  in ext4_mb_use_group_pa()
4713 	ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart,  in ext4_mb_use_group_pa()
4714 					&ac->ac_b_ex.fe_group,  in ext4_mb_use_group_pa()
4715 					&ac->ac_b_ex.fe_start);  in ext4_mb_use_group_pa()
4716 	ac->ac_b_ex.fe_len = len;  in ext4_mb_use_group_pa()
4717 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_group_pa()
4718 	ac->ac_pa = pa;  in ext4_mb_use_group_pa()
4723 	 * in on-disk bitmap -- see ext4_mb_release_context()  in ext4_mb_use_group_pa()
4726 	mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n",  in ext4_mb_use_group_pa()
4727 		 pa->pa_lstart, len, pa);  in ext4_mb_use_group_pa()
4744 		atomic_inc(&pa->pa_count);  in ext4_mb_check_group_pa()
4747 	cur_distance = abs(goal_block - cpa->pa_pstart);  in ext4_mb_check_group_pa()
4748 	new_distance = abs(goal_block - pa->pa_pstart);  in ext4_mb_check_group_pa()
4754 	atomic_dec(&cpa->pa_count);  in ext4_mb_check_group_pa()
4755 	atomic_inc(&pa->pa_count);  in ext4_mb_check_group_pa()
4766 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_pa_goal_check()
4769 	if (likely(!(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)))  in ext4_mb_pa_goal_check()
4778 	start = pa->pa_pstart +  in ext4_mb_pa_goal_check()
4779 		(ac->ac_g_ex.fe_logical - pa->pa_lstart);  in ext4_mb_pa_goal_check()
4780 	if (ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex) != start)  in ext4_mb_pa_goal_check()
4783 	if (ac->ac_g_ex.fe_len > pa->pa_len -  in ext4_mb_pa_goal_check()
4784 	    EXT4_B2C(sbi, ac->ac_g_ex.fe_logical - pa->pa_lstart))  in ext4_mb_pa_goal_check()
4796 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_preallocated()
4798 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_use_preallocated()
4805 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_use_preallocated()
4809 	 * first, try per-file preallocation by searching the inode pa rbtree.  in ext4_mb_use_preallocated()
4815 	read_lock(&ei->i_prealloc_lock);  in ext4_mb_use_preallocated()
4817 	if (RB_EMPTY_ROOT(&ei->i_prealloc_node)) {  in ext4_mb_use_preallocated()
4825 	 * (tmp_pa->pa_lstart never changes so we can skip locking for it).  in ext4_mb_use_preallocated()
4827 	for (iter = ei->i_prealloc_node.rb_node; iter;  in ext4_mb_use_preallocated()
4828 	     iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical,  in ext4_mb_use_preallocated()
4829 					    tmp_pa->pa_lstart, iter)) {  in ext4_mb_use_preallocated()
4839 	if (tmp_pa->pa_lstart > ac->ac_o_ex.fe_logical) {  in ext4_mb_use_preallocated()
4841 		tmp = rb_prev(&tmp_pa->pa_node.inode_node);  in ext4_mb_use_preallocated()
4856 	BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical));  in ext4_mb_use_preallocated()
4863 	for (iter = &tmp_pa->pa_node.inode_node;; iter = rb_prev(iter)) {  in ext4_mb_use_preallocated()
4873 		spin_lock(&tmp_pa->pa_lock);  in ext4_mb_use_preallocated()
4874 		if (tmp_pa->pa_deleted == 0) {  in ext4_mb_use_preallocated()
4884 			spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_use_preallocated()
4888 	BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical));  in ext4_mb_use_preallocated()
4889 	BUG_ON(tmp_pa->pa_deleted == 1);  in ext4_mb_use_preallocated()
4896 	if (ac->ac_o_ex.fe_logical >= pa_logical_end(sbi, tmp_pa)) {  in ext4_mb_use_preallocated()
4897 		spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_use_preallocated()
4901 	/* non-extent files can't have physical blocks past 2^32 */  in ext4_mb_use_preallocated()
4902 	if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) &&  in ext4_mb_use_preallocated()
4903 	    (tmp_pa->pa_pstart + EXT4_C2B(sbi, tmp_pa->pa_len) >  in ext4_mb_use_preallocated()
4909 		spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_use_preallocated()
4913 	if (tmp_pa->pa_free && likely(ext4_mb_pa_goal_check(ac, tmp_pa))) {  in ext4_mb_use_preallocated()
4914 		atomic_inc(&tmp_pa->pa_count);  in ext4_mb_use_preallocated()
4916 		spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_use_preallocated()
4917 		read_unlock(&ei->i_prealloc_lock);  in ext4_mb_use_preallocated()
4947 		WARN_ON_ONCE(tmp_pa->pa_free == 0);  in ext4_mb_use_preallocated()
4949 	spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_use_preallocated()
4951 	read_unlock(&ei->i_prealloc_lock);  in ext4_mb_use_preallocated()
4954 	if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC))  in ext4_mb_use_preallocated()
4957 	/* inode may have no locality group for some reason */  in ext4_mb_use_preallocated()
4958 	lg = ac->ac_lg;  in ext4_mb_use_preallocated()
4961 	order  = fls(ac->ac_o_ex.fe_len) - 1;  in ext4_mb_use_preallocated()
4962 	if (order > PREALLOC_TB_SIZE - 1)  in ext4_mb_use_preallocated()
4964 		order = PREALLOC_TB_SIZE - 1;  in ext4_mb_use_preallocated()
4966 	goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex);  in ext4_mb_use_preallocated()
4973 		list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[i],  in ext4_mb_use_preallocated()
4975 			spin_lock(&tmp_pa->pa_lock);  in ext4_mb_use_preallocated()
4976 			if (tmp_pa->pa_deleted == 0 &&  in ext4_mb_use_preallocated()
4977 					tmp_pa->pa_free >= ac->ac_o_ex.fe_len) {  in ext4_mb_use_preallocated()
4982 			spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_use_preallocated()
4995  * used in in-core bitmap. buddy must be generated from this bitmap
5021 	list_for_each(cur, &grp->bb_prealloc_list) {  in ext4_mb_generate_from_pa()
5023 		spin_lock(&pa->pa_lock);  in ext4_mb_generate_from_pa()
5024 		ext4_get_group_no_and_offset(sb, pa->pa_pstart,  in ext4_mb_generate_from_pa()
5026 		len = pa->pa_len;  in ext4_mb_generate_from_pa()
5027 		spin_unlock(&pa->pa_lock);  in ext4_mb_generate_from_pa()
5042 	if (pa->pa_deleted) {  in ext4_mb_mark_pa_deleted()
5044 			     pa->pa_type, pa->pa_pstart, pa->pa_lstart,  in ext4_mb_mark_pa_deleted()
5045 			     pa->pa_len);  in ext4_mb_mark_pa_deleted()
5049 	pa->pa_deleted = 1;  in ext4_mb_mark_pa_deleted()
5051 	if (pa->pa_type == MB_INODE_PA) {  in ext4_mb_mark_pa_deleted()
5052 		ei = EXT4_I(pa->pa_inode);  in ext4_mb_mark_pa_deleted()
5053 		atomic_dec(&ei->i_prealloc_active);  in ext4_mb_mark_pa_deleted()
5060 	BUG_ON(atomic_read(&pa->pa_count));  in ext4_mb_pa_free()
5061 	BUG_ON(pa->pa_deleted == 0);  in ext4_mb_pa_free()
5082 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_put_pa()
5085 	spin_lock(&pa->pa_lock);  in ext4_mb_put_pa()
5086 	if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) {  in ext4_mb_put_pa()
5087 		spin_unlock(&pa->pa_lock);  in ext4_mb_put_pa()
5091 	if (pa->pa_deleted == 1) {  in ext4_mb_put_pa()
5092 		spin_unlock(&pa->pa_lock);  in ext4_mb_put_pa()
5097 	spin_unlock(&pa->pa_lock);  in ext4_mb_put_pa()
5099 	grp_blk = pa->pa_pstart;  in ext4_mb_put_pa()
5101 	 * If doing group-based preallocation, pa_pstart may be in the  in ext4_mb_put_pa()
5104 	if (pa->pa_type == MB_GROUP_PA)  in ext4_mb_put_pa()
5105 		grp_blk--;  in ext4_mb_put_pa()
5114 	 *  copy on-disk bitmap to buddy  in ext4_mb_put_pa()
5115 	 *  					mark B in on-disk bitmap  in ext4_mb_put_pa()
5124 	list_del(&pa->pa_group_list);  in ext4_mb_put_pa()
5127 	if (pa->pa_type == MB_INODE_PA) {  in ext4_mb_put_pa()
5128 		write_lock(pa->pa_node_lock.inode_lock);  in ext4_mb_put_pa()
5129 		rb_erase(&pa->pa_node.inode_node, &ei->i_prealloc_node);  in ext4_mb_put_pa()
5130 		write_unlock(pa->pa_node_lock.inode_lock);  in ext4_mb_put_pa()
5133 		spin_lock(pa->pa_node_lock.lg_lock);  in ext4_mb_put_pa()
5134 		list_del_rcu(&pa->pa_node.lg_list);  in ext4_mb_put_pa()
5135 		spin_unlock(pa->pa_node_lock.lg_lock);  in ext4_mb_put_pa()
5136 		call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_mb_put_pa()
5142 	struct rb_node **iter = &root->rb_node, *parent = NULL;  in ext4_mb_pa_rb_insert()
5151 		iter_start = iter_pa->pa_lstart;  in ext4_mb_pa_rb_insert()
5152 		new_start = new_pa->pa_lstart;  in ext4_mb_pa_rb_insert()
5156 			iter = &((*iter)->rb_left);  in ext4_mb_pa_rb_insert()
5158 			iter = &((*iter)->rb_right);  in ext4_mb_pa_rb_insert()
5171 	struct super_block *sb = ac->ac_sb;  in ext4_mb_new_inode_pa()
5178 	BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
5179 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_new_inode_pa()
5180 	BUG_ON(!S_ISREG(ac->ac_inode->i_mode));  in ext4_mb_new_inode_pa()
5181 	BUG_ON(ac->ac_pa == NULL);  in ext4_mb_new_inode_pa()
5183 	pa = ac->ac_pa;  in ext4_mb_new_inode_pa()
5185 	if (ac->ac_b_ex.fe_len < ac->ac_orig_goal_len) {  in ext4_mb_new_inode_pa()
5187 			.fe_logical = ac->ac_g_ex.fe_logical,  in ext4_mb_new_inode_pa()
5188 			.fe_len = ac->ac_orig_goal_len,  in ext4_mb_new_inode_pa()
5191 		loff_t o_ex_end = extent_logical_end(sbi, &ac->ac_o_ex);  in ext4_mb_new_inode_pa()
5198 		 * orig_ex:2045/2055(10), isize:8417280 -> normalized:0/2048  in ext4_mb_new_inode_pa()
5199 		 * best_ex:0/200(200) -> adjusted: 1848/2048(200)  in ext4_mb_new_inode_pa()
5201 		BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical);  in ext4_mb_new_inode_pa()
5202 		BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len);  in ext4_mb_new_inode_pa()
5215 		ex.fe_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_inode_pa()
5217 		ex.fe_logical = orig_goal_end - EXT4_C2B(sbi, ex.fe_len);  in ext4_mb_new_inode_pa()
5218 		if (ac->ac_o_ex.fe_logical >= ex.fe_logical)  in ext4_mb_new_inode_pa()
5221 		ex.fe_logical = ac->ac_g_ex.fe_logical;  in ext4_mb_new_inode_pa()
5225 		ex.fe_logical = ac->ac_o_ex.fe_logical;  in ext4_mb_new_inode_pa()
5227 		ac->ac_b_ex.fe_logical = ex.fe_logical;  in ext4_mb_new_inode_pa()
5229 		BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical);  in ext4_mb_new_inode_pa()
5233 	pa->pa_lstart = ac->ac_b_ex.fe_logical;  in ext4_mb_new_inode_pa()
5234 	pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_inode_pa()
5235 	pa->pa_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_inode_pa()
5236 	pa->pa_free = pa->pa_len;  in ext4_mb_new_inode_pa()
5237 	spin_lock_init(&pa->pa_lock);  in ext4_mb_new_inode_pa()
5238 	INIT_LIST_HEAD(&pa->pa_group_list);  in ext4_mb_new_inode_pa()
5239 	pa->pa_deleted = 0;  in ext4_mb_new_inode_pa()
5240 	pa->pa_type = MB_INODE_PA;  in ext4_mb_new_inode_pa()
5242 	mb_debug(sb, "new inode pa %p: %llu/%d for %u\n", pa, pa->pa_pstart,  in ext4_mb_new_inode_pa()
5243 		 pa->pa_len, pa->pa_lstart);  in ext4_mb_new_inode_pa()
5246 	atomic_add(pa->pa_free, &sbi->s_mb_preallocated);  in ext4_mb_new_inode_pa()
5249 	ei = EXT4_I(ac->ac_inode);  in ext4_mb_new_inode_pa()
5250 	grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_inode_pa()
5254 	pa->pa_node_lock.inode_lock = &ei->i_prealloc_lock;  in ext4_mb_new_inode_pa()
5255 	pa->pa_inode = ac->ac_inode;  in ext4_mb_new_inode_pa()
5257 	list_add(&pa->pa_group_list, &grp->bb_prealloc_list);  in ext4_mb_new_inode_pa()
5259 	write_lock(pa->pa_node_lock.inode_lock);  in ext4_mb_new_inode_pa()
5260 	ext4_mb_pa_rb_insert(&ei->i_prealloc_node, &pa->pa_node.inode_node);  in ext4_mb_new_inode_pa()
5261 	write_unlock(pa->pa_node_lock.inode_lock);  in ext4_mb_new_inode_pa()
5262 	atomic_inc(&ei->i_prealloc_active);  in ext4_mb_new_inode_pa()
5266  * creates new preallocated space for locality group inodes belongs to
5271 	struct super_block *sb = ac->ac_sb;  in ext4_mb_new_group_pa()
5277 	BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);  in ext4_mb_new_group_pa()
5278 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_new_group_pa()
5279 	BUG_ON(!S_ISREG(ac->ac_inode->i_mode));  in ext4_mb_new_group_pa()
5280 	BUG_ON(ac->ac_pa == NULL);  in ext4_mb_new_group_pa()
5282 	pa = ac->ac_pa;  in ext4_mb_new_group_pa()
5284 	pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_group_pa()
5285 	pa->pa_lstart = pa->pa_pstart;  in ext4_mb_new_group_pa()
5286 	pa->pa_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_group_pa()
5287 	pa->pa_free = pa->pa_len;  in ext4_mb_new_group_pa()
5288 	spin_lock_init(&pa->pa_lock);  in ext4_mb_new_group_pa()
5289 	INIT_LIST_HEAD(&pa->pa_node.lg_list);  in ext4_mb_new_group_pa()
5290 	INIT_LIST_HEAD(&pa->pa_group_list);  in ext4_mb_new_group_pa()
5291 	pa->pa_deleted = 0;  in ext4_mb_new_group_pa()
5292 	pa->pa_type = MB_GROUP_PA;  in ext4_mb_new_group_pa()
5294 	mb_debug(sb, "new group pa %p: %llu/%d for %u\n", pa, pa->pa_pstart,  in ext4_mb_new_group_pa()
5295 		 pa->pa_len, pa->pa_lstart);  in ext4_mb_new_group_pa()
5299 	atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);  in ext4_mb_new_group_pa()
5301 	grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_group_pa()
5304 	lg = ac->ac_lg;  in ext4_mb_new_group_pa()
5307 	pa->pa_node_lock.lg_lock = &lg->lg_prealloc_lock;  in ext4_mb_new_group_pa()
5308 	pa->pa_inode = NULL;  in ext4_mb_new_group_pa()
5310 	list_add(&pa->pa_group_list, &grp->bb_prealloc_list);  in ext4_mb_new_group_pa()
5320 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)  in ext4_mb_new_preallocation()
5327  * finds all unused blocks in on-disk bitmap, frees them in
5328  * in-core bitmap and buddy.
5332  * TODO: optimize the case when there are no in-core structures yet
5338 	struct super_block *sb = e4b->bd_sb;  in ext4_mb_release_inode_pa()
5347 	BUG_ON(pa->pa_deleted == 0);  in ext4_mb_release_inode_pa()
5348 	ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);  in ext4_mb_release_inode_pa()
5349 	grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit);  in ext4_mb_release_inode_pa()
5350 	BUG_ON(group != e4b->bd_group && pa->pa_len != 0);  in ext4_mb_release_inode_pa()
5351 	end = bit + pa->pa_len;  in ext4_mb_release_inode_pa()
5354 		bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit);  in ext4_mb_release_inode_pa()
5357 		next = mb_find_next_bit(bitmap_bh->b_data, end, bit);  in ext4_mb_release_inode_pa()
5360 			 (unsigned) next - bit, (unsigned) group);  in ext4_mb_release_inode_pa()
5361 		free += next - bit;  in ext4_mb_release_inode_pa()
5363 		trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit);  in ext4_mb_release_inode_pa()
5366 					       next - bit);  in ext4_mb_release_inode_pa()
5367 		mb_free_blocks(pa->pa_inode, e4b, bit, next - bit);  in ext4_mb_release_inode_pa()
5370 	if (free != pa->pa_free) {  in ext4_mb_release_inode_pa()
5371 		ext4_msg(e4b->bd_sb, KERN_CRIT,  in ext4_mb_release_inode_pa()
5373 			 pa, (unsigned long) pa->pa_lstart,  in ext4_mb_release_inode_pa()
5374 			 (unsigned long) pa->pa_pstart,  in ext4_mb_release_inode_pa()
5375 			 pa->pa_len);  in ext4_mb_release_inode_pa()
5377 					free, pa->pa_free);  in ext4_mb_release_inode_pa()
5383 	atomic_add(free, &sbi->s_mb_discarded);  in ext4_mb_release_inode_pa()
5392 	struct super_block *sb = e4b->bd_sb;  in ext4_mb_release_group_pa()
5397 	BUG_ON(pa->pa_deleted == 0);  in ext4_mb_release_group_pa()
5398 	ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);  in ext4_mb_release_group_pa()
5399 	if (unlikely(group != e4b->bd_group && pa->pa_len != 0)) {  in ext4_mb_release_group_pa()
5401 			     e4b->bd_group, group, pa->pa_pstart);  in ext4_mb_release_group_pa()
5404 	mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len);  in ext4_mb_release_group_pa()
5405 	atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded);  in ext4_mb_release_group_pa()
5406 	trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len);  in ext4_mb_release_group_pa()
5415  * - when do we discard
5417  * - how many do we discard
5436 	if (list_empty(&grp->bb_prealloc_list))  in ext4_mb_discard_group_preallocations()
5442 		ext4_error_err(sb, -err,  in ext4_mb_discard_group_preallocations()
5458 				&grp->bb_prealloc_list, pa_group_list) {  in ext4_mb_discard_group_preallocations()
5459 		spin_lock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
5460 		if (atomic_read(&pa->pa_count)) {  in ext4_mb_discard_group_preallocations()
5461 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
5465 		if (pa->pa_deleted) {  in ext4_mb_discard_group_preallocations()
5466 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
5477 		free += pa->pa_free;  in ext4_mb_discard_group_preallocations()
5479 		spin_unlock(&pa->pa_lock);  in ext4_mb_discard_group_preallocations()
5481 		list_del(&pa->pa_group_list);  in ext4_mb_discard_group_preallocations()
5482 		list_add(&pa->u.pa_tmp_list, &list);  in ext4_mb_discard_group_preallocations()
5488 		/* remove from object (inode or locality group) */  in ext4_mb_discard_group_preallocations()
5489 		if (pa->pa_type == MB_GROUP_PA) {  in ext4_mb_discard_group_preallocations()
5490 			spin_lock(pa->pa_node_lock.lg_lock);  in ext4_mb_discard_group_preallocations()
5491 			list_del_rcu(&pa->pa_node.lg_list);  in ext4_mb_discard_group_preallocations()
5492 			spin_unlock(pa->pa_node_lock.lg_lock);  in ext4_mb_discard_group_preallocations()
5494 			write_lock(pa->pa_node_lock.inode_lock);  in ext4_mb_discard_group_preallocations()
5495 			ei = EXT4_I(pa->pa_inode);  in ext4_mb_discard_group_preallocations()
5496 			rb_erase(&pa->pa_node.inode_node, &ei->i_prealloc_node);  in ext4_mb_discard_group_preallocations()
5497 			write_unlock(pa->pa_node_lock.inode_lock);  in ext4_mb_discard_group_preallocations()
5500 		list_del(&pa->u.pa_tmp_list);  in ext4_mb_discard_group_preallocations()
5502 		if (pa->pa_type == MB_GROUP_PA) {  in ext4_mb_discard_group_preallocations()
5504 			call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_mb_discard_group_preallocations()
5516 		 free, group, grp->bb_free);  in ext4_mb_discard_group_preallocations()
5521  * releases all non-used preallocated blocks for given inode
5532 	struct super_block *sb = inode->i_sb;  in ext4_discard_preallocations()
5541 	if (!S_ISREG(inode->i_mode)) {  in ext4_discard_preallocations()
5545 	if (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)  in ext4_discard_preallocations()
5549 		 inode->i_ino);  in ext4_discard_preallocations()
5551 			atomic_read(&ei->i_prealloc_active), needed);  in ext4_discard_preallocations()
5558 	write_lock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
5559 	for (iter = rb_first(&ei->i_prealloc_node); iter && needed;  in ext4_discard_preallocations()
5563 		BUG_ON(pa->pa_node_lock.inode_lock != &ei->i_prealloc_lock);  in ext4_discard_preallocations()
5565 		spin_lock(&pa->pa_lock);  in ext4_discard_preallocations()
5566 		if (atomic_read(&pa->pa_count)) {  in ext4_discard_preallocations()
5567 			/* this shouldn't happen often - nobody should  in ext4_discard_preallocations()
5569 			spin_unlock(&pa->pa_lock);  in ext4_discard_preallocations()
5570 			write_unlock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
5572 				 "uh-oh! used pa while discarding");  in ext4_discard_preallocations()
5578 		if (pa->pa_deleted == 0) {  in ext4_discard_preallocations()
5580 			spin_unlock(&pa->pa_lock);  in ext4_discard_preallocations()
5581 			rb_erase(&pa->pa_node.inode_node, &ei->i_prealloc_node);  in ext4_discard_preallocations()
5582 			list_add(&pa->u.pa_tmp_list, &list);  in ext4_discard_preallocations()
5583 			needed--;  in ext4_discard_preallocations()
5588 		spin_unlock(&pa->pa_lock);  in ext4_discard_preallocations()
5589 		write_unlock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
5594 		 * ->clear_inode() the inode will get freed  in ext4_discard_preallocations()
5597 		 * freed memory, bad-bad-bad */  in ext4_discard_preallocations()
5601 		 * of ->clear_inode(), but not in case of  in ext4_discard_preallocations()
5606 	write_unlock(&ei->i_prealloc_lock);  in ext4_discard_preallocations()
5609 		BUG_ON(pa->pa_type != MB_INODE_PA);  in ext4_discard_preallocations()
5610 		group = ext4_get_group_number(sb, pa->pa_pstart);  in ext4_discard_preallocations()
5615 			ext4_error_err(sb, -err, "Error %d loading buddy information for %u",  in ext4_discard_preallocations()
5623 			ext4_error_err(sb, -err, "Error %d reading block bitmap for %u",  in ext4_discard_preallocations()
5630 		list_del(&pa->pa_group_list);  in ext4_discard_preallocations()
5637 		list_del(&pa->u.pa_tmp_list);  in ext4_discard_preallocations()
5649 		return -ENOMEM;  in ext4_mb_pa_alloc()
5650 	atomic_set(&pa->pa_count, 1);  in ext4_mb_pa_alloc()
5651 	ac->ac_pa = pa;  in ext4_mb_pa_alloc()
5657 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_mb_pa_put_free()
5660 	ac->ac_pa = NULL;  in ext4_mb_pa_put_free()
5661 	WARN_ON(!atomic_dec_and_test(&pa->pa_count));  in ext4_mb_pa_put_free()
5665 	 * been added to grp->bb_prealloc_list. So we don't need to lock it  in ext4_mb_pa_put_free()
5667 	pa->pa_deleted = 1;  in ext4_mb_pa_put_free()
5690 		list_for_each(cur, &grp->bb_prealloc_list) {  in ext4_mb_show_pa()
5693 			spin_lock(&pa->pa_lock);  in ext4_mb_show_pa()
5694 			ext4_get_group_no_and_offset(sb, pa->pa_pstart,  in ext4_mb_show_pa()
5696 			spin_unlock(&pa->pa_lock);  in ext4_mb_show_pa()
5698 				 pa->pa_len);  in ext4_mb_show_pa()
5701 		mb_debug(sb, "%u: %d/%d\n", i, grp->bb_free,  in ext4_mb_show_pa()
5702 			 grp->bb_fragments);  in ext4_mb_show_pa()
5708 	struct super_block *sb = ac->ac_sb;  in ext4_mb_show_ac()
5716 			ac->ac_status, ac->ac_flags);  in ext4_mb_show_ac()
5720 			(unsigned long)ac->ac_o_ex.fe_group,  in ext4_mb_show_ac()
5721 			(unsigned long)ac->ac_o_ex.fe_start,  in ext4_mb_show_ac()
5722 			(unsigned long)ac->ac_o_ex.fe_len,  in ext4_mb_show_ac()
5723 			(unsigned long)ac->ac_o_ex.fe_logical,  in ext4_mb_show_ac()
5724 			(unsigned long)ac->ac_g_ex.fe_group,  in ext4_mb_show_ac()
5725 			(unsigned long)ac->ac_g_ex.fe_start,  in ext4_mb_show_ac()
5726 			(unsigned long)ac->ac_g_ex.fe_len,  in ext4_mb_show_ac()
5727 			(unsigned long)ac->ac_g_ex.fe_logical,  in ext4_mb_show_ac()
5728 			(unsigned long)ac->ac_b_ex.fe_group,  in ext4_mb_show_ac()
5729 			(unsigned long)ac->ac_b_ex.fe_start,  in ext4_mb_show_ac()
5730 			(unsigned long)ac->ac_b_ex.fe_len,  in ext4_mb_show_ac()
5731 			(unsigned long)ac->ac_b_ex.fe_logical,  in ext4_mb_show_ac()
5732 			(int)ac->ac_criteria);  in ext4_mb_show_ac()
5733 	mb_debug(sb, "%u found", ac->ac_found);  in ext4_mb_show_ac()
5734 	mb_debug(sb, "used pa: %s, ", ac->ac_pa ? "yes" : "no");  in ext4_mb_show_ac()
5735 	if (ac->ac_pa)  in ext4_mb_show_ac()
5736 		mb_debug(sb, "pa_type %s\n", ac->ac_pa->pa_type == MB_GROUP_PA ?  in ext4_mb_show_ac()
5746 	ext4_mb_show_pa(ac->ac_sb);  in ext4_mb_show_ac()
5751  * We use locality group preallocation for small size file. The size of the
5759 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_group_or_file()
5760 	int bsbits = ac->ac_sb->s_blocksize_bits;  in ext4_mb_group_or_file()
5764 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_group_or_file()
5767 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_group_or_file()
5770 	group_pa_eligible = sbi->s_mb_group_prealloc > 0;  in ext4_mb_group_or_file()
5772 	size = extent_logical_end(sbi, &ac->ac_o_ex);  in ext4_mb_group_or_file()
5773 	isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1)  in ext4_mb_group_or_file()
5778 	    !inode_is_open_for_write(ac->ac_inode))  in ext4_mb_group_or_file()
5783 	if (size > sbi->s_mb_stream_request)  in ext4_mb_group_or_file()
5788 			ac->ac_flags |= EXT4_MB_STREAM_ALLOC;  in ext4_mb_group_or_file()
5790 			ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC;  in ext4_mb_group_or_file()
5794 	BUG_ON(ac->ac_lg != NULL);  in ext4_mb_group_or_file()
5796 	 * locality group prealloc space are per cpu. The reason for having  in ext4_mb_group_or_file()
5797 	 * per cpu locality group is to reduce the contention between block  in ext4_mb_group_or_file()
5800 	ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups);  in ext4_mb_group_or_file()
5803 	ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC;  in ext4_mb_group_or_file()
5806 	mutex_lock(&ac->ac_lg->lg_mutex);  in ext4_mb_group_or_file()
5813 	struct super_block *sb = ar->inode->i_sb;  in ext4_mb_initialize_context()
5815 	struct ext4_super_block *es = sbi->s_es;  in ext4_mb_initialize_context()
5822 	len = ar->len;  in ext4_mb_initialize_context()
5829 	goal = ar->goal;  in ext4_mb_initialize_context()
5830 	if (goal < le32_to_cpu(es->s_first_data_block) ||  in ext4_mb_initialize_context()
5832 		goal = le32_to_cpu(es->s_first_data_block);  in ext4_mb_initialize_context()
5836 	ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical);  in ext4_mb_initialize_context()
5837 	ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_initialize_context()
5838 	ac->ac_sb = sb;  in ext4_mb_initialize_context()
5839 	ac->ac_inode = ar->inode;  in ext4_mb_initialize_context()
5840 	ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical;  in ext4_mb_initialize_context()
5841 	ac->ac_o_ex.fe_group = group;  in ext4_mb_initialize_context()
5842 	ac->ac_o_ex.fe_start = block;  in ext4_mb_initialize_context()
5843 	ac->ac_o_ex.fe_len = len;  in ext4_mb_initialize_context()
5844 	ac->ac_g_ex = ac->ac_o_ex;  in ext4_mb_initialize_context()
5845 	ac->ac_orig_goal_len = ac->ac_g_ex.fe_len;  in ext4_mb_initialize_context()
5846 	ac->ac_flags = ar->flags;  in ext4_mb_initialize_context()
5849 	 * locality group. this is a policy, actually */  in ext4_mb_initialize_context()
5854 			(unsigned) ar->len, (unsigned) ar->logical,  in ext4_mb_initialize_context()
5855 			(unsigned) ar->goal, ac->ac_flags, ac->ac_2order,  in ext4_mb_initialize_context()
5856 			(unsigned) ar->lleft, (unsigned) ar->pleft,  in ext4_mb_initialize_context()
5857 			(unsigned) ar->lright, (unsigned) ar->pright,  in ext4_mb_initialize_context()
5858 			inode_is_open_for_write(ar->inode) ? "" : "non-");  in ext4_mb_initialize_context()
5871 	mb_debug(sb, "discard locality group preallocation\n");  in ext4_mb_discard_lg_preallocations()
5873 	spin_lock(&lg->lg_prealloc_lock);  in ext4_mb_discard_lg_preallocations()
5874 	list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order],  in ext4_mb_discard_lg_preallocations()
5876 				lockdep_is_held(&lg->lg_prealloc_lock)) {  in ext4_mb_discard_lg_preallocations()
5877 		spin_lock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
5878 		if (atomic_read(&pa->pa_count)) {  in ext4_mb_discard_lg_preallocations()
5884 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
5887 		if (pa->pa_deleted) {  in ext4_mb_discard_lg_preallocations()
5888 			spin_unlock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
5892 		BUG_ON(pa->pa_type != MB_GROUP_PA);  in ext4_mb_discard_lg_preallocations()
5896 		spin_unlock(&pa->pa_lock);  in ext4_mb_discard_lg_preallocations()
5898 		list_del_rcu(&pa->pa_node.lg_list);  in ext4_mb_discard_lg_preallocations()
5899 		list_add(&pa->u.pa_tmp_list, &discard_list);  in ext4_mb_discard_lg_preallocations()
5901 		total_entries--;  in ext4_mb_discard_lg_preallocations()
5912 	spin_unlock(&lg->lg_prealloc_lock);  in ext4_mb_discard_lg_preallocations()
5917 		group = ext4_get_group_number(sb, pa->pa_pstart);  in ext4_mb_discard_lg_preallocations()
5921 			ext4_error_err(sb, -err, "Error %d loading buddy information for %u",  in ext4_mb_discard_lg_preallocations()
5926 		list_del(&pa->pa_group_list);  in ext4_mb_discard_lg_preallocations()
5931 		list_del(&pa->u.pa_tmp_list);  in ext4_mb_discard_lg_preallocations()
5932 		call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);  in ext4_mb_discard_lg_preallocations()
5948 	struct super_block *sb = ac->ac_sb;  in ext4_mb_add_n_trim()
5949 	struct ext4_locality_group *lg = ac->ac_lg;  in ext4_mb_add_n_trim()
5950 	struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa;  in ext4_mb_add_n_trim()
5952 	order = fls(pa->pa_free) - 1;  in ext4_mb_add_n_trim()
5953 	if (order > PREALLOC_TB_SIZE - 1)  in ext4_mb_add_n_trim()
5955 		order = PREALLOC_TB_SIZE - 1;  in ext4_mb_add_n_trim()
5957 	spin_lock(&lg->lg_prealloc_lock);  in ext4_mb_add_n_trim()
5958 	list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order],  in ext4_mb_add_n_trim()
5960 				lockdep_is_held(&lg->lg_prealloc_lock)) {  in ext4_mb_add_n_trim()
5961 		spin_lock(&tmp_pa->pa_lock);  in ext4_mb_add_n_trim()
5962 		if (tmp_pa->pa_deleted) {  in ext4_mb_add_n_trim()
5963 			spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_add_n_trim()
5966 		if (!added && pa->pa_free < tmp_pa->pa_free) {  in ext4_mb_add_n_trim()
5968 			list_add_tail_rcu(&pa->pa_node.lg_list,  in ext4_mb_add_n_trim()
5969 						&tmp_pa->pa_node.lg_list);  in ext4_mb_add_n_trim()
5976 		spin_unlock(&tmp_pa->pa_lock);  in ext4_mb_add_n_trim()
5980 		list_add_tail_rcu(&pa->pa_node.lg_list,  in ext4_mb_add_n_trim()
5981 					&lg->lg_prealloc_list[order]);  in ext4_mb_add_n_trim()
5982 	spin_unlock(&lg->lg_prealloc_lock);  in ext4_mb_add_n_trim()
5995 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_release_context()
5996 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_mb_release_context()
5998 		if (pa->pa_type == MB_GROUP_PA) {  in ext4_mb_release_context()
6000 			spin_lock(&pa->pa_lock);  in ext4_mb_release_context()
6001 			pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
6002 			pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
6003 			pa->pa_free -= ac->ac_b_ex.fe_len;  in ext4_mb_release_context()
6004 			pa->pa_len -= ac->ac_b_ex.fe_len;  in ext4_mb_release_context()
6005 			spin_unlock(&pa->pa_lock);  in ext4_mb_release_context()
6013 			if (likely(pa->pa_free)) {  in ext4_mb_release_context()
6014 				spin_lock(pa->pa_node_lock.lg_lock);  in ext4_mb_release_context()
6015 				list_del_rcu(&pa->pa_node.lg_list);  in ext4_mb_release_context()
6016 				spin_unlock(pa->pa_node_lock.lg_lock);  in ext4_mb_release_context()
6021 		ext4_mb_put_pa(ac, ac->ac_sb, pa);  in ext4_mb_release_context()
6023 	if (ac->ac_bitmap_page)  in ext4_mb_release_context()
6024 		put_page(ac->ac_bitmap_page);  in ext4_mb_release_context()
6025 	if (ac->ac_buddy_page)  in ext4_mb_release_context()
6026 		put_page(ac->ac_buddy_page);  in ext4_mb_release_context()
6027 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)  in ext4_mb_release_context()
6028 		mutex_unlock(&ac->ac_lg->lg_mutex);  in ext4_mb_release_context()
6048 		needed -= ret;  in ext4_mb_discard_preallocations()
6067 	freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len);  in ext4_mb_discard_preallocations_should_retry()
6073 	if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) {  in ext4_mb_discard_preallocations_should_retry()
6074 		ac->ac_flags |= EXT4_MB_STRICT_CHECK;  in ext4_mb_discard_preallocations_should_retry()
6093 	struct super_block *sb = ar->inode->i_sb;  in ext4_mb_new_blocks_simple()
6100 	struct ext4_super_block *es = sbi->s_es;  in ext4_mb_new_blocks_simple()
6102 	goal = ar->goal;  in ext4_mb_new_blocks_simple()
6103 	if (goal < le32_to_cpu(es->s_first_data_block) ||  in ext4_mb_new_blocks_simple()
6105 		goal = le32_to_cpu(es->s_first_data_block);  in ext4_mb_new_blocks_simple()
6107 	ar->len = 0;  in ext4_mb_new_blocks_simple()
6109 	for (nr = ext4_get_groups_count(sb); nr > 0; nr--) {  in ext4_mb_new_blocks_simple()
6118 			i = mb_find_next_zero_bit(bitmap_bh->b_data, max,  in ext4_mb_new_blocks_simple()
6140 		*errp = -ENOSPC;  in ext4_mb_new_blocks_simple()
6146 	ar->len = 1;  in ext4_mb_new_blocks_simple()
6170 	sb = ar->inode->i_sb;  in ext4_mb_new_blocks()
6174 	if (sbi->s_mount_state & EXT4_FC_REPLAY)  in ext4_mb_new_blocks()
6178 	if (ext4_is_quota_file(ar->inode))  in ext4_mb_new_blocks()
6179 		ar->flags |= EXT4_MB_USE_ROOT_BLOCKS;  in ext4_mb_new_blocks()
6181 	if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) {  in ext4_mb_new_blocks()
6186 		while (ar->len &&  in ext4_mb_new_blocks()
6187 			ext4_claim_free_clusters(sbi, ar->len, ar->flags)) {  in ext4_mb_new_blocks()
6191 			ar->len = ar->len >> 1;  in ext4_mb_new_blocks()
6193 		if (!ar->len) {  in ext4_mb_new_blocks()
6195 			*errp = -ENOSPC;  in ext4_mb_new_blocks()
6198 		reserv_clstrs = ar->len;  in ext4_mb_new_blocks()
6199 		if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) {  in ext4_mb_new_blocks()
6200 			dquot_alloc_block_nofail(ar->inode,  in ext4_mb_new_blocks()
6201 						 EXT4_C2B(sbi, ar->len));  in ext4_mb_new_blocks()
6203 			while (ar->len &&  in ext4_mb_new_blocks()
6204 				dquot_alloc_block(ar->inode,  in ext4_mb_new_blocks()
6205 						  EXT4_C2B(sbi, ar->len))) {  in ext4_mb_new_blocks()
6207 				ar->flags |= EXT4_MB_HINT_NOPREALLOC;  in ext4_mb_new_blocks()
6208 				ar->len--;  in ext4_mb_new_blocks()
6211 		inquota = ar->len;  in ext4_mb_new_blocks()
6212 		if (ar->len == 0) {  in ext4_mb_new_blocks()
6213 			*errp = -EDQUOT;  in ext4_mb_new_blocks()
6220 		ar->len = 0;  in ext4_mb_new_blocks()
6221 		*errp = -ENOMEM;  in ext4_mb_new_blocks()
6227 	ac->ac_op = EXT4_MB_HISTORY_PREALLOC;  in ext4_mb_new_blocks()
6230 		ac->ac_op = EXT4_MB_HISTORY_ALLOC;  in ext4_mb_new_blocks()
6240 		 * pa allocated above is added to grp->bb_prealloc_list only  in ext4_mb_new_blocks()
6242 		 * ac->ac_status == AC_STATUS_FOUND.  in ext4_mb_new_blocks()
6243 		 * And error from above mean ac->ac_status != AC_STATUS_FOUND  in ext4_mb_new_blocks()
6251 		if (ac->ac_status == AC_STATUS_FOUND &&  in ext4_mb_new_blocks()
6252 			ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len)  in ext4_mb_new_blocks()
6255 	if (likely(ac->ac_status == AC_STATUS_FOUND)) {  in ext4_mb_new_blocks()
6261 			block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_blocks()
6262 			ar->len = ac->ac_b_ex.fe_len;  in ext4_mb_new_blocks()
6273 		*errp = -ENOSPC;  in ext4_mb_new_blocks()
6278 		ac->ac_b_ex.fe_len = 0;  in ext4_mb_new_blocks()
6279 		ar->len = 0;  in ext4_mb_new_blocks()
6285 	if (inquota && ar->len < inquota)  in ext4_mb_new_blocks()
6286 		dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len));  in ext4_mb_new_blocks()
6287 	if (!ar->len) {  in ext4_mb_new_blocks()
6288 		if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0)  in ext4_mb_new_blocks()
6290 			percpu_counter_sub(&sbi->s_dirtyclusters_counter,  in ext4_mb_new_blocks()
6309 	if ((entry->efd_tid != new_entry->efd_tid) ||  in ext4_try_merge_freed_extent()
6310 	    (entry->efd_group != new_entry->efd_group))  in ext4_try_merge_freed_extent()
6312 	if (entry->efd_start_cluster + entry->efd_count ==  in ext4_try_merge_freed_extent()
6313 	    new_entry->efd_start_cluster) {  in ext4_try_merge_freed_extent()
6314 		new_entry->efd_start_cluster = entry->efd_start_cluster;  in ext4_try_merge_freed_extent()
6315 		new_entry->efd_count += entry->efd_count;  in ext4_try_merge_freed_extent()
6316 	} else if (new_entry->efd_start_cluster + new_entry->efd_count ==  in ext4_try_merge_freed_extent()
6317 		   entry->efd_start_cluster) {  in ext4_try_merge_freed_extent()
6318 		new_entry->efd_count += entry->efd_count;  in ext4_try_merge_freed_extent()
6321 	spin_lock(&sbi->s_md_lock);  in ext4_try_merge_freed_extent()
6322 	list_del(&entry->efd_list);  in ext4_try_merge_freed_extent()
6323 	spin_unlock(&sbi->s_md_lock);  in ext4_try_merge_freed_extent()
6324 	rb_erase(&entry->efd_node, entry_rb_root);  in ext4_try_merge_freed_extent()
6332 	ext4_group_t group = e4b->bd_group;  in ext4_mb_free_metadata()
6334 	ext4_grpblk_t clusters = new_entry->efd_count;  in ext4_mb_free_metadata()
6336 	struct ext4_group_info *db = e4b->bd_info;  in ext4_mb_free_metadata()
6337 	struct super_block *sb = e4b->bd_sb;  in ext4_mb_free_metadata()
6339 	struct rb_node **n = &db->bb_free_root.rb_node, *node;  in ext4_mb_free_metadata()
6343 	BUG_ON(e4b->bd_bitmap_page == NULL);  in ext4_mb_free_metadata()
6344 	BUG_ON(e4b->bd_buddy_page == NULL);  in ext4_mb_free_metadata()
6346 	new_node = &new_entry->efd_node;  in ext4_mb_free_metadata()
6347 	cluster = new_entry->efd_start_cluster;  in ext4_mb_free_metadata()
6353 		 * on-disk bitmap and lose not-yet-available  in ext4_mb_free_metadata()
6355 		get_page(e4b->bd_buddy_page);  in ext4_mb_free_metadata()
6356 		get_page(e4b->bd_bitmap_page);  in ext4_mb_free_metadata()
6361 		if (cluster < entry->efd_start_cluster)  in ext4_mb_free_metadata()
6362 			n = &(*n)->rb_left;  in ext4_mb_free_metadata()
6363 		else if (cluster >= (entry->efd_start_cluster + entry->efd_count))  in ext4_mb_free_metadata()
6364 			n = &(*n)->rb_right;  in ext4_mb_free_metadata()
6369 				"Block already on to-be-freed list");  in ext4_mb_free_metadata()
6376 	rb_insert_color(new_node, &db->bb_free_root);  in ext4_mb_free_metadata()
6383 					    &(db->bb_free_root));  in ext4_mb_free_metadata()
6390 					    &(db->bb_free_root));  in ext4_mb_free_metadata()
6393 	spin_lock(&sbi->s_md_lock);  in ext4_mb_free_metadata()
6394 	list_add_tail(&new_entry->efd_list, &sbi->s_freed_data_list);  in ext4_mb_free_metadata()
6395 	sbi->s_mb_free_pending += clusters;  in ext4_mb_free_metadata()
6396 	spin_unlock(&sbi->s_md_lock);  in ext4_mb_free_metadata()
6403 	struct super_block *sb = inode->i_sb;  in ext4_free_blocks_simple()
6421 		if (!mb_test_bit(blkoff + i, bitmap_bh->b_data))  in ext4_free_blocks_simple()
6424 	mb_clear_bits(bitmap_bh->b_data, blkoff, count);  in ext4_free_blocks_simple()
6430 		count - already_freed);  in ext4_free_blocks_simple()
6442  * ext4_mb_clear_bb() -- helper function for freeing blocks.
6455 	struct super_block *sb = inode->i_sb;  in ext4_mb_clear_bb()
6472 		ext4_error(sb, "Freeing blocks in system zone - "  in ext4_mb_clear_bb()
6492 		overflow = EXT4_C2B(sbi, bit) + count -  in ext4_mb_clear_bb()
6494 		count -= overflow;  in ext4_mb_clear_bb()
6507 		err = -EIO;  in ext4_mb_clear_bb()
6513 		ext4_error(sb, "Freeing blocks in system zone - "  in ext4_mb_clear_bb()
6527 	 * to unshare ->b_data if a currently-committing transaction is  in ext4_mb_clear_bb()
6538 			BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data));  in ext4_mb_clear_bb()
6565 		new_entry->efd_start_cluster = bit;  in ext4_mb_clear_bb()
6566 		new_entry->efd_group = block_group;  in ext4_mb_clear_bb()
6567 		new_entry->efd_count = count_clusters;  in ext4_mb_clear_bb()
6568 		new_entry->efd_tid = handle->h_transaction->t_tid;  in ext4_mb_clear_bb()
6571 		mb_clear_bits(bitmap_bh->b_data, bit, count_clusters);  in ext4_mb_clear_bb()
6574 		/* need to update group_info->bb_free and bitmap  in ext4_mb_clear_bb()
6581 			if (err && err != -EOPNOTSUPP)  in ext4_mb_clear_bb()
6591 		mb_clear_bits(bitmap_bh->b_data, bit, count_clusters);  in ext4_mb_clear_bb()
6601 	if (sbi->s_log_groups_per_flex) {  in ext4_mb_clear_bb()
6605 						  flex_group)->free_clusters);  in ext4_mb_clear_bb()
6616 		percpu_counter_add(&sbi->s_freeclusters_counter,  in ext4_mb_clear_bb()
6646  * ext4_free_blocks() -- Free given blocks and update quota
6658 	struct super_block *sb = inode->i_sb;  in ext4_free_blocks()
6666 			BUG_ON(block != bh->b_blocknr);  in ext4_free_blocks()
6668 			block = bh->b_blocknr;  in ext4_free_blocks()
6671 	if (sbi->s_mount_state & EXT4_FC_REPLAY) {  in ext4_free_blocks()
6680 		ext4_error(sb, "Freeing blocks not in datazone - "  in ext4_free_blocks()
6706 			overflow = sbi->s_cluster_ratio - overflow;  in ext4_free_blocks()
6709 				count -= overflow;  in ext4_free_blocks()
6713 			block -= overflow;  in ext4_free_blocks()
6723 				count -= overflow;  in ext4_free_blocks()
6727 			count += sbi->s_cluster_ratio - overflow;  in ext4_free_blocks()
6739 				bh = sb_find_get_block(inode->i_sb, block + i);  in ext4_free_blocks()
6748  * ext4_group_add_blocks() -- Add given blocks to an existing group
6770 	ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1);  in ext4_group_add_blocks()
6771 	unsigned long cluster_count = last_cluster - first_cluster + 1;  in ext4_group_add_blocks()
6773 	ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1);  in ext4_group_add_blocks()
6786 		err = -EINVAL;  in ext4_group_add_blocks()
6799 		err = -EIO;  in ext4_group_add_blocks()
6804 		ext4_error(sb, "Adding blocks in system zones - "  in ext4_group_add_blocks()
6807 		err = -EINVAL;  in ext4_group_add_blocks()
6819 	 * to unshare ->b_data if a currently-committing transaction is  in ext4_group_add_blocks()
6829 		if (!mb_test_bit(bit + i, bitmap_bh->b_data)) {  in ext4_group_add_blocks()
6843 	 * need to update group_info->bb_free and bitmap  in ext4_group_add_blocks()
6848 	mb_clear_bits(bitmap_bh->b_data, bit, cluster_count);  in ext4_group_add_blocks()
6856 	percpu_counter_add(&sbi->s_freeclusters_counter,  in ext4_group_add_blocks()
6859 	if (sbi->s_log_groups_per_flex) {  in ext4_group_add_blocks()
6863 						  flex_group)->free_clusters);  in ext4_group_add_blocks()
6885  * ext4_trim_extent -- function to TRIM one single free extent in the group
6901 	ext4_group_t group = e4b->bd_group;  in ext4_trim_extent()
6929 	if (grp < (ext4_get_groups_count(sb) - 1))  in ext4_last_grp_cluster()
6932 		nr_clusters_in_group = (ext4_blocks_count(EXT4_SB(sb)->s_es) -  in ext4_last_grp_cluster()
6936 	return nr_clusters_in_group - 1;  in ext4_last_grp_cluster()
6947 __acquires(ext4_group_lock_ptr(sb, e4b->bd_group))  in ext4_try_to_trim_range()
6948 __releases(ext4_group_lock_ptr(sb, e4b->bd_group))  in ext4_try_to_trim_range()
6954 	if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))  in ext4_try_to_trim_range()
6957 	last = ext4_last_grp_cluster(sb, e4b->bd_group);  in ext4_try_to_trim_range()
6958 	bitmap = e4b->bd_bitmap;  in ext4_try_to_trim_range()
6962 	start = max(e4b->bd_info->bb_first_free, start);  in ext4_try_to_trim_range()
6975 		if ((next - start) >= minblocks) {  in ext4_try_to_trim_range()
6976 			int ret = ext4_trim_extent(sb, start, next - start, e4b);  in ext4_try_to_trim_range()
6978 			if (ret && ret != -EOPNOTSUPP)  in ext4_try_to_trim_range()
6980 			count += next - start;  in ext4_try_to_trim_range()
6982 		free_count += next - start;  in ext4_try_to_trim_range()
6989 			ext4_unlock_group(sb, e4b->bd_group);  in ext4_try_to_trim_range()
6991 			ext4_lock_group(sb, e4b->bd_group);  in ext4_try_to_trim_range()
6994 		if ((e4b->bd_info->bb_free - free_count) < minblocks)  in ext4_try_to_trim_range()
6999 		EXT4_MB_GRP_SET_TRIMMED(e4b->bd_info);  in ext4_try_to_trim_range()
7005  * ext4_trim_all_free -- function to trim all free space in alloc. group
7037 	    minblocks < EXT4_SB(sb)->s_last_trim_minblks)  in ext4_trim_all_free()
7052  * ext4_trim_fs() -- trim ioctl handle function
7065 	unsigned int discard_granularity = bdev_discard_granularity(sb->s_bdev);  in ext4_trim_fs()
7071 			le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);  in ext4_trim_fs()
7072 	ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es);  in ext4_trim_fs()
7075 	start = range->start >> sb->s_blocksize_bits;  in ext4_trim_fs()
7076 	end = start + (range->len >> sb->s_blocksize_bits) - 1;  in ext4_trim_fs()
7078 			      range->minlen >> sb->s_blocksize_bits);  in ext4_trim_fs()
7082 	    range->len < sb->s_blocksize)  in ext4_trim_fs()
7083 		return -EINVAL;  in ext4_trim_fs()
7085 	if (range->minlen < discard_granularity) {  in ext4_trim_fs()
7087 				discard_granularity >> sb->s_blocksize_bits);  in ext4_trim_fs()
7091 	if (end >= max_blks - 1)  in ext4_trim_fs()
7092 		end = max_blks - 1;  in ext4_trim_fs()
7105 	end = EXT4_CLUSTERS_PER_GROUP(sb) - 1;  in ext4_trim_fs()
7122 		 * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to  in ext4_trim_fs()
7128 		if (grp->bb_free >= minlen) {  in ext4_trim_fs()
7146 		EXT4_SB(sb)->s_last_trim_minblks = minlen;  in ext4_trim_fs()
7149 	range->len = EXT4_C2B(EXT4_SB(sb), trimmed) << sb->s_blocksize_bits;  in ext4_trim_fs()
7176 	start = max(e4b.bd_info->bb_first_free, first);  in ext4_mballoc_query_range()
7178 		end = EXT4_CLUSTERS_PER_GROUP(sb) - 1;  in ext4_mballoc_query_range()
7183 		error = meta_formatter(sb, group, first, start - first,  in ext4_mballoc_query_range()
7196 		error = formatter(sb, group, start, next - start, priv);  in ext4_mballoc_query_range()