Lines Matching +full:1 +full:ac
101 * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
134 * 1) Array of largest free order lists (sbi->s_mb_largest_free_orders)
149 * average fragment size >= 2^i and < 2^(i+1). The average fragment size
160 * in the data structure (1) above where largest_free_order = order of the
163 * lookup in O(1) time.
168 * size group lists (data structure 2) in O(1) time.
177 * suitable block group in O(1) time and results in faster allocation at the
274 * 1) if buddy is referenced, it's already initialized
420 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac);
422 static bool ext4_mb_good_group(struct ext4_allocation_context *ac,
431 * 1. We sample the percpu discard_pa_seq counter before trying for block
533 if (order > e4b->bd_blkbits + 1) { in mb_find_buddy()
540 *max = 1 << (e4b->bd_blkbits + 3); in mb_find_buddy()
683 int order = e4b->bd_blkbits + 1; in __mb_check_buddy()
700 while (order > 1) { in __mb_check_buddy()
703 buddy2 = mb_find_buddy(e4b, order - 1, &max2); in __mb_check_buddy()
713 if (!mb_test_bit(i << 1, buddy2)) { in __mb_check_buddy()
715 mb_test_bit((i<<1)+1, buddy2)); in __mb_check_buddy()
720 /* both bits in buddy2 must be 1 */ in __mb_check_buddy()
721 MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2)); in __mb_check_buddy()
722 MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2)); in __mb_check_buddy()
724 for (j = 0; j < (1 << order); j++) { in __mb_check_buddy()
725 k = (i * (1 << order)) + j; in __mb_check_buddy()
735 fstart = -1; in __mb_check_buddy()
740 if (fstart == -1) { in __mb_check_buddy()
746 fstart = -1; in __mb_check_buddy()
748 for (j = 0; j < e4b->bd_blkbits + 1; j++) { in __mb_check_buddy()
801 max = ffs(first | border) - 1; in ext4_mb_mark_free_simple()
804 min = fls(len) - 1; in ext4_mb_mark_free_simple()
808 chunk = 1 << min; in ext4_mb_mark_free_simple()
826 * We don't bother with a special lists groups with only 1 block free in mb_avg_fragment_size_order()
852 if (grp->bb_avg_fragment_size_order != -1) { in mb_update_avg_fragment_size()
872 static void ext4_mb_choose_next_group_p2_aligned(struct ext4_allocation_context *ac, in ext4_mb_choose_next_group_p2_aligned() argument
875 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_choose_next_group_p2_aligned()
879 if (ac->ac_status == AC_STATUS_FOUND) in ext4_mb_choose_next_group_p2_aligned()
882 if (unlikely(sbi->s_mb_stats && ac->ac_flags & EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED)) in ext4_mb_choose_next_group_p2_aligned()
885 for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) { in ext4_mb_choose_next_group_p2_aligned()
897 if (likely(ext4_mb_good_group(ac, iter->bb_group, CR_POWER2_ALIGNED))) { in ext4_mb_choose_next_group_p2_aligned()
899 ac->ac_flags |= EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED; in ext4_mb_choose_next_group_p2_aligned()
915 ext4_mb_find_good_group_avg_frag_lists(struct ext4_allocation_context *ac, int order) in ext4_mb_find_good_group_avg_frag_lists() argument
917 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_find_good_group_avg_frag_lists()
921 enum criteria cr = ac->ac_criteria; in ext4_mb_find_good_group_avg_frag_lists()
933 if (likely(ext4_mb_good_group(ac, iter->bb_group, cr))) { in ext4_mb_find_good_group_avg_frag_lists()
946 static void ext4_mb_choose_next_group_goal_fast(struct ext4_allocation_context *ac, in ext4_mb_choose_next_group_goal_fast() argument
949 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_choose_next_group_goal_fast()
953 if (unlikely(ac->ac_flags & EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED)) { in ext4_mb_choose_next_group_goal_fast()
958 for (i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len); in ext4_mb_choose_next_group_goal_fast()
959 i < MB_NUM_ORDERS(ac->ac_sb); i++) { in ext4_mb_choose_next_group_goal_fast()
960 grp = ext4_mb_find_good_group_avg_frag_lists(ac, i); in ext4_mb_choose_next_group_goal_fast()
963 ac->ac_flags |= EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED; in ext4_mb_choose_next_group_goal_fast()
976 if (ac->ac_flags & EXT4_MB_HINT_DATA) in ext4_mb_choose_next_group_goal_fast()
991 static void ext4_mb_choose_next_group_best_avail(struct ext4_allocation_context *ac, in ext4_mb_choose_next_group_best_avail() argument
994 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_choose_next_group_best_avail()
999 if (unlikely(ac->ac_flags & EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED)) { in ext4_mb_choose_next_group_best_avail()
1006 * retrieving back the length using (1 << order) inaccurate. Hence, use in ext4_mb_choose_next_group_best_avail()
1010 order = fls(ac->ac_g_ex.fe_len) - 1; in ext4_mb_choose_next_group_best_avail()
1021 if (1 << min_order < num_stripe_clusters) in ext4_mb_choose_next_group_best_avail()
1023 * We consider 1 order less because later we round in ext4_mb_choose_next_group_best_avail()
1026 min_order = fls(num_stripe_clusters) - 1; in ext4_mb_choose_next_group_best_avail()
1029 if (1 << min_order < ac->ac_o_ex.fe_len) in ext4_mb_choose_next_group_best_avail()
1030 min_order = fls(ac->ac_o_ex.fe_len); in ext4_mb_choose_next_group_best_avail()
1039 ac->ac_g_ex.fe_len = 1 << i; in ext4_mb_choose_next_group_best_avail()
1047 ac->ac_g_ex.fe_len = roundup(ac->ac_g_ex.fe_len, in ext4_mb_choose_next_group_best_avail()
1051 frag_order = mb_avg_fragment_size_order(ac->ac_sb, in ext4_mb_choose_next_group_best_avail()
1052 ac->ac_g_ex.fe_len); in ext4_mb_choose_next_group_best_avail()
1054 grp = ext4_mb_find_good_group_avg_frag_lists(ac, frag_order); in ext4_mb_choose_next_group_best_avail()
1057 ac->ac_flags |= EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED; in ext4_mb_choose_next_group_best_avail()
1063 ac->ac_g_ex.fe_len = ac->ac_orig_goal_len; in ext4_mb_choose_next_group_best_avail()
1067 static inline int should_optimize_scan(struct ext4_allocation_context *ac) in should_optimize_scan() argument
1069 if (unlikely(!test_opt2(ac->ac_sb, MB_OPTIMIZE_SCAN))) in should_optimize_scan()
1071 if (ac->ac_criteria >= CR_GOAL_LEN_SLOW) in should_optimize_scan()
1073 if (!ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) in should_optimize_scan()
1075 return 1; in should_optimize_scan()
1083 next_linear_group(struct ext4_allocation_context *ac, ext4_group_t group, in next_linear_group() argument
1086 if (!should_optimize_scan(ac)) in next_linear_group()
1089 if (ac->ac_groups_linear_remaining) { in next_linear_group()
1090 ac->ac_groups_linear_remaining--; in next_linear_group()
1100 return group + 1 >= ngroups ? 0 : group + 1; in next_linear_group()
1106 * @ac Allocation Context
1116 static void ext4_mb_choose_next_group(struct ext4_allocation_context *ac, in ext4_mb_choose_next_group() argument
1119 *new_cr = ac->ac_criteria; in ext4_mb_choose_next_group()
1121 if (!should_optimize_scan(ac) || ac->ac_groups_linear_remaining) { in ext4_mb_choose_next_group()
1122 *group = next_linear_group(ac, *group, ngroups); in ext4_mb_choose_next_group()
1127 ext4_mb_choose_next_group_p2_aligned(ac, new_cr, group, ngroups); in ext4_mb_choose_next_group()
1129 ext4_mb_choose_next_group_goal_fast(ac, new_cr, group, ngroups); in ext4_mb_choose_next_group()
1131 ext4_mb_choose_next_group_best_avail(ac, new_cr, group, ngroups); in ext4_mb_choose_next_group()
1137 WARN_ON(1); in ext4_mb_choose_next_group()
1151 for (i = MB_NUM_ORDERS(sb) - 1; i >= 0; i--) in mb_set_largest_free_order()
1203 if (len > 1) in ext4_mb_generate_buddy()
1238 int order = 1; in mb_regenerate_buddy()
1260 * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
1299 groups_per_page = blocks_per_page >> 1; in ext4_mb_init_cache()
1301 groups_per_page = 1; in ext4_mb_init_cache()
1304 if (groups_per_page > 1) { in ext4_mb_init_cache()
1354 group = (first_block + i) >> 1; in ext4_mb_init_cache()
1385 if ((first_block + i) & 1) { in ext4_mb_init_cache()
1740 int order = 1, max; in mb_find_order_for_block()
1744 BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); in mb_find_order_for_block()
1746 while (order <= e4b->bd_blkbits + 1) { in mb_find_order_for_block()
1776 * will return first found zero bit if any, -1 otherwise
1781 int zero_bit = -1; in mb_test_and_clear_bits()
1788 if (*addr != (__u32)(-1) && zero_bit == -1) in mb_test_and_clear_bits()
1794 if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1) in mb_test_and_clear_bits()
1825 return 1; in mb_buddy_adjust_border()
1830 return -1; in mb_buddy_adjust_border()
1837 int order = 1; in mb_buddy_mark_free()
1854 * | 1 | 1 | 1 | 1 | in mb_buddy_mark_free()
1856 * | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | in mb_buddy_mark_free()
1858 * 0 1 2 3 4 5 6 7 in mb_buddy_mark_free()
1861 * Neither [1] nor [6] is aligned to above layer. in mb_buddy_mark_free()
1872 if (first & 1) in mb_buddy_mark_free()
1873 e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1); in mb_buddy_mark_free()
1874 if (!(last & 1)) in mb_buddy_mark_free()
1875 e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1); in mb_buddy_mark_free()
1882 mb_clear_bits(buddy, first, last - first + 1); in mb_buddy_mark_free()
1883 e4b->bd_info->bb_counters[order - 1] += last - first + 1; in mb_buddy_mark_free()
1886 first >>= 1; in mb_buddy_mark_free()
1887 last >>= 1; in mb_buddy_mark_free()
1898 int last = first + count - 1; in mb_free_blocks()
1916 left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap); in mb_free_blocks()
1918 if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0]) in mb_free_blocks()
1919 right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap); in mb_free_blocks()
1921 if (unlikely(block != -1)) { in mb_free_blocks()
1962 if (first & 1) { in mb_free_blocks()
1964 e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1; in mb_free_blocks()
1966 if (!(last & 1)) { in mb_free_blocks()
1968 e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1; in mb_free_blocks()
1972 mb_buddy_mark_free(e4b, first >> 1, last >> 1); in mb_free_blocks()
2004 ex->fe_len = 1 << order; in mb_find_extent()
2016 if (block + 1 >= max) in mb_find_extent()
2019 next = (block + 1) * (1 << order); in mb_find_extent()
2026 ex->fe_len += 1 << order; in mb_find_extent()
2031 WARN_ON(1); in mb_find_extent()
2070 mlen = !mb_test_bit(start - 1, e4b->bd_bitmap); in mb_mark_used()
2083 if (((start >> ord) << ord) == start && len >= (1 << ord)) { in mb_mark_used()
2085 mlen = 1 << ord; in mb_mark_used()
2110 cur = (start >> ord) & ~1U; in mb_mark_used()
2113 mb_clear_bit(cur + 1, buddy); in mb_mark_used()
2130 static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, in ext4_mb_use_best_found() argument
2133 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_best_found()
2136 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); in ext4_mb_use_best_found()
2137 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_use_best_found()
2139 ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); in ext4_mb_use_best_found()
2140 ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; in ext4_mb_use_best_found()
2141 ret = mb_mark_used(e4b, &ac->ac_b_ex); in ext4_mb_use_best_found()
2145 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_use_best_found()
2147 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_best_found()
2148 ac->ac_tail = ret & 0xffff; in ext4_mb_use_best_found()
2149 ac->ac_buddy = ret >> 16; in ext4_mb_use_best_found()
2158 ac->ac_bitmap_page = e4b->bd_bitmap_page; in ext4_mb_use_best_found()
2159 get_page(ac->ac_bitmap_page); in ext4_mb_use_best_found()
2160 ac->ac_buddy_page = e4b->bd_buddy_page; in ext4_mb_use_best_found()
2161 get_page(ac->ac_buddy_page); in ext4_mb_use_best_found()
2163 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_use_best_found()
2165 sbi->s_mb_last_group = ac->ac_f_ex.fe_group; in ext4_mb_use_best_found()
2166 sbi->s_mb_last_start = ac->ac_f_ex.fe_start; in ext4_mb_use_best_found()
2174 if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) in ext4_mb_use_best_found()
2175 ext4_mb_new_preallocation(ac); in ext4_mb_use_best_found()
2179 static void ext4_mb_check_limits(struct ext4_allocation_context *ac, in ext4_mb_check_limits() argument
2183 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_check_limits()
2184 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_check_limits()
2185 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_check_limits()
2187 if (ac->ac_status == AC_STATUS_FOUND) in ext4_mb_check_limits()
2192 if (ac->ac_found > sbi->s_mb_max_to_scan && in ext4_mb_check_limits()
2193 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_check_limits()
2194 ac->ac_status = AC_STATUS_BREAK; in ext4_mb_check_limits()
2204 if (finish_group || ac->ac_found > sbi->s_mb_min_to_scan) in ext4_mb_check_limits()
2205 ext4_mb_use_best_found(ac, e4b); in ext4_mb_check_limits()
2232 static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, in ext4_mb_measure_extent() argument
2236 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_measure_extent()
2237 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_measure_extent()
2240 BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
2241 BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
2242 BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); in ext4_mb_measure_extent()
2244 ac->ac_found++; in ext4_mb_measure_extent()
2245 ac->ac_cX_found[ac->ac_criteria]++; in ext4_mb_measure_extent()
2250 if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_measure_extent()
2252 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
2261 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
2289 ext4_mb_check_limits(ac, e4b, 0); in ext4_mb_measure_extent()
2293 void ext4_mb_try_best_found(struct ext4_allocation_context *ac, in ext4_mb_try_best_found() argument
2296 struct ext4_free_extent ex = ac->ac_b_ex; in ext4_mb_try_best_found()
2302 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_try_best_found()
2306 ext4_lock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
2313 ac->ac_b_ex = ex; in ext4_mb_try_best_found()
2314 ext4_mb_use_best_found(ac, e4b); in ext4_mb_try_best_found()
2318 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
2323 int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, in ext4_mb_find_by_goal() argument
2326 ext4_group_t group = ac->ac_g_ex.fe_group; in ext4_mb_find_by_goal()
2329 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_find_by_goal()
2330 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_find_by_goal()
2335 if (!(ac->ac_flags & (EXT4_MB_HINT_TRY_GOAL | EXT4_MB_HINT_GOAL_ONLY))) in ext4_mb_find_by_goal()
2340 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_find_by_goal()
2344 ext4_lock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
2348 max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, in ext4_mb_find_by_goal()
2349 ac->ac_g_ex.fe_len, &ex); in ext4_mb_find_by_goal()
2352 if (max >= ac->ac_g_ex.fe_len && in ext4_mb_find_by_goal()
2353 ac->ac_g_ex.fe_len == EXT4_B2C(sbi, sbi->s_stripe)) { in ext4_mb_find_by_goal()
2356 start = ext4_grp_offs_to_block(ac->ac_sb, &ex); in ext4_mb_find_by_goal()
2359 ac->ac_found++; in ext4_mb_find_by_goal()
2360 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
2361 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
2363 } else if (max >= ac->ac_g_ex.fe_len) { in ext4_mb_find_by_goal()
2365 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
2366 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
2367 ac->ac_found++; in ext4_mb_find_by_goal()
2368 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
2369 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
2370 } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { in ext4_mb_find_by_goal()
2374 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
2375 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
2376 ac->ac_found++; in ext4_mb_find_by_goal()
2377 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
2378 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
2381 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
2392 void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, in ext4_mb_simple_scan_group() argument
2395 struct super_block *sb = ac->ac_sb; in ext4_mb_simple_scan_group()
2402 BUG_ON(ac->ac_2order <= 0); in ext4_mb_simple_scan_group()
2403 for (i = ac->ac_2order; i < MB_NUM_ORDERS(sb); i++) { in ext4_mb_simple_scan_group()
2414 ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0, in ext4_mb_simple_scan_group()
2417 ext4_mark_group_bitmap_corrupted(ac->ac_sb, in ext4_mb_simple_scan_group()
2422 ac->ac_found++; in ext4_mb_simple_scan_group()
2423 ac->ac_cX_found[ac->ac_criteria]++; in ext4_mb_simple_scan_group()
2425 ac->ac_b_ex.fe_len = 1 << i; in ext4_mb_simple_scan_group()
2426 ac->ac_b_ex.fe_start = k << i; in ext4_mb_simple_scan_group()
2427 ac->ac_b_ex.fe_group = e4b->bd_group; in ext4_mb_simple_scan_group()
2429 ext4_mb_use_best_found(ac, e4b); in ext4_mb_simple_scan_group()
2431 BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len); in ext4_mb_simple_scan_group()
2446 void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, in ext4_mb_complex_scan_group() argument
2449 struct super_block *sb = ac->ac_sb; in ext4_mb_complex_scan_group()
2461 while (free && ac->ac_status == AC_STATUS_CONTINUE) { in ext4_mb_complex_scan_group()
2479 if (!ext4_mb_cr_expensive(ac->ac_criteria)) { in ext4_mb_complex_scan_group()
2490 if (freelen < ac->ac_g_ex.fe_len) { in ext4_mb_complex_scan_group()
2497 mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); in ext4_mb_complex_scan_group()
2515 ext4_mb_measure_extent(ac, &ex, e4b); in ext4_mb_complex_scan_group()
2521 ext4_mb_check_limits(ac, e4b, 1); in ext4_mb_complex_scan_group()
2529 void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, in ext4_mb_scan_aligned() argument
2532 struct super_block *sb = ac->ac_sb; in ext4_mb_scan_aligned()
2546 a = first_group_block + sbi->s_stripe - 1; in ext4_mb_scan_aligned()
2556 ac->ac_found++; in ext4_mb_scan_aligned()
2557 ac->ac_cX_found[ac->ac_criteria]++; in ext4_mb_scan_aligned()
2559 ac->ac_b_ex = ex; in ext4_mb_scan_aligned()
2560 ext4_mb_use_best_found(ac, e4b); in ext4_mb_scan_aligned()
2570 * Returns either 1 or 0 indicating that the group is either suitable
2573 static bool ext4_mb_good_group(struct ext4_allocation_context *ac, in ext4_mb_good_group() argument
2577 int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); in ext4_mb_good_group()
2578 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group()
2595 BUG_ON(ac->ac_2order == 0); in ext4_mb_good_group()
2598 if ((ac->ac_flags & EXT4_MB_HINT_DATA) && in ext4_mb_good_group()
2603 if (free < ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2606 if (ac->ac_2order >= MB_NUM_ORDERS(ac->ac_sb)) in ext4_mb_good_group()
2609 if (grp->bb_largest_free_order < ac->ac_2order) in ext4_mb_good_group()
2615 if ((free / fragments) >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2619 if (free >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2642 static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac, in ext4_mb_good_group_nolock() argument
2645 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group_nolock()
2646 struct super_block *sb = ac->ac_sb; in ext4_mb_good_group_nolock()
2648 bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK; in ext4_mb_good_group_nolock()
2655 atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]); in ext4_mb_good_group_nolock()
2668 if (cr < CR_ANY_FREE && free < ac->ac_g_ex.fe_len) in ext4_mb_good_group_nolock()
2694 ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) && in ext4_mb_good_group_nolock()
2707 ret = ext4_mb_good_group(ac, group, cr); in ext4_mb_good_group_nolock()
2791 ext4_mb_regular_allocator(struct ext4_allocation_context *ac) in ext4_mb_regular_allocator() argument
2802 sb = ac->ac_sb; in ext4_mb_regular_allocator()
2806 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) in ext4_mb_regular_allocator()
2809 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_regular_allocator()
2812 err = ext4_mb_find_by_goal(ac, &e4b); in ext4_mb_regular_allocator()
2813 if (err || ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
2816 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_regular_allocator()
2820 * ac->ac_2order is set only if the fe_len is a power of 2 in ext4_mb_regular_allocator()
2821 * if ac->ac_2order is set we also set criteria to CR_POWER2_ALIGNED in ext4_mb_regular_allocator()
2824 i = fls(ac->ac_g_ex.fe_len); in ext4_mb_regular_allocator()
2825 ac->ac_2order = 0; in ext4_mb_regular_allocator()
2834 if (is_power_of_2(ac->ac_g_ex.fe_len)) in ext4_mb_regular_allocator()
2835 ac->ac_2order = array_index_nospec(i - 1, in ext4_mb_regular_allocator()
2840 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_regular_allocator()
2843 ac->ac_g_ex.fe_group = sbi->s_mb_last_group; in ext4_mb_regular_allocator()
2844 ac->ac_g_ex.fe_start = sbi->s_mb_last_start; in ext4_mb_regular_allocator()
2853 if (ac->ac_2order) in ext4_mb_regular_allocator()
2856 for (; cr < EXT4_MB_NUM_CRS && ac->ac_status == AC_STATUS_CONTINUE; cr++) { in ext4_mb_regular_allocator()
2857 ac->ac_criteria = cr; in ext4_mb_regular_allocator()
2862 group = ac->ac_g_ex.fe_group; in ext4_mb_regular_allocator()
2863 ac->ac_groups_linear_remaining = sbi->s_mb_max_linear_groups; in ext4_mb_regular_allocator()
2867 ext4_mb_choose_next_group(ac, &new_cr, &group, ngroups)) { in ext4_mb_regular_allocator()
2887 nr = 1 << sbi->s_log_groups_per_flex; in ext4_mb_regular_allocator()
2888 nr -= group & (nr - 1); in ext4_mb_regular_allocator()
2896 ret = ext4_mb_good_group_nolock(ac, group, cr); in ext4_mb_regular_allocator()
2913 ret = ext4_mb_good_group(ac, group, cr); in ext4_mb_regular_allocator()
2920 ac->ac_groups_scanned++; in ext4_mb_regular_allocator()
2922 ext4_mb_simple_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2926 !(ac->ac_g_ex.fe_len % in ext4_mb_regular_allocator()
2928 ext4_mb_scan_aligned(ac, &e4b); in ext4_mb_regular_allocator()
2930 ext4_mb_complex_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2935 if (ac->ac_status != AC_STATUS_CONTINUE) in ext4_mb_regular_allocator()
2942 if (i == ngroups && ac->ac_criteria == CR_BEST_AVAIL_LEN) in ext4_mb_regular_allocator()
2945 ac->ac_g_ex.fe_len = ac->ac_orig_goal_len; in ext4_mb_regular_allocator()
2948 if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && in ext4_mb_regular_allocator()
2949 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_regular_allocator()
2954 ext4_mb_try_best_found(ac, &e4b); in ext4_mb_regular_allocator()
2955 if (ac->ac_status != AC_STATUS_FOUND) { in ext4_mb_regular_allocator()
2963 ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start, in ext4_mb_regular_allocator()
2964 ac->ac_b_ex.fe_len, lost); in ext4_mb_regular_allocator()
2966 ac->ac_b_ex.fe_group = 0; in ext4_mb_regular_allocator()
2967 ac->ac_b_ex.fe_start = 0; in ext4_mb_regular_allocator()
2968 ac->ac_b_ex.fe_len = 0; in ext4_mb_regular_allocator()
2969 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_regular_allocator()
2970 ac->ac_flags |= EXT4_MB_HINT_FIRST; in ext4_mb_regular_allocator()
2976 if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
2977 atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]); in ext4_mb_regular_allocator()
2979 if (!err && ac->ac_status != AC_STATUS_FOUND && first_err) in ext4_mb_regular_allocator()
2983 ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status, in ext4_mb_regular_allocator()
2984 ac->ac_flags, cr, err); in ext4_mb_regular_allocator()
2999 group = *pos + 1; in ext4_mb_seq_groups_start()
3011 group = *pos + 1; in ext4_mb_seq_groups_next()
3034 " 2^0 2^1 2^2 2^3 2^4 2^5 2^6 " in ext4_mb_seq_groups_show()
3050 buddy_loaded = 1; in ext4_mb_seq_groups_show()
3061 seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ? in ext4_mb_seq_groups_show()
3089 "\tTo enable, please write \"1\" to sysfs file mb_stats.\n"); in ext4_seq_mb_stats_show()
3194 position = *pos + 1; in ext4_mb_seq_structs_summary_start()
3206 position = *pos + 1; in ext4_mb_seq_structs_summary_next()
3237 test_opt2(sb, MB_OPTIMIZE_SCAN) ? 1 : 0); in ext4_mb_seq_structs_summary_show()
3282 size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> in ext4_mb_alloc_groupinfo()
3339 i = group & (EXT4_DESC_PER_BLOCK(sb) - 1); in ext4_mb_add_groupinfo()
3367 meta_group_info[i]->bb_largest_free_order = -1; /* uninit */ in ext4_mb_add_groupinfo()
3368 meta_group_info[i]->bb_avg_fragment_size_order = -1; /* uninit */ in ext4_mb_add_groupinfo()
3433 sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex, in ext4_mb_init_backend()
3565 fd->efd_start_cluster + fd->efd_count - 1, 1); in ext4_discard_work()
3606 i = 1; in ext4_mb_init()
3608 offset_incr = 1 << (sb->s_blocksize_bits - 1); in ext4_mb_init()
3614 offset_incr = offset_incr >> 1; in ext4_mb_init()
3615 max = max >> 1; in ext4_mb_init()
3674 * is 1 megabyte, then group preallocation size becomes half a in ext4_mb_init()
3679 * size is 256k, and 32 megs when the cluster size is 1 meg, in ext4_mb_init()
3685 * If there is a s_stripe > 1, then we set the s_mb_group_prealloc in ext4_mb_init()
3692 if (sbi->s_stripe > 1) { in ext4_mb_init()
3787 EXT4_DESC_PER_BLOCK(sb) - 1) >> in ext4_mb_release()
3898 mb_debug(sb, "freed %d blocks in 1 structures\n", count); in ext4_free_data_in_buddy()
3982 * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
3986 ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, in ext4_mb_mark_diskspace_used() argument
3997 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_mark_diskspace_used()
3998 BUG_ON(ac->ac_b_ex.fe_len <= 0); in ext4_mb_mark_diskspace_used()
4000 sb = ac->ac_sb; in ext4_mb_mark_diskspace_used()
4003 bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
4015 gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); in ext4_mb_mark_diskspace_used()
4019 ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, in ext4_mb_mark_diskspace_used()
4027 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_mark_diskspace_used()
4029 len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
4030 if (!ext4_inode_block_valid(ac->ac_inode, block, len)) { in ext4_mb_mark_diskspace_used()
4037 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
4038 mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
4039 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
4040 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
4047 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
4051 for (i = 0; i < ac->ac_b_ex.fe_len; i++) { in ext4_mb_mark_diskspace_used()
4052 BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, in ext4_mb_mark_diskspace_used()
4057 mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
4058 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
4064 ac->ac_b_ex.fe_group, gdp)); in ext4_mb_mark_diskspace_used()
4066 len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; in ext4_mb_mark_diskspace_used()
4069 ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp); in ext4_mb_mark_diskspace_used()
4071 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
4072 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
4076 if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) in ext4_mb_mark_diskspace_used()
4083 ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
4084 atomic64_sub(ac->ac_b_ex.fe_len, in ext4_mb_mark_diskspace_used()
4220 static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) in ext4_mb_normalize_group_request() argument
4222 struct super_block *sb = ac->ac_sb; in ext4_mb_normalize_group_request()
4223 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_normalize_group_request()
4226 ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; in ext4_mb_normalize_group_request()
4227 mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len); in ext4_mb_normalize_group_request()
4249 ext4_mb_pa_assert_overlap(struct ext4_allocation_context *ac, in ext4_mb_pa_assert_overlap() argument
4252 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_pa_assert_overlap()
4253 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_pa_assert_overlap()
4276 * Given an allocation context "ac" and a range "start", "end", check
4281 * ac allocation context
4286 ext4_mb_pa_adjust_overlap(struct ext4_allocation_context *ac, in ext4_mb_pa_adjust_overlap() argument
4289 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_pa_adjust_overlap()
4290 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_pa_adjust_overlap()
4293 ext4_lblk_t new_start, tmp_pa_start, right_pa_start = -1; in ext4_mb_pa_adjust_overlap()
4294 loff_t new_end, tmp_pa_end, left_pa_end = -1; in ext4_mb_pa_adjust_overlap()
4306 /* Step 1: find any one immediate neighboring PA of the normalized range */ in ext4_mb_pa_adjust_overlap()
4308 iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical, in ext4_mb_pa_adjust_overlap()
4318 BUG_ON(!(ac->ac_o_ex.fe_logical >= tmp_pa_end || in ext4_mb_pa_adjust_overlap()
4319 ac->ac_o_ex.fe_logical < tmp_pa_start)); in ext4_mb_pa_adjust_overlap()
4328 if (tmp_pa->pa_lstart < ac->ac_o_ex.fe_logical) { in ext4_mb_pa_adjust_overlap()
4394 BUG_ON(left_pa_end > ac->ac_o_ex.fe_logical); in ext4_mb_pa_adjust_overlap()
4399 BUG_ON(right_pa_start <= ac->ac_o_ex.fe_logical); in ext4_mb_pa_adjust_overlap()
4415 ext4_mb_pa_assert_overlap(ac, new_start, new_end); in ext4_mb_pa_adjust_overlap()
4426 ext4_mb_normalize_request(struct ext4_allocation_context *ac, in ext4_mb_normalize_request() argument
4429 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_normalize_request()
4438 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_normalize_request()
4442 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_normalize_request()
4447 if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) in ext4_mb_normalize_request()
4450 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { in ext4_mb_normalize_request()
4451 ext4_mb_normalize_group_request(ac); in ext4_mb_normalize_request()
4455 bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_normalize_request()
4459 size = extent_logical_end(sbi, &ac->ac_o_ex); in ext4_mb_normalize_request()
4461 if (size < i_size_read(ac->ac_inode)) in ext4_mb_normalize_request()
4462 size = i_size_read(ac->ac_inode); in ext4_mb_normalize_request()
4489 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
4493 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
4496 } else if (NRL_CHECK_SIZE(EXT4_C2B(sbi, ac->ac_o_ex.fe_len), in ext4_mb_normalize_request()
4498 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
4502 start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; in ext4_mb_normalize_request()
4504 ac->ac_o_ex.fe_len) << bsbits; in ext4_mb_normalize_request()
4515 start = max(start, rounddown(ac->ac_o_ex.fe_logical, in ext4_mb_normalize_request()
4516 (ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb))); in ext4_mb_normalize_request()
4524 size -= ar->lleft + 1 - start; in ext4_mb_normalize_request()
4525 start = ar->lleft + 1; in ext4_mb_normalize_request()
4527 if (ar->pright && start + size - 1 >= ar->lright) in ext4_mb_normalize_request()
4534 if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) in ext4_mb_normalize_request()
4535 size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb); in ext4_mb_normalize_request()
4539 ext4_mb_pa_adjust_overlap(ac, &start, &end); in ext4_mb_normalize_request()
4547 * ac->ac_o_ex.fe_logical & fe_len should always lie within "start" and in ext4_mb_normalize_request()
4558 if (start + size <= ac->ac_o_ex.fe_logical || in ext4_mb_normalize_request()
4559 start > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
4560 ext4_msg(ac->ac_sb, KERN_ERR, in ext4_mb_normalize_request()
4563 (unsigned long) ac->ac_o_ex.fe_logical); in ext4_mb_normalize_request()
4566 BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); in ext4_mb_normalize_request()
4572 ac->ac_g_ex.fe_logical = start; in ext4_mb_normalize_request()
4573 ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); in ext4_mb_normalize_request()
4574 ac->ac_orig_goal_len = ac->ac_g_ex.fe_len; in ext4_mb_normalize_request()
4581 ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, in ext4_mb_normalize_request()
4582 &ac->ac_g_ex.fe_group, in ext4_mb_normalize_request()
4583 &ac->ac_g_ex.fe_start); in ext4_mb_normalize_request()
4584 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
4586 if (ar->pleft && (ar->lleft + 1 == start) && in ext4_mb_normalize_request()
4587 ar->pleft + 1 < ext4_blocks_count(es)) { in ext4_mb_normalize_request()
4589 ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, in ext4_mb_normalize_request()
4590 &ac->ac_g_ex.fe_group, in ext4_mb_normalize_request()
4591 &ac->ac_g_ex.fe_start); in ext4_mb_normalize_request()
4592 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
4595 mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size, in ext4_mb_normalize_request()
4599 static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) in ext4_mb_collect_stats() argument
4601 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_collect_stats()
4603 if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) { in ext4_mb_collect_stats()
4605 atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); in ext4_mb_collect_stats()
4606 if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) in ext4_mb_collect_stats()
4609 atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); in ext4_mb_collect_stats()
4611 atomic_add(ac->ac_cX_found[i], &sbi->s_bal_cX_ex_scanned[i]); in ext4_mb_collect_stats()
4614 atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned); in ext4_mb_collect_stats()
4615 if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && in ext4_mb_collect_stats()
4616 ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) in ext4_mb_collect_stats()
4619 if (ac->ac_f_ex.fe_len == ac->ac_orig_goal_len) in ext4_mb_collect_stats()
4622 if (ac->ac_found > sbi->s_mb_max_to_scan) in ext4_mb_collect_stats()
4626 if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) in ext4_mb_collect_stats()
4627 trace_ext4_mballoc_alloc(ac); in ext4_mb_collect_stats()
4629 trace_ext4_mballoc_prealloc(ac); in ext4_mb_collect_stats()
4636 * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
4638 static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) in ext4_discard_allocated_blocks() argument
4640 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_discard_allocated_blocks()
4645 if (ac->ac_f_ex.fe_len == 0) in ext4_discard_allocated_blocks()
4647 err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b); in ext4_discard_allocated_blocks()
4656 ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
4657 mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start, in ext4_discard_allocated_blocks()
4658 ac->ac_f_ex.fe_len); in ext4_discard_allocated_blocks()
4659 ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
4665 pa->pa_free += ac->ac_b_ex.fe_len; in ext4_discard_allocated_blocks()
4673 static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, in ext4_mb_use_inode_pa() argument
4676 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_inode_pa()
4682 start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); in ext4_mb_use_inode_pa()
4684 start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); in ext4_mb_use_inode_pa()
4686 ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, in ext4_mb_use_inode_pa()
4687 &ac->ac_b_ex.fe_start); in ext4_mb_use_inode_pa()
4688 ac->ac_b_ex.fe_len = len; in ext4_mb_use_inode_pa()
4689 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_inode_pa()
4690 ac->ac_pa = pa; in ext4_mb_use_inode_pa()
4695 BUG_ON(ac->ac_b_ex.fe_len <= 0); in ext4_mb_use_inode_pa()
4698 mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa); in ext4_mb_use_inode_pa()
4704 static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, in ext4_mb_use_group_pa() argument
4707 unsigned int len = ac->ac_o_ex.fe_len; in ext4_mb_use_group_pa()
4709 ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, in ext4_mb_use_group_pa()
4710 &ac->ac_b_ex.fe_group, in ext4_mb_use_group_pa()
4711 &ac->ac_b_ex.fe_start); in ext4_mb_use_group_pa()
4712 ac->ac_b_ex.fe_len = len; in ext4_mb_use_group_pa()
4713 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_group_pa()
4714 ac->ac_pa = pa; in ext4_mb_use_group_pa()
4722 mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n", in ext4_mb_use_group_pa()
4759 ext4_mb_pa_goal_check(struct ext4_allocation_context *ac, in ext4_mb_pa_goal_check() argument
4762 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_pa_goal_check()
4765 if (likely(!(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))) in ext4_mb_pa_goal_check()
4775 (ac->ac_g_ex.fe_logical - pa->pa_lstart); in ext4_mb_pa_goal_check()
4776 if (ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex) != start) in ext4_mb_pa_goal_check()
4779 if (ac->ac_g_ex.fe_len > pa->pa_len - in ext4_mb_pa_goal_check()
4780 EXT4_B2C(sbi, ac->ac_g_ex.fe_logical - pa->pa_lstart)) in ext4_mb_pa_goal_check()
4790 ext4_mb_use_preallocated(struct ext4_allocation_context *ac) in ext4_mb_use_preallocated() argument
4792 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_preallocated()
4794 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_use_preallocated()
4801 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_use_preallocated()
4818 * Step 1: Find a pa with logical start immediately adjacent to the in ext4_mb_use_preallocated()
4824 iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical, in ext4_mb_use_preallocated()
4835 if (tmp_pa->pa_lstart > ac->ac_o_ex.fe_logical) { in ext4_mb_use_preallocated()
4852 BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical)); in ext4_mb_use_preallocated()
4884 BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical)); in ext4_mb_use_preallocated()
4885 BUG_ON(tmp_pa->pa_deleted == 1); in ext4_mb_use_preallocated()
4892 if (ac->ac_o_ex.fe_logical >= pa_logical_end(sbi, tmp_pa)) { in ext4_mb_use_preallocated()
4898 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && in ext4_mb_use_preallocated()
4909 if (tmp_pa->pa_free && likely(ext4_mb_pa_goal_check(ac, tmp_pa))) { in ext4_mb_use_preallocated()
4911 ext4_mb_use_inode_pa(ac, tmp_pa); in ext4_mb_use_preallocated()
4921 * 1. When a new inode pa is added to rbtree it must have in ext4_mb_use_preallocated()
4950 if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) in ext4_mb_use_preallocated()
4954 lg = ac->ac_lg; in ext4_mb_use_preallocated()
4957 order = fls(ac->ac_o_ex.fe_len) - 1; in ext4_mb_use_preallocated()
4958 if (order > PREALLOC_TB_SIZE - 1) in ext4_mb_use_preallocated()
4960 order = PREALLOC_TB_SIZE - 1; in ext4_mb_use_preallocated()
4962 goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); in ext4_mb_use_preallocated()
4973 tmp_pa->pa_free >= ac->ac_o_ex.fe_len) { in ext4_mb_use_preallocated()
4983 ext4_mb_use_group_pa(ac, cpa); in ext4_mb_use_preallocated()
5045 pa->pa_deleted = 1; in ext4_mb_mark_pa_deleted()
5073 static void ext4_mb_put_pa(struct ext4_allocation_context *ac, in ext4_mb_put_pa() argument
5078 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_put_pa()
5087 if (pa->pa_deleted == 1) { in ext4_mb_put_pa()
5165 ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) in ext4_mb_new_inode_pa() argument
5167 struct super_block *sb = ac->ac_sb; in ext4_mb_new_inode_pa()
5174 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
5175 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_inode_pa()
5176 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_inode_pa()
5177 BUG_ON(ac->ac_pa == NULL); in ext4_mb_new_inode_pa()
5179 pa = ac->ac_pa; in ext4_mb_new_inode_pa()
5181 if (ac->ac_b_ex.fe_len < ac->ac_orig_goal_len) { in ext4_mb_new_inode_pa()
5183 .fe_logical = ac->ac_g_ex.fe_logical, in ext4_mb_new_inode_pa()
5184 .fe_len = ac->ac_orig_goal_len, in ext4_mb_new_inode_pa()
5191 BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); in ext4_mb_new_inode_pa()
5192 BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); in ext4_mb_new_inode_pa()
5199 * 1. Check if best ex can be kept at end of goal (before in ext4_mb_new_inode_pa()
5205 ex.fe_len = ac->ac_b_ex.fe_len; in ext4_mb_new_inode_pa()
5208 if (ac->ac_o_ex.fe_logical >= ex.fe_logical) in ext4_mb_new_inode_pa()
5211 ex.fe_logical = ac->ac_g_ex.fe_logical; in ext4_mb_new_inode_pa()
5212 if (ac->ac_o_ex.fe_logical < extent_logical_end(sbi, &ex)) in ext4_mb_new_inode_pa()
5215 ex.fe_logical = ac->ac_o_ex.fe_logical; in ext4_mb_new_inode_pa()
5217 ac->ac_b_ex.fe_logical = ex.fe_logical; in ext4_mb_new_inode_pa()
5219 BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); in ext4_mb_new_inode_pa()
5220 BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
5224 pa->pa_lstart = ac->ac_b_ex.fe_logical; in ext4_mb_new_inode_pa()
5225 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_inode_pa()
5226 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_inode_pa()
5235 trace_ext4_mb_new_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
5238 ext4_mb_use_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
5240 ei = EXT4_I(ac->ac_inode); in ext4_mb_new_inode_pa()
5241 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_inode_pa()
5246 pa->pa_inode = ac->ac_inode; in ext4_mb_new_inode_pa()
5260 ext4_mb_new_group_pa(struct ext4_allocation_context *ac) in ext4_mb_new_group_pa() argument
5262 struct super_block *sb = ac->ac_sb; in ext4_mb_new_group_pa()
5268 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_group_pa()
5269 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_group_pa()
5270 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_group_pa()
5271 BUG_ON(ac->ac_pa == NULL); in ext4_mb_new_group_pa()
5273 pa = ac->ac_pa; in ext4_mb_new_group_pa()
5275 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_group_pa()
5277 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_group_pa()
5287 trace_ext4_mb_new_group_pa(ac, pa); in ext4_mb_new_group_pa()
5289 ext4_mb_use_group_pa(ac, pa); in ext4_mb_new_group_pa()
5292 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_group_pa()
5295 lg = ac->ac_lg; in ext4_mb_new_group_pa()
5309 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac) in ext4_mb_new_preallocation() argument
5311 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_new_preallocation()
5312 ext4_mb_new_group_pa(ac); in ext4_mb_new_preallocation()
5314 ext4_mb_new_inode_pa(ac); in ext4_mb_new_preallocation()
5359 bit = next + 1; in ext4_mb_release_inode_pa()
5407 * 1) ENOSPC
5409 * 1) how many requested
5453 *busy = 1; in ext4_mb_discard_group_preallocations()
5564 WARN_ON(1); in ext4_discard_preallocations()
5633 static int ext4_mb_pa_alloc(struct ext4_allocation_context *ac) in ext4_mb_pa_alloc() argument
5641 atomic_set(&pa->pa_count, 1); in ext4_mb_pa_alloc()
5642 ac->ac_pa = pa; in ext4_mb_pa_alloc()
5646 static void ext4_mb_pa_put_free(struct ext4_allocation_context *ac) in ext4_mb_pa_put_free() argument
5648 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_pa_put_free()
5651 ac->ac_pa = NULL; in ext4_mb_pa_put_free()
5658 pa->pa_deleted = 1; in ext4_mb_pa_put_free()
5697 static void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
5699 struct super_block *sb = ac->ac_sb; in ext4_mb_show_ac()
5707 ac->ac_status, ac->ac_flags); in ext4_mb_show_ac()
5711 (unsigned long)ac->ac_o_ex.fe_group, in ext4_mb_show_ac()
5712 (unsigned long)ac->ac_o_ex.fe_start, in ext4_mb_show_ac()
5713 (unsigned long)ac->ac_o_ex.fe_len, in ext4_mb_show_ac()
5714 (unsigned long)ac->ac_o_ex.fe_logical, in ext4_mb_show_ac()
5715 (unsigned long)ac->ac_g_ex.fe_group, in ext4_mb_show_ac()
5716 (unsigned long)ac->ac_g_ex.fe_start, in ext4_mb_show_ac()
5717 (unsigned long)ac->ac_g_ex.fe_len, in ext4_mb_show_ac()
5718 (unsigned long)ac->ac_g_ex.fe_logical, in ext4_mb_show_ac()
5719 (unsigned long)ac->ac_b_ex.fe_group, in ext4_mb_show_ac()
5720 (unsigned long)ac->ac_b_ex.fe_start, in ext4_mb_show_ac()
5721 (unsigned long)ac->ac_b_ex.fe_len, in ext4_mb_show_ac()
5722 (unsigned long)ac->ac_b_ex.fe_logical, in ext4_mb_show_ac()
5723 (int)ac->ac_criteria); in ext4_mb_show_ac()
5724 mb_debug(sb, "%u found", ac->ac_found); in ext4_mb_show_ac()
5725 mb_debug(sb, "used pa: %s, ", ac->ac_pa ? "yes" : "no"); in ext4_mb_show_ac()
5726 if (ac->ac_pa) in ext4_mb_show_ac()
5727 mb_debug(sb, "pa_type %s\n", ac->ac_pa->pa_type == MB_GROUP_PA ? in ext4_mb_show_ac()
5735 static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
5737 ext4_mb_show_pa(ac->ac_sb); in ext4_mb_show_ac()
5748 static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) in ext4_mb_group_or_file() argument
5750 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_group_or_file()
5751 int bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_group_or_file()
5755 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_group_or_file()
5758 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_group_or_file()
5763 size = extent_logical_end(sbi, &ac->ac_o_ex); in ext4_mb_group_or_file()
5764 isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) in ext4_mb_group_or_file()
5769 !inode_is_open_for_write(ac->ac_inode)) in ext4_mb_group_or_file()
5779 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
5781 ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; in ext4_mb_group_or_file()
5785 BUG_ON(ac->ac_lg != NULL); in ext4_mb_group_or_file()
5791 ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); in ext4_mb_group_or_file()
5794 ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; in ext4_mb_group_or_file()
5797 mutex_lock(&ac->ac_lg->lg_mutex); in ext4_mb_group_or_file()
5801 ext4_mb_initialize_context(struct ext4_allocation_context *ac, in ext4_mb_initialize_context() argument
5827 ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); in ext4_mb_initialize_context()
5828 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_initialize_context()
5829 ac->ac_sb = sb; in ext4_mb_initialize_context()
5830 ac->ac_inode = ar->inode; in ext4_mb_initialize_context()
5831 ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; in ext4_mb_initialize_context()
5832 ac->ac_o_ex.fe_group = group; in ext4_mb_initialize_context()
5833 ac->ac_o_ex.fe_start = block; in ext4_mb_initialize_context()
5834 ac->ac_o_ex.fe_len = len; in ext4_mb_initialize_context()
5835 ac->ac_g_ex = ac->ac_o_ex; in ext4_mb_initialize_context()
5836 ac->ac_orig_goal_len = ac->ac_g_ex.fe_len; in ext4_mb_initialize_context()
5837 ac->ac_flags = ar->flags; in ext4_mb_initialize_context()
5841 ext4_mb_group_or_file(ac); in ext4_mb_initialize_context()
5843 mb_debug(sb, "init ac: %u blocks @ %u, goal %u, flags 0x%x, 2^%d, " in ext4_mb_initialize_context()
5846 (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, in ext4_mb_initialize_context()
5936 static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) in ext4_mb_add_n_trim() argument
5938 int order, added = 0, lg_prealloc_count = 1; in ext4_mb_add_n_trim()
5939 struct super_block *sb = ac->ac_sb; in ext4_mb_add_n_trim()
5940 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_add_n_trim()
5941 struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; in ext4_mb_add_n_trim()
5943 order = fls(pa->pa_free) - 1; in ext4_mb_add_n_trim()
5944 if (order > PREALLOC_TB_SIZE - 1) in ext4_mb_add_n_trim()
5946 order = PREALLOC_TB_SIZE - 1; in ext4_mb_add_n_trim()
5961 added = 1; in ext4_mb_add_n_trim()
5984 static int ext4_mb_release_context(struct ext4_allocation_context *ac) in ext4_mb_release_context() argument
5986 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_release_context()
5987 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_release_context()
5992 pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
5993 pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
5994 pa->pa_free -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
5995 pa->pa_len -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
6008 ext4_mb_add_n_trim(ac); in ext4_mb_release_context()
6012 ext4_mb_put_pa(ac, ac->ac_sb, pa); in ext4_mb_release_context()
6014 if (ac->ac_bitmap_page) in ext4_mb_release_context()
6015 put_page(ac->ac_bitmap_page); in ext4_mb_release_context()
6016 if (ac->ac_buddy_page) in ext4_mb_release_context()
6017 put_page(ac->ac_buddy_page); in ext4_mb_release_context()
6018 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_release_context()
6019 mutex_unlock(&ac->ac_lg->lg_mutex); in ext4_mb_release_context()
6020 ext4_mb_collect_stats(ac); in ext4_mb_release_context()
6034 needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1; in ext4_mb_discard_preallocations()
6052 struct ext4_allocation_context *ac, u64 *seq) in ext4_mb_discard_preallocations_should_retry() argument
6058 freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); in ext4_mb_discard_preallocations_should_retry()
6064 if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) { in ext4_mb_discard_preallocations_should_retry()
6065 ac->ac_flags |= EXT4_MB_STRICT_CHECK; in ext4_mb_discard_preallocations_should_retry()
6108 while (1) { in ext4_mb_new_blocks_simple()
6116 blkoff = i + 1; in ext4_mb_new_blocks_simple()
6136 ext4_mb_mark_bb(sb, block, 1, 1); in ext4_mb_new_blocks_simple()
6137 ar->len = 1; in ext4_mb_new_blocks_simple()
6150 struct ext4_allocation_context *ac = NULL; in ext4_mb_new_blocks() local
6181 ar->len = ar->len >> 1; in ext4_mb_new_blocks()
6208 ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS); in ext4_mb_new_blocks()
6209 if (!ac) { in ext4_mb_new_blocks()
6215 ext4_mb_initialize_context(ac, ar); in ext4_mb_new_blocks()
6217 ac->ac_op = EXT4_MB_HISTORY_PREALLOC; in ext4_mb_new_blocks()
6219 if (!ext4_mb_use_preallocated(ac)) { in ext4_mb_new_blocks()
6220 ac->ac_op = EXT4_MB_HISTORY_ALLOC; in ext4_mb_new_blocks()
6221 ext4_mb_normalize_request(ac, ar); in ext4_mb_new_blocks()
6223 *errp = ext4_mb_pa_alloc(ac); in ext4_mb_new_blocks()
6228 *errp = ext4_mb_regular_allocator(ac); in ext4_mb_new_blocks()
6232 * ac->ac_status == AC_STATUS_FOUND. in ext4_mb_new_blocks()
6233 * And error from above mean ac->ac_status != AC_STATUS_FOUND in ext4_mb_new_blocks()
6237 ext4_mb_pa_put_free(ac); in ext4_mb_new_blocks()
6238 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
6241 if (ac->ac_status == AC_STATUS_FOUND && in ext4_mb_new_blocks()
6242 ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len) in ext4_mb_new_blocks()
6243 ext4_mb_pa_put_free(ac); in ext4_mb_new_blocks()
6245 if (likely(ac->ac_status == AC_STATUS_FOUND)) { in ext4_mb_new_blocks()
6246 *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); in ext4_mb_new_blocks()
6248 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
6251 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_blocks()
6252 ar->len = ac->ac_b_ex.fe_len; in ext4_mb_new_blocks()
6256 ext4_mb_discard_preallocations_should_retry(sb, ac, &seq)) in ext4_mb_new_blocks()
6262 ext4_mb_pa_put_free(ac); in ext4_mb_new_blocks()
6268 ac->ac_b_ex.fe_len = 0; in ext4_mb_new_blocks()
6270 ext4_mb_show_ac(ac); in ext4_mb_new_blocks()
6272 ext4_mb_release_context(ac); in ext4_mb_new_blocks()
6273 kmem_cache_free(ext4_ac_cachep, ac); in ext4_mb_new_blocks()
6679 BUG_ON(count > 1); in ext4_free_blocks()
6759 ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1); in ext4_group_add_blocks()
6760 unsigned long cluster_count = last_cluster - first_cluster + 1; in ext4_group_add_blocks()
6762 ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1); in ext4_group_add_blocks()
6918 if (grp < (ext4_get_groups_count(sb) - 1)) in ext4_last_grp_cluster()
6925 return nr_clusters_in_group - 1; in ext4_last_grp_cluster()
6953 start = mb_find_next_zero_bit(bitmap, max + 1, start); in ext4_try_to_trim_range()
6957 next = mb_find_next_bit(bitmap, last + 1, start); in ext4_try_to_trim_range()
6969 start = next + 1; in ext4_try_to_trim_range()
7062 end = start + (range->len >> sb->s_blocksize_bits) - 1; in ext4_trim_fs()
7077 if (end >= max_blks - 1) in ext4_trim_fs()
7078 end = max_blks - 1; in ext4_trim_fs()
7091 end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; in ext4_trim_fs()
7108 * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to in ext4_trim_fs()
7163 end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; in ext4_mballoc_query_range()
7166 start = mb_find_next_zero_bit(bitmap, end + 1, start); in ext4_mballoc_query_range()
7169 next = mb_find_next_bit(bitmap, end + 1, start); in ext4_mballoc_query_range()
7177 start = next + 1; in ext4_mballoc_query_range()