Lines Matching refs:ac
1641 static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, in ext4_mb_use_best_found() argument
1644 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_best_found()
1647 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); in ext4_mb_use_best_found()
1648 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_use_best_found()
1650 ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); in ext4_mb_use_best_found()
1651 ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; in ext4_mb_use_best_found()
1652 ret = mb_mark_used(e4b, &ac->ac_b_ex); in ext4_mb_use_best_found()
1656 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_use_best_found()
1658 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_best_found()
1659 ac->ac_tail = ret & 0xffff; in ext4_mb_use_best_found()
1660 ac->ac_buddy = ret >> 16; in ext4_mb_use_best_found()
1669 ac->ac_bitmap_page = e4b->bd_bitmap_page; in ext4_mb_use_best_found()
1670 get_page(ac->ac_bitmap_page); in ext4_mb_use_best_found()
1671 ac->ac_buddy_page = e4b->bd_buddy_page; in ext4_mb_use_best_found()
1672 get_page(ac->ac_buddy_page); in ext4_mb_use_best_found()
1674 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_use_best_found()
1676 sbi->s_mb_last_group = ac->ac_f_ex.fe_group; in ext4_mb_use_best_found()
1677 sbi->s_mb_last_start = ac->ac_f_ex.fe_start; in ext4_mb_use_best_found()
1686 static void ext4_mb_check_limits(struct ext4_allocation_context *ac, in ext4_mb_check_limits() argument
1690 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_check_limits()
1691 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_check_limits()
1692 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_check_limits()
1696 if (ac->ac_status == AC_STATUS_FOUND) in ext4_mb_check_limits()
1701 if (ac->ac_found > sbi->s_mb_max_to_scan && in ext4_mb_check_limits()
1702 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_check_limits()
1703 ac->ac_status = AC_STATUS_BREAK; in ext4_mb_check_limits()
1713 if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) in ext4_mb_check_limits()
1720 ext4_mb_use_best_found(ac, e4b); in ext4_mb_check_limits()
1736 static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, in ext4_mb_measure_extent() argument
1740 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_measure_extent()
1741 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_measure_extent()
1744 BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
1745 BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
1746 BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); in ext4_mb_measure_extent()
1748 ac->ac_found++; in ext4_mb_measure_extent()
1753 if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_measure_extent()
1755 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
1764 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
1792 ext4_mb_check_limits(ac, e4b, 0); in ext4_mb_measure_extent()
1796 int ext4_mb_try_best_found(struct ext4_allocation_context *ac, in ext4_mb_try_best_found() argument
1799 struct ext4_free_extent ex = ac->ac_b_ex; in ext4_mb_try_best_found()
1805 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_try_best_found()
1809 ext4_lock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
1813 ac->ac_b_ex = ex; in ext4_mb_try_best_found()
1814 ext4_mb_use_best_found(ac, e4b); in ext4_mb_try_best_found()
1817 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
1824 int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, in ext4_mb_find_by_goal() argument
1827 ext4_group_t group = ac->ac_g_ex.fe_group; in ext4_mb_find_by_goal()
1830 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_find_by_goal()
1831 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_find_by_goal()
1834 if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL)) in ext4_mb_find_by_goal()
1839 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_find_by_goal()
1848 ext4_lock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
1849 max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, in ext4_mb_find_by_goal()
1850 ac->ac_g_ex.fe_len, &ex); in ext4_mb_find_by_goal()
1853 if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { in ext4_mb_find_by_goal()
1856 start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) + in ext4_mb_find_by_goal()
1860 ac->ac_found++; in ext4_mb_find_by_goal()
1861 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1862 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
1864 } else if (max >= ac->ac_g_ex.fe_len) { in ext4_mb_find_by_goal()
1866 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
1867 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
1868 ac->ac_found++; in ext4_mb_find_by_goal()
1869 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1870 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
1871 } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { in ext4_mb_find_by_goal()
1875 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
1876 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
1877 ac->ac_found++; in ext4_mb_find_by_goal()
1878 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1879 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
1881 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
1892 void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, in ext4_mb_simple_scan_group() argument
1895 struct super_block *sb = ac->ac_sb; in ext4_mb_simple_scan_group()
1902 BUG_ON(ac->ac_2order <= 0); in ext4_mb_simple_scan_group()
1903 for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) { in ext4_mb_simple_scan_group()
1913 ac->ac_found++; in ext4_mb_simple_scan_group()
1915 ac->ac_b_ex.fe_len = 1 << i; in ext4_mb_simple_scan_group()
1916 ac->ac_b_ex.fe_start = k << i; in ext4_mb_simple_scan_group()
1917 ac->ac_b_ex.fe_group = e4b->bd_group; in ext4_mb_simple_scan_group()
1919 ext4_mb_use_best_found(ac, e4b); in ext4_mb_simple_scan_group()
1921 BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len); in ext4_mb_simple_scan_group()
1936 void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, in ext4_mb_complex_scan_group() argument
1939 struct super_block *sb = ac->ac_sb; in ext4_mb_complex_scan_group()
1950 while (free && ac->ac_status == AC_STATUS_CONTINUE) { in ext4_mb_complex_scan_group()
1966 mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); in ext4_mb_complex_scan_group()
1981 ext4_mb_measure_extent(ac, &ex, e4b); in ext4_mb_complex_scan_group()
1987 ext4_mb_check_limits(ac, e4b, 1); in ext4_mb_complex_scan_group()
1995 void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, in ext4_mb_scan_aligned() argument
1998 struct super_block *sb = ac->ac_sb; in ext4_mb_scan_aligned()
2020 ac->ac_found++; in ext4_mb_scan_aligned()
2022 ac->ac_b_ex = ex; in ext4_mb_scan_aligned()
2023 ext4_mb_use_best_found(ac, e4b); in ext4_mb_scan_aligned()
2037 static int ext4_mb_good_group(struct ext4_allocation_context *ac, in ext4_mb_good_group() argument
2041 int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); in ext4_mb_good_group()
2042 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group()
2049 if (cr <= 2 && free < ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2057 int ret = ext4_mb_init_group(ac->ac_sb, group, GFP_NOFS); in ext4_mb_good_group()
2068 BUG_ON(ac->ac_2order == 0); in ext4_mb_good_group()
2071 if ((ac->ac_flags & EXT4_MB_HINT_DATA) && in ext4_mb_good_group()
2076 if ((ac->ac_2order > ac->ac_sb->s_blocksize_bits+1) || in ext4_mb_good_group()
2077 (free / fragments) >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2080 if (grp->bb_largest_free_order < ac->ac_2order) in ext4_mb_good_group()
2085 if ((free / fragments) >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2089 if (free >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2102 ext4_mb_regular_allocator(struct ext4_allocation_context *ac) in ext4_mb_regular_allocator() argument
2111 sb = ac->ac_sb; in ext4_mb_regular_allocator()
2115 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) in ext4_mb_regular_allocator()
2118 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_regular_allocator()
2121 err = ext4_mb_find_by_goal(ac, &e4b); in ext4_mb_regular_allocator()
2122 if (err || ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
2125 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_regular_allocator()
2133 i = fls(ac->ac_g_ex.fe_len); in ext4_mb_regular_allocator()
2134 ac->ac_2order = 0; in ext4_mb_regular_allocator()
2146 if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) in ext4_mb_regular_allocator()
2147 ac->ac_2order = i - 1; in ext4_mb_regular_allocator()
2151 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_regular_allocator()
2154 ac->ac_g_ex.fe_group = sbi->s_mb_last_group; in ext4_mb_regular_allocator()
2155 ac->ac_g_ex.fe_start = sbi->s_mb_last_start; in ext4_mb_regular_allocator()
2160 cr = ac->ac_2order ? 0 : 1; in ext4_mb_regular_allocator()
2166 for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { in ext4_mb_regular_allocator()
2167 ac->ac_criteria = cr; in ext4_mb_regular_allocator()
2172 group = ac->ac_g_ex.fe_group; in ext4_mb_regular_allocator()
2185 ret = ext4_mb_good_group(ac, group, cr); in ext4_mb_regular_allocator()
2202 ret = ext4_mb_good_group(ac, group, cr); in ext4_mb_regular_allocator()
2211 ac->ac_groups_scanned++; in ext4_mb_regular_allocator()
2213 ext4_mb_simple_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2215 !(ac->ac_g_ex.fe_len % sbi->s_stripe)) in ext4_mb_regular_allocator()
2216 ext4_mb_scan_aligned(ac, &e4b); in ext4_mb_regular_allocator()
2218 ext4_mb_complex_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2223 if (ac->ac_status != AC_STATUS_CONTINUE) in ext4_mb_regular_allocator()
2228 if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && in ext4_mb_regular_allocator()
2229 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_regular_allocator()
2235 ext4_mb_try_best_found(ac, &e4b); in ext4_mb_regular_allocator()
2236 if (ac->ac_status != AC_STATUS_FOUND) { in ext4_mb_regular_allocator()
2243 ac->ac_b_ex.fe_group = 0; in ext4_mb_regular_allocator()
2244 ac->ac_b_ex.fe_start = 0; in ext4_mb_regular_allocator()
2245 ac->ac_b_ex.fe_len = 0; in ext4_mb_regular_allocator()
2246 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_regular_allocator()
2247 ac->ac_flags |= EXT4_MB_HINT_FIRST; in ext4_mb_regular_allocator()
2254 if (!err && ac->ac_status != AC_STATUS_FOUND && first_err) in ext4_mb_regular_allocator()
2899 ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, in ext4_mb_mark_diskspace_used() argument
2910 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_mark_diskspace_used()
2911 BUG_ON(ac->ac_b_ex.fe_len <= 0); in ext4_mb_mark_diskspace_used()
2913 sb = ac->ac_sb; in ext4_mb_mark_diskspace_used()
2916 bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
2929 gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); in ext4_mb_mark_diskspace_used()
2933 ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, in ext4_mb_mark_diskspace_used()
2941 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_mark_diskspace_used()
2943 len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
2951 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
2952 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
2953 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
2954 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
2961 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
2965 for (i = 0; i < ac->ac_b_ex.fe_len; i++) { in ext4_mb_mark_diskspace_used()
2966 BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, in ext4_mb_mark_diskspace_used()
2971 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
2972 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
2977 ac->ac_b_ex.fe_group, gdp)); in ext4_mb_mark_diskspace_used()
2979 len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; in ext4_mb_mark_diskspace_used()
2981 ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh); in ext4_mb_mark_diskspace_used()
2982 ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp); in ext4_mb_mark_diskspace_used()
2984 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
2985 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
2989 if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) in ext4_mb_mark_diskspace_used()
2996 ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
2997 atomic64_sub(ac->ac_b_ex.fe_len, in ext4_mb_mark_diskspace_used()
3020 static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) in ext4_mb_normalize_group_request() argument
3022 struct super_block *sb = ac->ac_sb; in ext4_mb_normalize_group_request()
3023 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_normalize_group_request()
3026 ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; in ext4_mb_normalize_group_request()
3028 current->pid, ac->ac_g_ex.fe_len); in ext4_mb_normalize_group_request()
3036 ext4_mb_normalize_request(struct ext4_allocation_context *ac, in ext4_mb_normalize_request() argument
3039 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_normalize_request()
3045 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_normalize_request()
3050 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_normalize_request()
3054 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_normalize_request()
3059 if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) in ext4_mb_normalize_request()
3062 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { in ext4_mb_normalize_request()
3063 ext4_mb_normalize_group_request(ac); in ext4_mb_normalize_request()
3067 bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_normalize_request()
3071 size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); in ext4_mb_normalize_request()
3073 if (size < i_size_read(ac->ac_inode)) in ext4_mb_normalize_request()
3074 size = i_size_read(ac->ac_inode); in ext4_mb_normalize_request()
3101 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
3105 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
3108 } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, in ext4_mb_normalize_request()
3110 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
3114 start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; in ext4_mb_normalize_request()
3115 size = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
3116 ac->ac_o_ex.fe_len) << bsbits; in ext4_mb_normalize_request()
3133 if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) in ext4_mb_normalize_request()
3134 size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb); in ext4_mb_normalize_request()
3151 pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
3155 BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || in ext4_mb_normalize_request()
3156 ac->ac_o_ex.fe_logical < pa->pa_lstart)); in ext4_mb_normalize_request()
3166 if (pa_end <= ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3169 } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3185 pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
3193 if (start + size <= ac->ac_o_ex.fe_logical && in ext4_mb_normalize_request()
3194 start > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3195 ext4_msg(ac->ac_sb, KERN_ERR, in ext4_mb_normalize_request()
3198 (unsigned long) ac->ac_o_ex.fe_logical); in ext4_mb_normalize_request()
3201 BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); in ext4_mb_normalize_request()
3207 ac->ac_g_ex.fe_logical = start; in ext4_mb_normalize_request()
3208 ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); in ext4_mb_normalize_request()
3213 ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, in ext4_mb_normalize_request()
3214 &ac->ac_f_ex.fe_group, in ext4_mb_normalize_request()
3215 &ac->ac_f_ex.fe_start); in ext4_mb_normalize_request()
3216 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
3220 ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, in ext4_mb_normalize_request()
3221 &ac->ac_f_ex.fe_group, in ext4_mb_normalize_request()
3222 &ac->ac_f_ex.fe_start); in ext4_mb_normalize_request()
3223 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
3230 static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) in ext4_mb_collect_stats() argument
3232 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_collect_stats()
3234 if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) { in ext4_mb_collect_stats()
3236 atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); in ext4_mb_collect_stats()
3237 if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) in ext4_mb_collect_stats()
3239 atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); in ext4_mb_collect_stats()
3240 if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && in ext4_mb_collect_stats()
3241 ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) in ext4_mb_collect_stats()
3243 if (ac->ac_found > sbi->s_mb_max_to_scan) in ext4_mb_collect_stats()
3247 if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) in ext4_mb_collect_stats()
3248 trace_ext4_mballoc_alloc(ac); in ext4_mb_collect_stats()
3250 trace_ext4_mballoc_prealloc(ac); in ext4_mb_collect_stats()
3259 static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) in ext4_discard_allocated_blocks() argument
3261 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_discard_allocated_blocks()
3266 if (ac->ac_f_ex.fe_len == 0) in ext4_discard_allocated_blocks()
3268 err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b); in ext4_discard_allocated_blocks()
3278 ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
3279 mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start, in ext4_discard_allocated_blocks()
3280 ac->ac_f_ex.fe_len); in ext4_discard_allocated_blocks()
3281 ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
3286 pa->pa_free += ac->ac_b_ex.fe_len; in ext4_discard_allocated_blocks()
3292 static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, in ext4_mb_use_inode_pa() argument
3295 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_inode_pa()
3301 start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); in ext4_mb_use_inode_pa()
3303 start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); in ext4_mb_use_inode_pa()
3305 ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, in ext4_mb_use_inode_pa()
3306 &ac->ac_b_ex.fe_start); in ext4_mb_use_inode_pa()
3307 ac->ac_b_ex.fe_len = len; in ext4_mb_use_inode_pa()
3308 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_inode_pa()
3309 ac->ac_pa = pa; in ext4_mb_use_inode_pa()
3322 static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, in ext4_mb_use_group_pa() argument
3325 unsigned int len = ac->ac_o_ex.fe_len; in ext4_mb_use_group_pa()
3327 ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, in ext4_mb_use_group_pa()
3328 &ac->ac_b_ex.fe_group, in ext4_mb_use_group_pa()
3329 &ac->ac_b_ex.fe_start); in ext4_mb_use_group_pa()
3330 ac->ac_b_ex.fe_len = len; in ext4_mb_use_group_pa()
3331 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_group_pa()
3332 ac->ac_pa = pa; in ext4_mb_use_group_pa()
3376 ext4_mb_use_preallocated(struct ext4_allocation_context *ac) in ext4_mb_use_preallocated() argument
3378 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_preallocated()
3380 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_use_preallocated()
3386 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_use_preallocated()
3395 if (ac->ac_o_ex.fe_logical < pa->pa_lstart || in ext4_mb_use_preallocated()
3396 ac->ac_o_ex.fe_logical >= (pa->pa_lstart + in ext4_mb_use_preallocated()
3401 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && in ext4_mb_use_preallocated()
3410 ext4_mb_use_inode_pa(ac, pa); in ext4_mb_use_preallocated()
3412 ac->ac_criteria = 10; in ext4_mb_use_preallocated()
3421 if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) in ext4_mb_use_preallocated()
3425 lg = ac->ac_lg; in ext4_mb_use_preallocated()
3428 order = fls(ac->ac_o_ex.fe_len) - 1; in ext4_mb_use_preallocated()
3433 goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); in ext4_mb_use_preallocated()
3444 pa->pa_free >= ac->ac_o_ex.fe_len) { in ext4_mb_use_preallocated()
3454 ext4_mb_use_group_pa(ac, cpa); in ext4_mb_use_preallocated()
3455 ac->ac_criteria = 20; in ext4_mb_use_preallocated()
3540 static void ext4_mb_put_pa(struct ext4_allocation_context *ac, in ext4_mb_put_pa() argument
3600 ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) in ext4_mb_new_inode_pa() argument
3602 struct super_block *sb = ac->ac_sb; in ext4_mb_new_inode_pa()
3609 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
3610 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_inode_pa()
3611 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_inode_pa()
3617 if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { in ext4_mb_new_inode_pa()
3626 BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); in ext4_mb_new_inode_pa()
3627 BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); in ext4_mb_new_inode_pa()
3632 winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical; in ext4_mb_new_inode_pa()
3635 wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len); in ext4_mb_new_inode_pa()
3640 offs = ac->ac_o_ex.fe_logical % in ext4_mb_new_inode_pa()
3641 EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
3645 ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical - in ext4_mb_new_inode_pa()
3647 BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); in ext4_mb_new_inode_pa()
3648 BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
3653 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_new_inode_pa()
3655 pa->pa_lstart = ac->ac_b_ex.fe_logical; in ext4_mb_new_inode_pa()
3656 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_inode_pa()
3657 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_inode_pa()
3668 trace_ext4_mb_new_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
3670 ext4_mb_use_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
3673 ei = EXT4_I(ac->ac_inode); in ext4_mb_new_inode_pa()
3674 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_inode_pa()
3677 pa->pa_inode = ac->ac_inode; in ext4_mb_new_inode_pa()
3679 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_inode_pa()
3681 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_inode_pa()
3694 ext4_mb_new_group_pa(struct ext4_allocation_context *ac) in ext4_mb_new_group_pa() argument
3696 struct super_block *sb = ac->ac_sb; in ext4_mb_new_group_pa()
3702 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_group_pa()
3703 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_group_pa()
3704 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_group_pa()
3713 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_new_group_pa()
3715 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_group_pa()
3717 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_group_pa()
3728 trace_ext4_mb_new_group_pa(ac, pa); in ext4_mb_new_group_pa()
3730 ext4_mb_use_group_pa(ac, pa); in ext4_mb_new_group_pa()
3733 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_group_pa()
3734 lg = ac->ac_lg; in ext4_mb_new_group_pa()
3740 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_group_pa()
3742 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_group_pa()
3751 static int ext4_mb_new_preallocation(struct ext4_allocation_context *ac) in ext4_mb_new_preallocation() argument
3755 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_new_preallocation()
3756 err = ext4_mb_new_group_pa(ac); in ext4_mb_new_preallocation()
3758 err = ext4_mb_new_inode_pa(ac); in ext4_mb_new_preallocation()
4070 static void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
4072 struct super_block *sb = ac->ac_sb; in ext4_mb_show_ac()
4079 ext4_msg(ac->ac_sb, KERN_ERR, "Can't allocate:" in ext4_mb_show_ac()
4081 ext4_msg(ac->ac_sb, KERN_ERR, "status %d flags %d", in ext4_mb_show_ac()
4082 ac->ac_status, ac->ac_flags); in ext4_mb_show_ac()
4083 ext4_msg(ac->ac_sb, KERN_ERR, "orig %lu/%lu/%lu@%lu, " in ext4_mb_show_ac()
4086 (unsigned long)ac->ac_o_ex.fe_group, in ext4_mb_show_ac()
4087 (unsigned long)ac->ac_o_ex.fe_start, in ext4_mb_show_ac()
4088 (unsigned long)ac->ac_o_ex.fe_len, in ext4_mb_show_ac()
4089 (unsigned long)ac->ac_o_ex.fe_logical, in ext4_mb_show_ac()
4090 (unsigned long)ac->ac_g_ex.fe_group, in ext4_mb_show_ac()
4091 (unsigned long)ac->ac_g_ex.fe_start, in ext4_mb_show_ac()
4092 (unsigned long)ac->ac_g_ex.fe_len, in ext4_mb_show_ac()
4093 (unsigned long)ac->ac_g_ex.fe_logical, in ext4_mb_show_ac()
4094 (unsigned long)ac->ac_b_ex.fe_group, in ext4_mb_show_ac()
4095 (unsigned long)ac->ac_b_ex.fe_start, in ext4_mb_show_ac()
4096 (unsigned long)ac->ac_b_ex.fe_len, in ext4_mb_show_ac()
4097 (unsigned long)ac->ac_b_ex.fe_logical, in ext4_mb_show_ac()
4098 (int)ac->ac_criteria); in ext4_mb_show_ac()
4099 ext4_msg(ac->ac_sb, KERN_ERR, "%d found", ac->ac_found); in ext4_mb_show_ac()
4100 ext4_msg(ac->ac_sb, KERN_ERR, "groups: "); in ext4_mb_show_ac()
4128 static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
4141 static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) in ext4_mb_group_or_file() argument
4143 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_group_or_file()
4144 int bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_group_or_file()
4147 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_group_or_file()
4150 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_group_or_file()
4153 size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); in ext4_mb_group_or_file()
4154 isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) in ext4_mb_group_or_file()
4159 (atomic_read(&ac->ac_inode->i_writecount) == 0)) { in ext4_mb_group_or_file()
4160 ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; in ext4_mb_group_or_file()
4165 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
4172 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
4176 BUG_ON(ac->ac_lg != NULL); in ext4_mb_group_or_file()
4182 ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); in ext4_mb_group_or_file()
4185 ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; in ext4_mb_group_or_file()
4188 mutex_lock(&ac->ac_lg->lg_mutex); in ext4_mb_group_or_file()
4192 ext4_mb_initialize_context(struct ext4_allocation_context *ac, in ext4_mb_initialize_context() argument
4218 ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); in ext4_mb_initialize_context()
4219 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_initialize_context()
4220 ac->ac_sb = sb; in ext4_mb_initialize_context()
4221 ac->ac_inode = ar->inode; in ext4_mb_initialize_context()
4222 ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; in ext4_mb_initialize_context()
4223 ac->ac_o_ex.fe_group = group; in ext4_mb_initialize_context()
4224 ac->ac_o_ex.fe_start = block; in ext4_mb_initialize_context()
4225 ac->ac_o_ex.fe_len = len; in ext4_mb_initialize_context()
4226 ac->ac_g_ex = ac->ac_o_ex; in ext4_mb_initialize_context()
4227 ac->ac_flags = ar->flags; in ext4_mb_initialize_context()
4231 ext4_mb_group_or_file(ac); in ext4_mb_initialize_context()
4236 (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, in ext4_mb_initialize_context()
4329 static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) in ext4_mb_add_n_trim() argument
4332 struct super_block *sb = ac->ac_sb; in ext4_mb_add_n_trim()
4333 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_add_n_trim()
4334 struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; in ext4_mb_add_n_trim()
4379 static int ext4_mb_release_context(struct ext4_allocation_context *ac) in ext4_mb_release_context() argument
4381 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_release_context()
4382 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_release_context()
4387 pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
4388 pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
4389 pa->pa_free -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
4390 pa->pa_len -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
4405 ext4_mb_add_n_trim(ac); in ext4_mb_release_context()
4407 ext4_mb_put_pa(ac, ac->ac_sb, pa); in ext4_mb_release_context()
4409 if (ac->ac_bitmap_page) in ext4_mb_release_context()
4410 put_page(ac->ac_bitmap_page); in ext4_mb_release_context()
4411 if (ac->ac_buddy_page) in ext4_mb_release_context()
4412 put_page(ac->ac_buddy_page); in ext4_mb_release_context()
4413 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_release_context()
4414 mutex_unlock(&ac->ac_lg->lg_mutex); in ext4_mb_release_context()
4415 ext4_mb_collect_stats(ac); in ext4_mb_release_context()
4444 struct ext4_allocation_context *ac = NULL; in ext4_mb_new_blocks() local
4497 ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS); in ext4_mb_new_blocks()
4498 if (!ac) { in ext4_mb_new_blocks()
4504 *errp = ext4_mb_initialize_context(ac, ar); in ext4_mb_new_blocks()
4510 ac->ac_op = EXT4_MB_HISTORY_PREALLOC; in ext4_mb_new_blocks()
4511 if (!ext4_mb_use_preallocated(ac)) { in ext4_mb_new_blocks()
4512 ac->ac_op = EXT4_MB_HISTORY_ALLOC; in ext4_mb_new_blocks()
4513 ext4_mb_normalize_request(ac, ar); in ext4_mb_new_blocks()
4516 *errp = ext4_mb_regular_allocator(ac); in ext4_mb_new_blocks()
4523 if (ac->ac_status == AC_STATUS_FOUND && in ext4_mb_new_blocks()
4524 ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) in ext4_mb_new_blocks()
4525 *errp = ext4_mb_new_preallocation(ac); in ext4_mb_new_blocks()
4528 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
4532 if (likely(ac->ac_status == AC_STATUS_FOUND)) { in ext4_mb_new_blocks()
4533 *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); in ext4_mb_new_blocks()
4535 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
4538 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_blocks()
4539 ar->len = ac->ac_b_ex.fe_len; in ext4_mb_new_blocks()
4542 freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); in ext4_mb_new_blocks()
4550 ac->ac_b_ex.fe_len = 0; in ext4_mb_new_blocks()
4552 ext4_mb_show_ac(ac); in ext4_mb_new_blocks()
4554 ext4_mb_release_context(ac); in ext4_mb_new_blocks()
4556 if (ac) in ext4_mb_new_blocks()
4557 kmem_cache_free(ext4_ac_cachep, ac); in ext4_mb_new_blocks()