Lines Matching refs:ac
1476 static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, in ext4_mb_use_best_found() argument
1479 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_best_found()
1482 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); in ext4_mb_use_best_found()
1483 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_use_best_found()
1485 ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); in ext4_mb_use_best_found()
1486 ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; in ext4_mb_use_best_found()
1487 ret = mb_mark_used(e4b, &ac->ac_b_ex); in ext4_mb_use_best_found()
1491 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_use_best_found()
1493 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_best_found()
1494 ac->ac_tail = ret & 0xffff; in ext4_mb_use_best_found()
1495 ac->ac_buddy = ret >> 16; in ext4_mb_use_best_found()
1504 ac->ac_bitmap_page = e4b->bd_bitmap_page; in ext4_mb_use_best_found()
1505 get_page(ac->ac_bitmap_page); in ext4_mb_use_best_found()
1506 ac->ac_buddy_page = e4b->bd_buddy_page; in ext4_mb_use_best_found()
1507 get_page(ac->ac_buddy_page); in ext4_mb_use_best_found()
1509 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_use_best_found()
1511 sbi->s_mb_last_group = ac->ac_f_ex.fe_group; in ext4_mb_use_best_found()
1512 sbi->s_mb_last_start = ac->ac_f_ex.fe_start; in ext4_mb_use_best_found()
1521 static void ext4_mb_check_limits(struct ext4_allocation_context *ac, in ext4_mb_check_limits() argument
1525 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_check_limits()
1526 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_check_limits()
1527 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_check_limits()
1531 if (ac->ac_status == AC_STATUS_FOUND) in ext4_mb_check_limits()
1536 if (ac->ac_found > sbi->s_mb_max_to_scan && in ext4_mb_check_limits()
1537 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_check_limits()
1538 ac->ac_status = AC_STATUS_BREAK; in ext4_mb_check_limits()
1548 if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) in ext4_mb_check_limits()
1555 ext4_mb_use_best_found(ac, e4b); in ext4_mb_check_limits()
1571 static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, in ext4_mb_measure_extent() argument
1575 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_measure_extent()
1576 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_measure_extent()
1579 BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
1580 BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
1581 BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); in ext4_mb_measure_extent()
1583 ac->ac_found++; in ext4_mb_measure_extent()
1588 if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_measure_extent()
1590 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
1599 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
1627 ext4_mb_check_limits(ac, e4b, 0); in ext4_mb_measure_extent()
1631 int ext4_mb_try_best_found(struct ext4_allocation_context *ac, in ext4_mb_try_best_found() argument
1634 struct ext4_free_extent ex = ac->ac_b_ex; in ext4_mb_try_best_found()
1640 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_try_best_found()
1644 ext4_lock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
1648 ac->ac_b_ex = ex; in ext4_mb_try_best_found()
1649 ext4_mb_use_best_found(ac, e4b); in ext4_mb_try_best_found()
1652 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
1659 int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, in ext4_mb_find_by_goal() argument
1662 ext4_group_t group = ac->ac_g_ex.fe_group; in ext4_mb_find_by_goal()
1665 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_find_by_goal()
1668 if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL)) in ext4_mb_find_by_goal()
1671 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_find_by_goal()
1675 ext4_lock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
1676 max = mb_find_extent(e4b, 0, ac->ac_g_ex.fe_start, in ext4_mb_find_by_goal()
1677 ac->ac_g_ex.fe_len, &ex); in ext4_mb_find_by_goal()
1679 if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { in ext4_mb_find_by_goal()
1682 start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) + in ext4_mb_find_by_goal()
1686 ac->ac_found++; in ext4_mb_find_by_goal()
1687 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1688 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
1690 } else if (max >= ac->ac_g_ex.fe_len) { in ext4_mb_find_by_goal()
1692 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
1693 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
1694 ac->ac_found++; in ext4_mb_find_by_goal()
1695 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1696 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
1697 } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { in ext4_mb_find_by_goal()
1701 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
1702 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
1703 ac->ac_found++; in ext4_mb_find_by_goal()
1704 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1705 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
1707 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
1718 void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, in ext4_mb_simple_scan_group() argument
1721 struct super_block *sb = ac->ac_sb; in ext4_mb_simple_scan_group()
1728 BUG_ON(ac->ac_2order <= 0); in ext4_mb_simple_scan_group()
1729 for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) { in ext4_mb_simple_scan_group()
1739 ac->ac_found++; in ext4_mb_simple_scan_group()
1741 ac->ac_b_ex.fe_len = 1 << i; in ext4_mb_simple_scan_group()
1742 ac->ac_b_ex.fe_start = k << i; in ext4_mb_simple_scan_group()
1743 ac->ac_b_ex.fe_group = e4b->bd_group; in ext4_mb_simple_scan_group()
1745 ext4_mb_use_best_found(ac, e4b); in ext4_mb_simple_scan_group()
1747 BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len); in ext4_mb_simple_scan_group()
1762 void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, in ext4_mb_complex_scan_group() argument
1765 struct super_block *sb = ac->ac_sb; in ext4_mb_complex_scan_group()
1776 while (free && ac->ac_status == AC_STATUS_CONTINUE) { in ext4_mb_complex_scan_group()
1792 mb_find_extent(e4b, 0, i, ac->ac_g_ex.fe_len, &ex); in ext4_mb_complex_scan_group()
1807 ext4_mb_measure_extent(ac, &ex, e4b); in ext4_mb_complex_scan_group()
1813 ext4_mb_check_limits(ac, e4b, 1); in ext4_mb_complex_scan_group()
1821 void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, in ext4_mb_scan_aligned() argument
1824 struct super_block *sb = ac->ac_sb; in ext4_mb_scan_aligned()
1846 ac->ac_found++; in ext4_mb_scan_aligned()
1847 ac->ac_b_ex = ex; in ext4_mb_scan_aligned()
1848 ext4_mb_use_best_found(ac, e4b); in ext4_mb_scan_aligned()
1857 static int ext4_mb_good_group(struct ext4_allocation_context *ac, in ext4_mb_good_group() argument
1861 int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); in ext4_mb_good_group()
1862 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group()
1868 int ret = ext4_mb_init_group(ac->ac_sb, group); in ext4_mb_good_group()
1882 BUG_ON(ac->ac_2order == 0); in ext4_mb_good_group()
1884 if (grp->bb_largest_free_order < ac->ac_2order) in ext4_mb_good_group()
1888 if ((ac->ac_flags & EXT4_MB_HINT_DATA) && in ext4_mb_good_group()
1895 if ((free / fragments) >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
1899 if (free >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
1912 ext4_mb_regular_allocator(struct ext4_allocation_context *ac) in ext4_mb_regular_allocator() argument
1921 sb = ac->ac_sb; in ext4_mb_regular_allocator()
1925 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) in ext4_mb_regular_allocator()
1928 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_regular_allocator()
1931 err = ext4_mb_find_by_goal(ac, &e4b); in ext4_mb_regular_allocator()
1932 if (err || ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
1935 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_regular_allocator()
1943 i = fls(ac->ac_g_ex.fe_len); in ext4_mb_regular_allocator()
1944 ac->ac_2order = 0; in ext4_mb_regular_allocator()
1954 if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) in ext4_mb_regular_allocator()
1955 ac->ac_2order = i - 1; in ext4_mb_regular_allocator()
1959 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_regular_allocator()
1962 ac->ac_g_ex.fe_group = sbi->s_mb_last_group; in ext4_mb_regular_allocator()
1963 ac->ac_g_ex.fe_start = sbi->s_mb_last_start; in ext4_mb_regular_allocator()
1968 cr = ac->ac_2order ? 0 : 1; in ext4_mb_regular_allocator()
1974 for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { in ext4_mb_regular_allocator()
1975 ac->ac_criteria = cr; in ext4_mb_regular_allocator()
1980 group = ac->ac_g_ex.fe_group; in ext4_mb_regular_allocator()
1991 if (!ext4_mb_good_group(ac, group, cr)) in ext4_mb_regular_allocator()
2004 if (!ext4_mb_good_group(ac, group, cr)) { in ext4_mb_regular_allocator()
2010 ac->ac_groups_scanned++; in ext4_mb_regular_allocator()
2012 ext4_mb_simple_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2014 !(ac->ac_g_ex.fe_len % sbi->s_stripe)) in ext4_mb_regular_allocator()
2015 ext4_mb_scan_aligned(ac, &e4b); in ext4_mb_regular_allocator()
2017 ext4_mb_complex_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2022 if (ac->ac_status != AC_STATUS_CONTINUE) in ext4_mb_regular_allocator()
2027 if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && in ext4_mb_regular_allocator()
2028 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_regular_allocator()
2034 ext4_mb_try_best_found(ac, &e4b); in ext4_mb_regular_allocator()
2035 if (ac->ac_status != AC_STATUS_FOUND) { in ext4_mb_regular_allocator()
2042 ac->ac_b_ex.fe_group = 0; in ext4_mb_regular_allocator()
2043 ac->ac_b_ex.fe_start = 0; in ext4_mb_regular_allocator()
2044 ac->ac_b_ex.fe_len = 0; in ext4_mb_regular_allocator()
2045 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_regular_allocator()
2046 ac->ac_flags |= EXT4_MB_HINT_FIRST; in ext4_mb_regular_allocator()
2728 ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, in ext4_mb_mark_diskspace_used() argument
2739 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_mark_diskspace_used()
2740 BUG_ON(ac->ac_b_ex.fe_len <= 0); in ext4_mb_mark_diskspace_used()
2742 sb = ac->ac_sb; in ext4_mb_mark_diskspace_used()
2746 bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
2755 gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); in ext4_mb_mark_diskspace_used()
2759 ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, in ext4_mb_mark_diskspace_used()
2766 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_mark_diskspace_used()
2768 len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
2776 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
2777 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
2778 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
2779 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
2786 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
2790 for (i = 0; i < ac->ac_b_ex.fe_len; i++) { in ext4_mb_mark_diskspace_used()
2791 BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, in ext4_mb_mark_diskspace_used()
2796 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
2797 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
2802 ac->ac_b_ex.fe_group, gdp)); in ext4_mb_mark_diskspace_used()
2804 len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; in ext4_mb_mark_diskspace_used()
2806 gdp->bg_checksum = ext4_group_desc_csum(sbi, ac->ac_b_ex.fe_group, gdp); in ext4_mb_mark_diskspace_used()
2808 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
2809 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
2813 if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) in ext4_mb_mark_diskspace_used()
2820 ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
2821 atomic64_sub(ac->ac_b_ex.fe_len, in ext4_mb_mark_diskspace_used()
2845 static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) in ext4_mb_normalize_group_request() argument
2847 struct super_block *sb = ac->ac_sb; in ext4_mb_normalize_group_request()
2848 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_normalize_group_request()
2851 ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; in ext4_mb_normalize_group_request()
2853 current->pid, ac->ac_g_ex.fe_len); in ext4_mb_normalize_group_request()
2861 ext4_mb_normalize_request(struct ext4_allocation_context *ac, in ext4_mb_normalize_request() argument
2864 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_normalize_request()
2870 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_normalize_request()
2875 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_normalize_request()
2879 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_normalize_request()
2884 if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) in ext4_mb_normalize_request()
2887 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { in ext4_mb_normalize_request()
2888 ext4_mb_normalize_group_request(ac); in ext4_mb_normalize_request()
2892 bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_normalize_request()
2896 size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); in ext4_mb_normalize_request()
2898 if (size < i_size_read(ac->ac_inode)) in ext4_mb_normalize_request()
2899 size = i_size_read(ac->ac_inode); in ext4_mb_normalize_request()
2926 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
2930 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
2933 } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, in ext4_mb_normalize_request()
2935 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
2939 start_off = (loff_t)ac->ac_o_ex.fe_logical << bsbits; in ext4_mb_normalize_request()
2940 size = ac->ac_o_ex.fe_len << bsbits; in ext4_mb_normalize_request()
2968 pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
2972 BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || in ext4_mb_normalize_request()
2973 ac->ac_o_ex.fe_logical < pa->pa_lstart)); in ext4_mb_normalize_request()
2983 if (pa_end <= ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
2986 } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3002 pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
3010 if (start + size <= ac->ac_o_ex.fe_logical && in ext4_mb_normalize_request()
3011 start > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3012 ext4_msg(ac->ac_sb, KERN_ERR, in ext4_mb_normalize_request()
3015 (unsigned long) ac->ac_o_ex.fe_logical); in ext4_mb_normalize_request()
3017 BUG_ON(start + size <= ac->ac_o_ex.fe_logical && in ext4_mb_normalize_request()
3018 start > ac->ac_o_ex.fe_logical); in ext4_mb_normalize_request()
3019 BUG_ON(size <= 0 || size > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_normalize_request()
3025 ac->ac_g_ex.fe_logical = start; in ext4_mb_normalize_request()
3026 ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); in ext4_mb_normalize_request()
3031 ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, in ext4_mb_normalize_request()
3032 &ac->ac_f_ex.fe_group, in ext4_mb_normalize_request()
3033 &ac->ac_f_ex.fe_start); in ext4_mb_normalize_request()
3034 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
3038 ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, in ext4_mb_normalize_request()
3039 &ac->ac_f_ex.fe_group, in ext4_mb_normalize_request()
3040 &ac->ac_f_ex.fe_start); in ext4_mb_normalize_request()
3041 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
3048 static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) in ext4_mb_collect_stats() argument
3050 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_collect_stats()
3052 if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) { in ext4_mb_collect_stats()
3054 atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); in ext4_mb_collect_stats()
3055 if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) in ext4_mb_collect_stats()
3057 atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); in ext4_mb_collect_stats()
3058 if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && in ext4_mb_collect_stats()
3059 ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) in ext4_mb_collect_stats()
3061 if (ac->ac_found > sbi->s_mb_max_to_scan) in ext4_mb_collect_stats()
3065 if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) in ext4_mb_collect_stats()
3066 trace_ext4_mballoc_alloc(ac); in ext4_mb_collect_stats()
3068 trace_ext4_mballoc_prealloc(ac); in ext4_mb_collect_stats()
3077 static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) in ext4_discard_allocated_blocks() argument
3079 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_discard_allocated_blocks()
3083 len = ac->ac_b_ex.fe_len; in ext4_discard_allocated_blocks()
3092 static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, in ext4_mb_use_inode_pa() argument
3095 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_inode_pa()
3101 start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); in ext4_mb_use_inode_pa()
3103 start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); in ext4_mb_use_inode_pa()
3105 ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, in ext4_mb_use_inode_pa()
3106 &ac->ac_b_ex.fe_start); in ext4_mb_use_inode_pa()
3107 ac->ac_b_ex.fe_len = len; in ext4_mb_use_inode_pa()
3108 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_inode_pa()
3109 ac->ac_pa = pa; in ext4_mb_use_inode_pa()
3122 static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, in ext4_mb_use_group_pa() argument
3125 unsigned int len = ac->ac_o_ex.fe_len; in ext4_mb_use_group_pa()
3127 ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, in ext4_mb_use_group_pa()
3128 &ac->ac_b_ex.fe_group, in ext4_mb_use_group_pa()
3129 &ac->ac_b_ex.fe_start); in ext4_mb_use_group_pa()
3130 ac->ac_b_ex.fe_len = len; in ext4_mb_use_group_pa()
3131 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_group_pa()
3132 ac->ac_pa = pa; in ext4_mb_use_group_pa()
3176 ext4_mb_use_preallocated(struct ext4_allocation_context *ac) in ext4_mb_use_preallocated() argument
3178 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_preallocated()
3180 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_use_preallocated()
3186 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_use_preallocated()
3195 if (ac->ac_o_ex.fe_logical < pa->pa_lstart || in ext4_mb_use_preallocated()
3196 ac->ac_o_ex.fe_logical >= (pa->pa_lstart + in ext4_mb_use_preallocated()
3201 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && in ext4_mb_use_preallocated()
3210 ext4_mb_use_inode_pa(ac, pa); in ext4_mb_use_preallocated()
3212 ac->ac_criteria = 10; in ext4_mb_use_preallocated()
3221 if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) in ext4_mb_use_preallocated()
3225 lg = ac->ac_lg; in ext4_mb_use_preallocated()
3228 order = fls(ac->ac_o_ex.fe_len) - 1; in ext4_mb_use_preallocated()
3233 goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); in ext4_mb_use_preallocated()
3244 pa->pa_free >= ac->ac_o_ex.fe_len) { in ext4_mb_use_preallocated()
3254 ext4_mb_use_group_pa(ac, cpa); in ext4_mb_use_preallocated()
3255 ac->ac_criteria = 20; in ext4_mb_use_preallocated()
3337 static void ext4_mb_put_pa(struct ext4_allocation_context *ac, in ext4_mb_put_pa() argument
3395 ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) in ext4_mb_new_inode_pa() argument
3397 struct super_block *sb = ac->ac_sb; in ext4_mb_new_inode_pa()
3404 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
3405 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_inode_pa()
3406 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_inode_pa()
3412 if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { in ext4_mb_new_inode_pa()
3421 BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); in ext4_mb_new_inode_pa()
3422 BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); in ext4_mb_new_inode_pa()
3427 winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical; in ext4_mb_new_inode_pa()
3430 wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len); in ext4_mb_new_inode_pa()
3435 offs = ac->ac_o_ex.fe_logical % in ext4_mb_new_inode_pa()
3436 EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
3440 ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical - in ext4_mb_new_inode_pa()
3442 BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); in ext4_mb_new_inode_pa()
3443 BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
3448 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_new_inode_pa()
3450 pa->pa_lstart = ac->ac_b_ex.fe_logical; in ext4_mb_new_inode_pa()
3451 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_inode_pa()
3452 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_inode_pa()
3463 trace_ext4_mb_new_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
3465 ext4_mb_use_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
3468 ei = EXT4_I(ac->ac_inode); in ext4_mb_new_inode_pa()
3469 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_inode_pa()
3472 pa->pa_inode = ac->ac_inode; in ext4_mb_new_inode_pa()
3474 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_inode_pa()
3476 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_inode_pa()
3489 ext4_mb_new_group_pa(struct ext4_allocation_context *ac) in ext4_mb_new_group_pa() argument
3491 struct super_block *sb = ac->ac_sb; in ext4_mb_new_group_pa()
3497 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_group_pa()
3498 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_group_pa()
3499 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_group_pa()
3508 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_new_group_pa()
3510 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_group_pa()
3512 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_group_pa()
3523 trace_ext4_mb_new_group_pa(ac, pa); in ext4_mb_new_group_pa()
3525 ext4_mb_use_group_pa(ac, pa); in ext4_mb_new_group_pa()
3528 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_group_pa()
3529 lg = ac->ac_lg; in ext4_mb_new_group_pa()
3535 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_group_pa()
3537 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_group_pa()
3546 static int ext4_mb_new_preallocation(struct ext4_allocation_context *ac) in ext4_mb_new_preallocation() argument
3550 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_new_preallocation()
3551 err = ext4_mb_new_group_pa(ac); in ext4_mb_new_preallocation()
3553 err = ext4_mb_new_inode_pa(ac); in ext4_mb_new_preallocation()
3864 static void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
3866 struct super_block *sb = ac->ac_sb; in ext4_mb_show_ac()
3873 ext4_msg(ac->ac_sb, KERN_ERR, "Can't allocate:" in ext4_mb_show_ac()
3875 ext4_msg(ac->ac_sb, KERN_ERR, "status %d flags %d", in ext4_mb_show_ac()
3876 ac->ac_status, ac->ac_flags); in ext4_mb_show_ac()
3877 ext4_msg(ac->ac_sb, KERN_ERR, "orig %lu/%lu/%lu@%lu, " in ext4_mb_show_ac()
3880 (unsigned long)ac->ac_o_ex.fe_group, in ext4_mb_show_ac()
3881 (unsigned long)ac->ac_o_ex.fe_start, in ext4_mb_show_ac()
3882 (unsigned long)ac->ac_o_ex.fe_len, in ext4_mb_show_ac()
3883 (unsigned long)ac->ac_o_ex.fe_logical, in ext4_mb_show_ac()
3884 (unsigned long)ac->ac_g_ex.fe_group, in ext4_mb_show_ac()
3885 (unsigned long)ac->ac_g_ex.fe_start, in ext4_mb_show_ac()
3886 (unsigned long)ac->ac_g_ex.fe_len, in ext4_mb_show_ac()
3887 (unsigned long)ac->ac_g_ex.fe_logical, in ext4_mb_show_ac()
3888 (unsigned long)ac->ac_b_ex.fe_group, in ext4_mb_show_ac()
3889 (unsigned long)ac->ac_b_ex.fe_start, in ext4_mb_show_ac()
3890 (unsigned long)ac->ac_b_ex.fe_len, in ext4_mb_show_ac()
3891 (unsigned long)ac->ac_b_ex.fe_logical, in ext4_mb_show_ac()
3892 (int)ac->ac_criteria); in ext4_mb_show_ac()
3893 ext4_msg(ac->ac_sb, KERN_ERR, "%lu scanned, %d found", in ext4_mb_show_ac()
3894 ac->ac_ex_scanned, ac->ac_found); in ext4_mb_show_ac()
3895 ext4_msg(ac->ac_sb, KERN_ERR, "groups: "); in ext4_mb_show_ac()
3923 static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
3936 static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) in ext4_mb_group_or_file() argument
3938 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_group_or_file()
3939 int bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_group_or_file()
3942 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_group_or_file()
3945 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_group_or_file()
3948 size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); in ext4_mb_group_or_file()
3949 isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) in ext4_mb_group_or_file()
3954 (atomic_read(&ac->ac_inode->i_writecount) == 0)) { in ext4_mb_group_or_file()
3955 ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; in ext4_mb_group_or_file()
3960 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
3967 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
3971 BUG_ON(ac->ac_lg != NULL); in ext4_mb_group_or_file()
3977 ac->ac_lg = __this_cpu_ptr(sbi->s_locality_groups); in ext4_mb_group_or_file()
3980 ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; in ext4_mb_group_or_file()
3983 mutex_lock(&ac->ac_lg->lg_mutex); in ext4_mb_group_or_file()
3987 ext4_mb_initialize_context(struct ext4_allocation_context *ac, in ext4_mb_initialize_context() argument
4013 memset(ac, 0, sizeof(struct ext4_allocation_context)); in ext4_mb_initialize_context()
4014 ac->ac_b_ex.fe_logical = ar->logical & ~(sbi->s_cluster_ratio - 1); in ext4_mb_initialize_context()
4015 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_initialize_context()
4016 ac->ac_sb = sb; in ext4_mb_initialize_context()
4017 ac->ac_inode = ar->inode; in ext4_mb_initialize_context()
4018 ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; in ext4_mb_initialize_context()
4019 ac->ac_o_ex.fe_group = group; in ext4_mb_initialize_context()
4020 ac->ac_o_ex.fe_start = block; in ext4_mb_initialize_context()
4021 ac->ac_o_ex.fe_len = len; in ext4_mb_initialize_context()
4022 ac->ac_g_ex = ac->ac_o_ex; in ext4_mb_initialize_context()
4023 ac->ac_flags = ar->flags; in ext4_mb_initialize_context()
4027 ext4_mb_group_or_file(ac); in ext4_mb_initialize_context()
4032 (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, in ext4_mb_initialize_context()
4122 static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) in ext4_mb_add_n_trim() argument
4125 struct super_block *sb = ac->ac_sb; in ext4_mb_add_n_trim()
4126 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_add_n_trim()
4127 struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; in ext4_mb_add_n_trim()
4172 static int ext4_mb_release_context(struct ext4_allocation_context *ac) in ext4_mb_release_context() argument
4174 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_release_context()
4175 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_release_context()
4180 pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
4181 pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
4182 pa->pa_free -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
4183 pa->pa_len -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
4198 ext4_mb_add_n_trim(ac); in ext4_mb_release_context()
4200 ext4_mb_put_pa(ac, ac->ac_sb, pa); in ext4_mb_release_context()
4202 if (ac->ac_bitmap_page) in ext4_mb_release_context()
4203 page_cache_release(ac->ac_bitmap_page); in ext4_mb_release_context()
4204 if (ac->ac_buddy_page) in ext4_mb_release_context()
4205 page_cache_release(ac->ac_buddy_page); in ext4_mb_release_context()
4206 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_release_context()
4207 mutex_unlock(&ac->ac_lg->lg_mutex); in ext4_mb_release_context()
4208 ext4_mb_collect_stats(ac); in ext4_mb_release_context()
4237 struct ext4_allocation_context *ac = NULL; in ext4_mb_new_blocks() local
4296 ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS); in ext4_mb_new_blocks()
4297 if (!ac) { in ext4_mb_new_blocks()
4303 *errp = ext4_mb_initialize_context(ac, ar); in ext4_mb_new_blocks()
4309 ac->ac_op = EXT4_MB_HISTORY_PREALLOC; in ext4_mb_new_blocks()
4310 if (!ext4_mb_use_preallocated(ac)) { in ext4_mb_new_blocks()
4311 ac->ac_op = EXT4_MB_HISTORY_ALLOC; in ext4_mb_new_blocks()
4312 ext4_mb_normalize_request(ac, ar); in ext4_mb_new_blocks()
4315 *errp = ext4_mb_regular_allocator(ac); in ext4_mb_new_blocks()
4322 if (ac->ac_status == AC_STATUS_FOUND && in ext4_mb_new_blocks()
4323 ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) in ext4_mb_new_blocks()
4324 ext4_mb_new_preallocation(ac); in ext4_mb_new_blocks()
4326 if (likely(ac->ac_status == AC_STATUS_FOUND)) { in ext4_mb_new_blocks()
4327 *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); in ext4_mb_new_blocks()
4333 ext4_mb_release_context(ac); in ext4_mb_new_blocks()
4334 ac->ac_b_ex.fe_group = 0; in ext4_mb_new_blocks()
4335 ac->ac_b_ex.fe_start = 0; in ext4_mb_new_blocks()
4336 ac->ac_b_ex.fe_len = 0; in ext4_mb_new_blocks()
4337 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_new_blocks()
4341 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
4343 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_blocks()
4344 ar->len = ac->ac_b_ex.fe_len; in ext4_mb_new_blocks()
4347 freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); in ext4_mb_new_blocks()
4354 ac->ac_b_ex.fe_len = 0; in ext4_mb_new_blocks()
4356 ext4_mb_show_ac(ac); in ext4_mb_new_blocks()
4358 ext4_mb_release_context(ac); in ext4_mb_new_blocks()
4360 if (ac) in ext4_mb_new_blocks()
4361 kmem_cache_free(ext4_ac_cachep, ac); in ext4_mb_new_blocks()