Lines Matching +full:1 +full:ac
101 * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
210 * 1) if buddy is referenced, it's already initialized
353 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac);
357 * 1. We sample the percpu discard_pa_seq counter before trying for block
459 if (order > e4b->bd_blkbits + 1) { in mb_find_buddy()
466 *max = 1 << (e4b->bd_blkbits + 3); in mb_find_buddy()
609 int order = e4b->bd_blkbits + 1; in __mb_check_buddy()
626 while (order > 1) { in __mb_check_buddy()
629 buddy2 = mb_find_buddy(e4b, order - 1, &max2); in __mb_check_buddy()
638 /* only single bit in buddy2 may be 1 */ in __mb_check_buddy()
639 if (!mb_test_bit(i << 1, buddy2)) { in __mb_check_buddy()
641 mb_test_bit((i<<1)+1, buddy2)); in __mb_check_buddy()
642 } else if (!mb_test_bit((i << 1) + 1, buddy2)) { in __mb_check_buddy()
644 mb_test_bit(i << 1, buddy2)); in __mb_check_buddy()
649 /* both bits in buddy2 must be 1 */ in __mb_check_buddy()
650 MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2)); in __mb_check_buddy()
651 MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2)); in __mb_check_buddy()
653 for (j = 0; j < (1 << order); j++) { in __mb_check_buddy()
654 k = (i * (1 << order)) + j; in __mb_check_buddy()
664 fstart = -1; in __mb_check_buddy()
669 if (fstart == -1) { in __mb_check_buddy()
675 fstart = -1; in __mb_check_buddy()
677 for (j = 0; j < e4b->bd_blkbits + 1; j++) { in __mb_check_buddy()
730 max = ffs(first | border) - 1; in ext4_mb_mark_free_simple()
733 min = fls(len) - 1; in ext4_mb_mark_free_simple()
737 chunk = 1 << min; in ext4_mb_mark_free_simple()
760 grp->bb_largest_free_order = -1; /* uninit */ in mb_set_largest_free_order()
762 bits = sb->s_blocksize_bits + 1; in mb_set_largest_free_order()
795 if (len > 1) in ext4_mb_generate_buddy()
829 int order = 1; in mb_regenerate_buddy()
851 * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
890 groups_per_page = blocks_per_page >> 1; in ext4_mb_init_cache()
892 groups_per_page = 1; in ext4_mb_init_cache()
895 if (groups_per_page > 1) { in ext4_mb_init_cache()
947 group = (first_block + i) >> 1; in ext4_mb_init_cache()
973 if ((first_block + i) & 1) { in ext4_mb_init_cache()
1322 int order = 1; in mb_find_order_for_block()
1323 int bb_incr = 1 << (e4b->bd_blkbits - 1); in mb_find_order_for_block()
1327 BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); in mb_find_order_for_block()
1330 while (order <= e4b->bd_blkbits + 1) { in mb_find_order_for_block()
1331 block = block >> 1; in mb_find_order_for_block()
1337 bb_incr >>= 1; in mb_find_order_for_block()
1362 * will return first found zero bit if any, -1 otherwise
1367 int zero_bit = -1; in mb_test_and_clear_bits()
1374 if (*addr != (__u32)(-1) && zero_bit == -1) in mb_test_and_clear_bits()
1380 if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1) in mb_test_and_clear_bits()
1411 return 1; in mb_buddy_adjust_border()
1416 return -1; in mb_buddy_adjust_border()
1423 int order = 1; in mb_buddy_mark_free()
1440 * | 1 | 1 | 1 | 1 | in mb_buddy_mark_free()
1442 * | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | in mb_buddy_mark_free()
1444 * 0 1 2 3 4 5 6 7 in mb_buddy_mark_free()
1447 * Neither [1] nor [6] is aligned to above layer. in mb_buddy_mark_free()
1458 if (first & 1) in mb_buddy_mark_free()
1459 e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1); in mb_buddy_mark_free()
1460 if (!(last & 1)) in mb_buddy_mark_free()
1461 e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1); in mb_buddy_mark_free()
1467 mb_clear_bits(buddy, first, last - first + 1); in mb_buddy_mark_free()
1468 e4b->bd_info->bb_counters[order - 1] += last - first + 1; in mb_buddy_mark_free()
1471 first >>= 1; in mb_buddy_mark_free()
1472 last >>= 1; in mb_buddy_mark_free()
1483 int last = first + count - 1; in mb_free_blocks()
1506 left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap); in mb_free_blocks()
1508 if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0]) in mb_free_blocks()
1509 right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap); in mb_free_blocks()
1511 if (unlikely(block != -1)) { in mb_free_blocks()
1544 if (first & 1) { in mb_free_blocks()
1546 e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1; in mb_free_blocks()
1548 if (!(last & 1)) { in mb_free_blocks()
1550 e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1; in mb_free_blocks()
1554 mb_buddy_mark_free(e4b, first >> 1, last >> 1); in mb_free_blocks()
1585 ex->fe_len = 1 << order; in mb_find_extent()
1597 if (block + 1 >= max) in mb_find_extent()
1600 next = (block + 1) * (1 << order); in mb_find_extent()
1607 ex->fe_len += 1 << order; in mb_find_extent()
1612 WARN_ON(1); in mb_find_extent()
1650 mlen = !mb_test_bit(start - 1, e4b->bd_bitmap); in mb_mark_used()
1662 if (((start >> ord) << ord) == start && len >= (1 << ord)) { in mb_mark_used()
1664 mlen = 1 << ord; in mb_mark_used()
1686 cur = (start >> ord) & ~1U; in mb_mark_used()
1689 mb_clear_bit(cur + 1, buddy); in mb_mark_used()
1704 static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, in ext4_mb_use_best_found() argument
1707 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_best_found()
1710 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); in ext4_mb_use_best_found()
1711 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_use_best_found()
1713 ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); in ext4_mb_use_best_found()
1714 ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; in ext4_mb_use_best_found()
1715 ret = mb_mark_used(e4b, &ac->ac_b_ex); in ext4_mb_use_best_found()
1719 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_use_best_found()
1721 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_best_found()
1722 ac->ac_tail = ret & 0xffff; in ext4_mb_use_best_found()
1723 ac->ac_buddy = ret >> 16; in ext4_mb_use_best_found()
1732 ac->ac_bitmap_page = e4b->bd_bitmap_page; in ext4_mb_use_best_found()
1733 get_page(ac->ac_bitmap_page); in ext4_mb_use_best_found()
1734 ac->ac_buddy_page = e4b->bd_buddy_page; in ext4_mb_use_best_found()
1735 get_page(ac->ac_buddy_page); in ext4_mb_use_best_found()
1737 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_use_best_found()
1739 sbi->s_mb_last_group = ac->ac_f_ex.fe_group; in ext4_mb_use_best_found()
1740 sbi->s_mb_last_start = ac->ac_f_ex.fe_start; in ext4_mb_use_best_found()
1748 if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) in ext4_mb_use_best_found()
1749 ext4_mb_new_preallocation(ac); in ext4_mb_use_best_found()
1753 static void ext4_mb_check_limits(struct ext4_allocation_context *ac, in ext4_mb_check_limits() argument
1757 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_check_limits()
1758 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_check_limits()
1759 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_check_limits()
1763 if (ac->ac_status == AC_STATUS_FOUND) in ext4_mb_check_limits()
1768 if (ac->ac_found > sbi->s_mb_max_to_scan && in ext4_mb_check_limits()
1769 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_check_limits()
1770 ac->ac_status = AC_STATUS_BREAK; in ext4_mb_check_limits()
1780 if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) in ext4_mb_check_limits()
1787 ext4_mb_use_best_found(ac, e4b); in ext4_mb_check_limits()
1803 static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, in ext4_mb_measure_extent() argument
1807 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_measure_extent()
1808 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_measure_extent()
1811 BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
1812 BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
1813 BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); in ext4_mb_measure_extent()
1815 ac->ac_found++; in ext4_mb_measure_extent()
1820 if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_measure_extent()
1822 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
1831 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
1859 ext4_mb_check_limits(ac, e4b, 0); in ext4_mb_measure_extent()
1863 int ext4_mb_try_best_found(struct ext4_allocation_context *ac, in ext4_mb_try_best_found() argument
1866 struct ext4_free_extent ex = ac->ac_b_ex; in ext4_mb_try_best_found()
1872 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_try_best_found()
1876 ext4_lock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
1883 ac->ac_b_ex = ex; in ext4_mb_try_best_found()
1884 ext4_mb_use_best_found(ac, e4b); in ext4_mb_try_best_found()
1888 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
1895 int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, in ext4_mb_find_by_goal() argument
1898 ext4_group_t group = ac->ac_g_ex.fe_group; in ext4_mb_find_by_goal()
1901 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_find_by_goal()
1902 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_find_by_goal()
1907 if (!(ac->ac_flags & (EXT4_MB_HINT_TRY_GOAL | EXT4_MB_HINT_GOAL_ONLY))) in ext4_mb_find_by_goal()
1912 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_find_by_goal()
1916 ext4_lock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
1920 max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, in ext4_mb_find_by_goal()
1921 ac->ac_g_ex.fe_len, &ex); in ext4_mb_find_by_goal()
1924 if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { in ext4_mb_find_by_goal()
1927 start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) + in ext4_mb_find_by_goal()
1931 ac->ac_found++; in ext4_mb_find_by_goal()
1932 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1933 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
1935 } else if (max >= ac->ac_g_ex.fe_len) { in ext4_mb_find_by_goal()
1937 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
1938 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
1939 ac->ac_found++; in ext4_mb_find_by_goal()
1940 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1941 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
1942 } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { in ext4_mb_find_by_goal()
1946 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
1947 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
1948 ac->ac_found++; in ext4_mb_find_by_goal()
1949 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1950 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
1953 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
1964 void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, in ext4_mb_simple_scan_group() argument
1967 struct super_block *sb = ac->ac_sb; in ext4_mb_simple_scan_group()
1974 BUG_ON(ac->ac_2order <= 0); in ext4_mb_simple_scan_group()
1975 for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) { in ext4_mb_simple_scan_group()
1984 ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0, in ext4_mb_simple_scan_group()
1987 ext4_mark_group_bitmap_corrupted(ac->ac_sb, in ext4_mb_simple_scan_group()
1992 ac->ac_found++; in ext4_mb_simple_scan_group()
1994 ac->ac_b_ex.fe_len = 1 << i; in ext4_mb_simple_scan_group()
1995 ac->ac_b_ex.fe_start = k << i; in ext4_mb_simple_scan_group()
1996 ac->ac_b_ex.fe_group = e4b->bd_group; in ext4_mb_simple_scan_group()
1998 ext4_mb_use_best_found(ac, e4b); in ext4_mb_simple_scan_group()
2000 BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len); in ext4_mb_simple_scan_group()
2015 void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, in ext4_mb_complex_scan_group() argument
2018 struct super_block *sb = ac->ac_sb; in ext4_mb_complex_scan_group()
2030 while (free && ac->ac_status == AC_STATUS_CONTINUE) { in ext4_mb_complex_scan_group()
2048 mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); in ext4_mb_complex_scan_group()
2066 ext4_mb_measure_extent(ac, &ex, e4b); in ext4_mb_complex_scan_group()
2072 ext4_mb_check_limits(ac, e4b, 1); in ext4_mb_complex_scan_group()
2080 void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, in ext4_mb_scan_aligned() argument
2083 struct super_block *sb = ac->ac_sb; in ext4_mb_scan_aligned()
2097 a = first_group_block + sbi->s_stripe - 1; in ext4_mb_scan_aligned()
2105 ac->ac_found++; in ext4_mb_scan_aligned()
2107 ac->ac_b_ex = ex; in ext4_mb_scan_aligned()
2108 ext4_mb_use_best_found(ac, e4b); in ext4_mb_scan_aligned()
2118 * Returns either 1 or 0 indicating that the group is either suitable
2121 static bool ext4_mb_good_group(struct ext4_allocation_context *ac, in ext4_mb_good_group() argument
2125 int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); in ext4_mb_good_group()
2126 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group()
2143 BUG_ON(ac->ac_2order == 0); in ext4_mb_good_group()
2146 if ((ac->ac_flags & EXT4_MB_HINT_DATA) && in ext4_mb_good_group()
2151 if (free < ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2154 if (ac->ac_2order > ac->ac_sb->s_blocksize_bits+1) in ext4_mb_good_group()
2157 if (grp->bb_largest_free_order < ac->ac_2order) in ext4_mb_good_group()
2161 case 1: in ext4_mb_good_group()
2162 if ((free / fragments) >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2166 if (free >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2183 static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac, in ext4_mb_good_group_nolock() argument
2186 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group_nolock()
2187 struct super_block *sb = ac->ac_sb; in ext4_mb_good_group_nolock()
2189 bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK; in ext4_mb_good_group_nolock()
2196 atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]); in ext4_mb_good_group_nolock()
2202 if (cr <= 2 && free < ac->ac_g_ex.fe_len) in ext4_mb_good_group_nolock()
2215 /* cr=0/1 is a very optimistic search to find large in ext4_mb_good_group_nolock()
2225 ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) && in ext4_mb_good_group_nolock()
2236 ret = ext4_mb_good_group(ac, group, cr); in ext4_mb_good_group_nolock()
2322 ext4_mb_regular_allocator(struct ext4_allocation_context *ac) in ext4_mb_regular_allocator() argument
2325 int cr = -1; in ext4_mb_regular_allocator()
2333 sb = ac->ac_sb; in ext4_mb_regular_allocator()
2337 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) in ext4_mb_regular_allocator()
2340 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_regular_allocator()
2343 err = ext4_mb_find_by_goal(ac, &e4b); in ext4_mb_regular_allocator()
2344 if (err || ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
2347 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_regular_allocator()
2351 * ac->ac_2order is set only if the fe_len is a power of 2 in ext4_mb_regular_allocator()
2352 * if ac->ac_2order is set we also set criteria to 0 so that we in ext4_mb_regular_allocator()
2355 i = fls(ac->ac_g_ex.fe_len); in ext4_mb_regular_allocator()
2356 ac->ac_2order = 0; in ext4_mb_regular_allocator()
2368 if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) in ext4_mb_regular_allocator()
2369 ac->ac_2order = array_index_nospec(i - 1, in ext4_mb_regular_allocator()
2374 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_regular_allocator()
2377 ac->ac_g_ex.fe_group = sbi->s_mb_last_group; in ext4_mb_regular_allocator()
2378 ac->ac_g_ex.fe_start = sbi->s_mb_last_start; in ext4_mb_regular_allocator()
2383 cr = ac->ac_2order ? 0 : 1; in ext4_mb_regular_allocator()
2389 for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { in ext4_mb_regular_allocator()
2390 ac->ac_criteria = cr; in ext4_mb_regular_allocator()
2395 group = ac->ac_g_ex.fe_group; in ext4_mb_regular_allocator()
2411 * prefetching at cr=0/1, otherwise mballoc can in ext4_mb_regular_allocator()
2415 (cr > 1 || in ext4_mb_regular_allocator()
2421 nr = 1 << sbi->s_log_groups_per_flex; in ext4_mb_regular_allocator()
2422 nr -= group & (nr - 1); in ext4_mb_regular_allocator()
2432 ret = ext4_mb_good_group_nolock(ac, group, cr); in ext4_mb_regular_allocator()
2449 ret = ext4_mb_good_group(ac, group, cr); in ext4_mb_regular_allocator()
2456 ac->ac_groups_scanned++; in ext4_mb_regular_allocator()
2458 ext4_mb_simple_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2459 else if (cr == 1 && sbi->s_stripe && in ext4_mb_regular_allocator()
2460 !(ac->ac_g_ex.fe_len % sbi->s_stripe)) in ext4_mb_regular_allocator()
2461 ext4_mb_scan_aligned(ac, &e4b); in ext4_mb_regular_allocator()
2463 ext4_mb_complex_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2468 if (ac->ac_status != AC_STATUS_CONTINUE) in ext4_mb_regular_allocator()
2476 if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && in ext4_mb_regular_allocator()
2477 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_regular_allocator()
2482 ext4_mb_try_best_found(ac, &e4b); in ext4_mb_regular_allocator()
2483 if (ac->ac_status != AC_STATUS_FOUND) { in ext4_mb_regular_allocator()
2491 ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start, in ext4_mb_regular_allocator()
2492 ac->ac_b_ex.fe_len, lost); in ext4_mb_regular_allocator()
2494 ac->ac_b_ex.fe_group = 0; in ext4_mb_regular_allocator()
2495 ac->ac_b_ex.fe_start = 0; in ext4_mb_regular_allocator()
2496 ac->ac_b_ex.fe_len = 0; in ext4_mb_regular_allocator()
2497 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_regular_allocator()
2498 ac->ac_flags |= EXT4_MB_HINT_FIRST; in ext4_mb_regular_allocator()
2504 if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
2505 atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]); in ext4_mb_regular_allocator()
2507 if (!err && ac->ac_status != AC_STATUS_FOUND && first_err) in ext4_mb_regular_allocator()
2511 ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status, in ext4_mb_regular_allocator()
2512 ac->ac_flags, cr, err); in ext4_mb_regular_allocator()
2527 group = *pos + 1; in ext4_mb_seq_groups_start()
2539 group = *pos + 1; in ext4_mb_seq_groups_next()
2562 " 2^0 2^1 2^2 2^3 2^4 2^5 2^6 " in ext4_mb_seq_groups_show()
2578 buddy_loaded = 1; in ext4_mb_seq_groups_show()
2589 seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ? in ext4_mb_seq_groups_show()
2615 seq_puts(seq, "\tTo enable, please write \"1\" to sysfs file mb_stats.\n"); in ext4_seq_mb_stats_show()
2631 seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[1])); in ext4_seq_mb_stats_show()
2633 atomic64_read(&sbi->s_bal_cX_groups_considered[1])); in ext4_seq_mb_stats_show()
2635 atomic64_read(&sbi->s_bal_cX_failed[1])); in ext4_seq_mb_stats_show()
2687 size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> in ext4_mb_alloc_groupinfo()
2744 i = group & (EXT4_DESC_PER_BLOCK(sb) - 1); in ext4_mb_add_groupinfo()
2770 meta_group_info[i]->bb_largest_free_order = -1; /* uninit */ in ext4_mb_add_groupinfo()
2835 sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex, in ext4_mb_init_backend()
2958 i = 1; in ext4_mb_init()
2960 offset_incr = 1 << (sb->s_blocksize_bits - 1); in ext4_mb_init()
2966 offset_incr = offset_incr >> 1; in ext4_mb_init()
2967 max = max >> 1; in ext4_mb_init()
2969 } while (i <= sb->s_blocksize_bits + 1); in ext4_mb_init()
2985 * is 1 megabyte, then group preallocation size becomes half a in ext4_mb_init()
2990 * size is 256k, and 32 megs when the cluster size is 1 meg, in ext4_mb_init()
2996 * If there is a s_stripe > 1, then we set the s_mb_group_prealloc in ext4_mb_init()
3003 if (sbi->s_stripe > 1) { in ext4_mb_init()
3082 EXT4_DESC_PER_BLOCK(sb) - 1) >> in ext4_mb_release()
3289 * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
3293 ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, in ext4_mb_mark_diskspace_used() argument
3304 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_mark_diskspace_used()
3305 BUG_ON(ac->ac_b_ex.fe_len <= 0); in ext4_mb_mark_diskspace_used()
3307 sb = ac->ac_sb; in ext4_mb_mark_diskspace_used()
3310 bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3323 gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); in ext4_mb_mark_diskspace_used()
3327 ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, in ext4_mb_mark_diskspace_used()
3335 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_mark_diskspace_used()
3337 len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3338 if (!ext4_inode_block_valid(ac->ac_inode, block, len)) { in ext4_mb_mark_diskspace_used()
3345 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3346 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
3347 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3348 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3355 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3359 for (i = 0; i < ac->ac_b_ex.fe_len; i++) { in ext4_mb_mark_diskspace_used()
3360 BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, in ext4_mb_mark_diskspace_used()
3365 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
3366 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3372 ac->ac_b_ex.fe_group, gdp)); in ext4_mb_mark_diskspace_used()
3374 len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; in ext4_mb_mark_diskspace_used()
3376 ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh); in ext4_mb_mark_diskspace_used()
3377 ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp); in ext4_mb_mark_diskspace_used()
3379 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3380 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3384 if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) in ext4_mb_mark_diskspace_used()
3391 ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3392 atomic64_sub(ac->ac_b_ex.fe_len, in ext4_mb_mark_diskspace_used()
3520 static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) in ext4_mb_normalize_group_request() argument
3522 struct super_block *sb = ac->ac_sb; in ext4_mb_normalize_group_request()
3523 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_normalize_group_request()
3526 ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; in ext4_mb_normalize_group_request()
3527 mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len); in ext4_mb_normalize_group_request()
3535 ext4_mb_normalize_request(struct ext4_allocation_context *ac, in ext4_mb_normalize_request() argument
3538 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_normalize_request()
3544 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_normalize_request()
3549 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_normalize_request()
3553 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_normalize_request()
3558 if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) in ext4_mb_normalize_request()
3561 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { in ext4_mb_normalize_request()
3562 ext4_mb_normalize_group_request(ac); in ext4_mb_normalize_request()
3566 bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_normalize_request()
3570 size = extent_logical_end(sbi, &ac->ac_o_ex); in ext4_mb_normalize_request()
3572 if (size < i_size_read(ac->ac_inode)) in ext4_mb_normalize_request()
3573 size = i_size_read(ac->ac_inode); in ext4_mb_normalize_request()
3600 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
3604 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
3607 } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, in ext4_mb_normalize_request()
3609 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
3613 start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; in ext4_mb_normalize_request()
3614 size = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
3615 ac->ac_o_ex.fe_len) << bsbits; in ext4_mb_normalize_request()
3626 start = max(start, rounddown(ac->ac_o_ex.fe_logical, in ext4_mb_normalize_request()
3627 (ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb))); in ext4_mb_normalize_request()
3635 size -= ar->lleft + 1 - start; in ext4_mb_normalize_request()
3636 start = ar->lleft + 1; in ext4_mb_normalize_request()
3638 if (ar->pright && start + size - 1 >= ar->lright) in ext4_mb_normalize_request()
3645 if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) in ext4_mb_normalize_request()
3646 size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb); in ext4_mb_normalize_request()
3663 pa_end = pa_logical_end(EXT4_SB(ac->ac_sb), pa); in ext4_mb_normalize_request()
3666 BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || in ext4_mb_normalize_request()
3667 ac->ac_o_ex.fe_logical < pa->pa_lstart)); in ext4_mb_normalize_request()
3677 if (pa_end <= ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3680 } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3696 pa_end = pa_logical_end(EXT4_SB(ac->ac_sb), pa); in ext4_mb_normalize_request()
3703 if (start + size <= ac->ac_o_ex.fe_logical && in ext4_mb_normalize_request()
3704 start > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3705 ext4_msg(ac->ac_sb, KERN_ERR, in ext4_mb_normalize_request()
3708 (unsigned long) ac->ac_o_ex.fe_logical); in ext4_mb_normalize_request()
3711 BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); in ext4_mb_normalize_request()
3717 ac->ac_g_ex.fe_logical = start; in ext4_mb_normalize_request()
3718 ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); in ext4_mb_normalize_request()
3725 ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, in ext4_mb_normalize_request()
3726 &ac->ac_g_ex.fe_group, in ext4_mb_normalize_request()
3727 &ac->ac_g_ex.fe_start); in ext4_mb_normalize_request()
3728 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
3730 if (ar->pleft && (ar->lleft + 1 == start) && in ext4_mb_normalize_request()
3731 ar->pleft + 1 < ext4_blocks_count(es)) { in ext4_mb_normalize_request()
3733 ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, in ext4_mb_normalize_request()
3734 &ac->ac_g_ex.fe_group, in ext4_mb_normalize_request()
3735 &ac->ac_g_ex.fe_start); in ext4_mb_normalize_request()
3736 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
3739 mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size, in ext4_mb_normalize_request()
3743 static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) in ext4_mb_collect_stats() argument
3745 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_collect_stats()
3747 if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) { in ext4_mb_collect_stats()
3749 atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); in ext4_mb_collect_stats()
3750 if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) in ext4_mb_collect_stats()
3752 atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); in ext4_mb_collect_stats()
3753 atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned); in ext4_mb_collect_stats()
3754 if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && in ext4_mb_collect_stats()
3755 ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) in ext4_mb_collect_stats()
3757 if (ac->ac_found > sbi->s_mb_max_to_scan) in ext4_mb_collect_stats()
3761 if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) in ext4_mb_collect_stats()
3762 trace_ext4_mballoc_alloc(ac); in ext4_mb_collect_stats()
3764 trace_ext4_mballoc_prealloc(ac); in ext4_mb_collect_stats()
3771 * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
3773 static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) in ext4_discard_allocated_blocks() argument
3775 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_discard_allocated_blocks()
3780 if (ac->ac_f_ex.fe_len == 0) in ext4_discard_allocated_blocks()
3782 err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b); in ext4_discard_allocated_blocks()
3789 WARN(1, "mb_load_buddy failed (%d)", err); in ext4_discard_allocated_blocks()
3792 ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
3793 mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start, in ext4_discard_allocated_blocks()
3794 ac->ac_f_ex.fe_len); in ext4_discard_allocated_blocks()
3795 ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
3800 pa->pa_free += ac->ac_b_ex.fe_len; in ext4_discard_allocated_blocks()
3806 static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, in ext4_mb_use_inode_pa() argument
3809 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_inode_pa()
3815 start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); in ext4_mb_use_inode_pa()
3817 start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); in ext4_mb_use_inode_pa()
3819 ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, in ext4_mb_use_inode_pa()
3820 &ac->ac_b_ex.fe_start); in ext4_mb_use_inode_pa()
3821 ac->ac_b_ex.fe_len = len; in ext4_mb_use_inode_pa()
3822 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_inode_pa()
3823 ac->ac_pa = pa; in ext4_mb_use_inode_pa()
3828 BUG_ON(ac->ac_b_ex.fe_len <= 0); in ext4_mb_use_inode_pa()
3831 mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa); in ext4_mb_use_inode_pa()
3837 static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, in ext4_mb_use_group_pa() argument
3840 unsigned int len = ac->ac_o_ex.fe_len; in ext4_mb_use_group_pa()
3842 ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, in ext4_mb_use_group_pa()
3843 &ac->ac_b_ex.fe_group, in ext4_mb_use_group_pa()
3844 &ac->ac_b_ex.fe_start); in ext4_mb_use_group_pa()
3845 ac->ac_b_ex.fe_len = len; in ext4_mb_use_group_pa()
3846 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_group_pa()
3847 ac->ac_pa = pa; in ext4_mb_use_group_pa()
3855 mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n", in ext4_mb_use_group_pa()
3892 ext4_mb_use_preallocated(struct ext4_allocation_context *ac) in ext4_mb_use_preallocated() argument
3894 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_preallocated()
3896 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_use_preallocated()
3902 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_use_preallocated()
3911 if (ac->ac_o_ex.fe_logical < pa->pa_lstart || in ext4_mb_use_preallocated()
3912 ac->ac_o_ex.fe_logical >= pa_logical_end(sbi, pa)) in ext4_mb_use_preallocated()
3916 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && in ext4_mb_use_preallocated()
3925 ext4_mb_use_inode_pa(ac, pa); in ext4_mb_use_preallocated()
3927 ac->ac_criteria = 10; in ext4_mb_use_preallocated()
3936 if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) in ext4_mb_use_preallocated()
3940 lg = ac->ac_lg; in ext4_mb_use_preallocated()
3943 order = fls(ac->ac_o_ex.fe_len) - 1; in ext4_mb_use_preallocated()
3944 if (order > PREALLOC_TB_SIZE - 1) in ext4_mb_use_preallocated()
3946 order = PREALLOC_TB_SIZE - 1; in ext4_mb_use_preallocated()
3948 goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); in ext4_mb_use_preallocated()
3959 pa->pa_free >= ac->ac_o_ex.fe_len) { in ext4_mb_use_preallocated()
3969 ext4_mb_use_group_pa(ac, cpa); in ext4_mb_use_preallocated()
3970 ac->ac_criteria = 20; in ext4_mb_use_preallocated()
4058 pa->pa_deleted = 1; in ext4_mb_mark_pa_deleted()
4080 static void ext4_mb_put_pa(struct ext4_allocation_context *ac, in ext4_mb_put_pa() argument
4093 if (pa->pa_deleted == 1) { in ext4_mb_put_pa()
4140 ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) in ext4_mb_new_inode_pa() argument
4142 struct super_block *sb = ac->ac_sb; in ext4_mb_new_inode_pa()
4149 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4150 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_inode_pa()
4151 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_inode_pa()
4152 BUG_ON(ac->ac_pa == NULL); in ext4_mb_new_inode_pa()
4154 pa = ac->ac_pa; in ext4_mb_new_inode_pa()
4156 if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { in ext4_mb_new_inode_pa()
4158 .fe_logical = ac->ac_g_ex.fe_logical, in ext4_mb_new_inode_pa()
4159 .fe_len = ac->ac_g_ex.fe_len, in ext4_mb_new_inode_pa()
4166 BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); in ext4_mb_new_inode_pa()
4167 BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); in ext4_mb_new_inode_pa()
4174 * 1. Check if best ex can be kept at end of goal and still in ext4_mb_new_inode_pa()
4180 ex.fe_len = ac->ac_b_ex.fe_len; in ext4_mb_new_inode_pa()
4183 if (ac->ac_o_ex.fe_logical >= ex.fe_logical) in ext4_mb_new_inode_pa()
4186 ex.fe_logical = ac->ac_g_ex.fe_logical; in ext4_mb_new_inode_pa()
4187 if (ac->ac_o_ex.fe_logical < extent_logical_end(sbi, &ex)) in ext4_mb_new_inode_pa()
4190 ex.fe_logical = ac->ac_o_ex.fe_logical; in ext4_mb_new_inode_pa()
4192 ac->ac_b_ex.fe_logical = ex.fe_logical; in ext4_mb_new_inode_pa()
4194 BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); in ext4_mb_new_inode_pa()
4195 BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4201 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_new_inode_pa()
4203 pa->pa_lstart = ac->ac_b_ex.fe_logical; in ext4_mb_new_inode_pa()
4204 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_inode_pa()
4205 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_inode_pa()
4215 trace_ext4_mb_new_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
4217 ext4_mb_use_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
4220 ei = EXT4_I(ac->ac_inode); in ext4_mb_new_inode_pa()
4221 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_inode_pa()
4226 pa->pa_inode = ac->ac_inode; in ext4_mb_new_inode_pa()
4240 ext4_mb_new_group_pa(struct ext4_allocation_context *ac) in ext4_mb_new_group_pa() argument
4242 struct super_block *sb = ac->ac_sb; in ext4_mb_new_group_pa()
4248 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_group_pa()
4249 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_group_pa()
4250 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_group_pa()
4251 BUG_ON(ac->ac_pa == NULL); in ext4_mb_new_group_pa()
4253 pa = ac->ac_pa; in ext4_mb_new_group_pa()
4257 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_new_group_pa()
4259 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_group_pa()
4261 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_group_pa()
4271 trace_ext4_mb_new_group_pa(ac, pa); in ext4_mb_new_group_pa()
4273 ext4_mb_use_group_pa(ac, pa); in ext4_mb_new_group_pa()
4276 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_group_pa()
4279 lg = ac->ac_lg; in ext4_mb_new_group_pa()
4293 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac) in ext4_mb_new_preallocation() argument
4295 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_new_preallocation()
4296 ext4_mb_new_group_pa(ac); in ext4_mb_new_preallocation()
4298 ext4_mb_new_inode_pa(ac); in ext4_mb_new_preallocation()
4343 bit = next + 1; in ext4_mb_release_inode_pa()
4391 * 1) ENOSPC
4393 * 1) how many requested
4437 *busy = 1; in ext4_mb_discard_group_preallocations()
4539 WARN_ON(1); in ext4_discard_preallocations()
4608 static int ext4_mb_pa_alloc(struct ext4_allocation_context *ac) in ext4_mb_pa_alloc() argument
4616 atomic_set(&pa->pa_count, 1); in ext4_mb_pa_alloc()
4617 ac->ac_pa = pa; in ext4_mb_pa_alloc()
4621 static void ext4_mb_pa_free(struct ext4_allocation_context *ac) in ext4_mb_pa_free() argument
4623 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_pa_free()
4626 ac->ac_pa = NULL; in ext4_mb_pa_free()
4666 static void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
4668 struct super_block *sb = ac->ac_sb; in ext4_mb_show_ac()
4676 ac->ac_status, ac->ac_flags); in ext4_mb_show_ac()
4680 (unsigned long)ac->ac_o_ex.fe_group, in ext4_mb_show_ac()
4681 (unsigned long)ac->ac_o_ex.fe_start, in ext4_mb_show_ac()
4682 (unsigned long)ac->ac_o_ex.fe_len, in ext4_mb_show_ac()
4683 (unsigned long)ac->ac_o_ex.fe_logical, in ext4_mb_show_ac()
4684 (unsigned long)ac->ac_g_ex.fe_group, in ext4_mb_show_ac()
4685 (unsigned long)ac->ac_g_ex.fe_start, in ext4_mb_show_ac()
4686 (unsigned long)ac->ac_g_ex.fe_len, in ext4_mb_show_ac()
4687 (unsigned long)ac->ac_g_ex.fe_logical, in ext4_mb_show_ac()
4688 (unsigned long)ac->ac_b_ex.fe_group, in ext4_mb_show_ac()
4689 (unsigned long)ac->ac_b_ex.fe_start, in ext4_mb_show_ac()
4690 (unsigned long)ac->ac_b_ex.fe_len, in ext4_mb_show_ac()
4691 (unsigned long)ac->ac_b_ex.fe_logical, in ext4_mb_show_ac()
4692 (int)ac->ac_criteria); in ext4_mb_show_ac()
4693 mb_debug(sb, "%u found", ac->ac_found); in ext4_mb_show_ac()
4701 static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
4703 ext4_mb_show_pa(ac->ac_sb); in ext4_mb_show_ac()
4715 static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) in ext4_mb_group_or_file() argument
4717 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_group_or_file()
4718 int bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_group_or_file()
4721 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_group_or_file()
4724 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_group_or_file()
4727 size = extent_logical_end(sbi, &ac->ac_o_ex); in ext4_mb_group_or_file()
4728 isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) in ext4_mb_group_or_file()
4732 !inode_is_open_for_write(ac->ac_inode)) { in ext4_mb_group_or_file()
4733 ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; in ext4_mb_group_or_file()
4738 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
4745 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
4749 BUG_ON(ac->ac_lg != NULL); in ext4_mb_group_or_file()
4755 ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); in ext4_mb_group_or_file()
4758 ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; in ext4_mb_group_or_file()
4761 mutex_lock(&ac->ac_lg->lg_mutex); in ext4_mb_group_or_file()
4765 ext4_mb_initialize_context(struct ext4_allocation_context *ac, in ext4_mb_initialize_context() argument
4791 ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); in ext4_mb_initialize_context()
4792 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_initialize_context()
4793 ac->ac_sb = sb; in ext4_mb_initialize_context()
4794 ac->ac_inode = ar->inode; in ext4_mb_initialize_context()
4795 ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; in ext4_mb_initialize_context()
4796 ac->ac_o_ex.fe_group = group; in ext4_mb_initialize_context()
4797 ac->ac_o_ex.fe_start = block; in ext4_mb_initialize_context()
4798 ac->ac_o_ex.fe_len = len; in ext4_mb_initialize_context()
4799 ac->ac_g_ex = ac->ac_o_ex; in ext4_mb_initialize_context()
4800 ac->ac_flags = ar->flags; in ext4_mb_initialize_context()
4804 ext4_mb_group_or_file(ac); in ext4_mb_initialize_context()
4806 mb_debug(sb, "init ac: %u blocks @ %u, goal %u, flags 0x%x, 2^%d, " in ext4_mb_initialize_context()
4809 (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, in ext4_mb_initialize_context()
4903 static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) in ext4_mb_add_n_trim() argument
4905 int order, added = 0, lg_prealloc_count = 1; in ext4_mb_add_n_trim()
4906 struct super_block *sb = ac->ac_sb; in ext4_mb_add_n_trim()
4907 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_add_n_trim()
4908 struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; in ext4_mb_add_n_trim()
4910 order = fls(pa->pa_free) - 1; in ext4_mb_add_n_trim()
4911 if (order > PREALLOC_TB_SIZE - 1) in ext4_mb_add_n_trim()
4913 order = PREALLOC_TB_SIZE - 1; in ext4_mb_add_n_trim()
4928 added = 1; in ext4_mb_add_n_trim()
4961 delta = (sbi->s_mb_max_inode_prealloc >> 2) + 1; in ext4_mb_trim_inode_pa()
4971 static int ext4_mb_release_context(struct ext4_allocation_context *ac) in ext4_mb_release_context() argument
4973 struct inode *inode = ac->ac_inode; in ext4_mb_release_context()
4975 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_release_context()
4976 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_release_context()
4981 pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
4982 pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
4983 pa->pa_free -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
4984 pa->pa_len -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
4997 ext4_mb_add_n_trim(ac); in ext4_mb_release_context()
5011 ext4_mb_put_pa(ac, ac->ac_sb, pa); in ext4_mb_release_context()
5013 if (ac->ac_bitmap_page) in ext4_mb_release_context()
5014 put_page(ac->ac_bitmap_page); in ext4_mb_release_context()
5015 if (ac->ac_buddy_page) in ext4_mb_release_context()
5016 put_page(ac->ac_buddy_page); in ext4_mb_release_context()
5017 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_release_context()
5018 mutex_unlock(&ac->ac_lg->lg_mutex); in ext4_mb_release_context()
5019 ext4_mb_collect_stats(ac); in ext4_mb_release_context()
5034 needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1; in ext4_mb_discard_preallocations()
5052 struct ext4_allocation_context *ac, u64 *seq) in ext4_mb_discard_preallocations_should_retry() argument
5058 freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); in ext4_mb_discard_preallocations_should_retry()
5064 if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) { in ext4_mb_discard_preallocations_should_retry()
5065 ac->ac_flags |= EXT4_MB_STRICT_CHECK; in ext4_mb_discard_preallocations_should_retry()
5086 struct ext4_allocation_context *ac = NULL; in ext4_mb_new_blocks() local
5117 ar->len = ar->len >> 1; in ext4_mb_new_blocks()
5144 ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS); in ext4_mb_new_blocks()
5145 if (!ac) { in ext4_mb_new_blocks()
5151 *errp = ext4_mb_initialize_context(ac, ar); in ext4_mb_new_blocks()
5157 ac->ac_op = EXT4_MB_HISTORY_PREALLOC; in ext4_mb_new_blocks()
5159 if (!ext4_mb_use_preallocated(ac)) { in ext4_mb_new_blocks()
5160 ac->ac_op = EXT4_MB_HISTORY_ALLOC; in ext4_mb_new_blocks()
5161 ext4_mb_normalize_request(ac, ar); in ext4_mb_new_blocks()
5163 *errp = ext4_mb_pa_alloc(ac); in ext4_mb_new_blocks()
5168 *errp = ext4_mb_regular_allocator(ac); in ext4_mb_new_blocks()
5172 * ac->ac_status == AC_STATUS_FOUND. in ext4_mb_new_blocks()
5173 * And error from above mean ac->ac_status != AC_STATUS_FOUND in ext4_mb_new_blocks()
5177 ext4_mb_pa_free(ac); in ext4_mb_new_blocks()
5178 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
5181 if (ac->ac_status == AC_STATUS_FOUND && in ext4_mb_new_blocks()
5182 ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len) in ext4_mb_new_blocks()
5183 ext4_mb_pa_free(ac); in ext4_mb_new_blocks()
5185 if (likely(ac->ac_status == AC_STATUS_FOUND)) { in ext4_mb_new_blocks()
5186 *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); in ext4_mb_new_blocks()
5188 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
5191 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_blocks()
5192 ar->len = ac->ac_b_ex.fe_len; in ext4_mb_new_blocks()
5196 ext4_mb_discard_preallocations_should_retry(sb, ac, &seq)) in ext4_mb_new_blocks()
5202 ext4_mb_pa_free(ac); in ext4_mb_new_blocks()
5208 ac->ac_b_ex.fe_len = 0; in ext4_mb_new_blocks()
5210 ext4_mb_show_ac(ac); in ext4_mb_new_blocks()
5212 ext4_mb_release_context(ac); in ext4_mb_new_blocks()
5214 if (ac) in ext4_mb_new_blocks()
5215 kmem_cache_free(ext4_ac_cachep, ac); in ext4_mb_new_blocks()
5366 while (1) { in ext4_mb_new_blocks_simple()
5373 blkoff = i + 1; in ext4_mb_new_blocks_simple()
5388 ext4_mb_mark_bb(sb, block, 1, 1); in ext4_mb_new_blocks_simple()
5389 ar->len = 1; in ext4_mb_new_blocks_simple()
5684 BUG_ON(count > 1); in ext4_free_blocks()
5765 ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1); in ext4_group_add_blocks()
5766 unsigned long cluster_count = last_cluster - first_cluster + 1; in ext4_group_add_blocks()
5768 ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1); in ext4_group_add_blocks()
5923 if (grp < (ext4_get_groups_count(sb) - 1)) in ext4_last_grp_cluster()
5930 return nr_clusters_in_group - 1; in ext4_last_grp_cluster()
5956 start = mb_find_next_zero_bit(bitmap, max + 1, start); in ext4_try_to_trim_range()
5960 next = mb_find_next_bit(bitmap, last + 1, start); in ext4_try_to_trim_range()
5972 start = next + 1; in ext4_try_to_trim_range()
6070 end = start + (range->len >> sb->s_blocksize_bits) - 1; in ext4_trim_fs()
6085 if (end >= max_blks - 1) in ext4_trim_fs()
6086 end = max_blks - 1; in ext4_trim_fs()
6099 end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; in ext4_trim_fs()
6116 * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to in ext4_trim_fs()
6171 end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; in ext4_mballoc_query_range()
6174 start = mb_find_next_zero_bit(bitmap, end + 1, start); in ext4_mballoc_query_range()
6177 next = mb_find_next_bit(bitmap, end + 1, start); in ext4_mballoc_query_range()
6185 start = next + 1; in ext4_mballoc_query_range()