Lines Matching +full:locality +full:- +full:specific
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
18 #include <linux/backing-dev.h>
23 * - test ext4_ext_search_left() and ext4_ext_search_right()
24 * - search for metadata in few groups
27 * - normalization should take into account whether file is still open
28 * - discard preallocations if no free space left (policy?)
29 * - don't normalize tails
30 * - quota
31 * - reservation for superuser
34 * - bitmap read-ahead (proposed by Oleg Drokin aka green)
35 * - track min/max extents in each group for better group selection
36 * - mb_mark_used() may allocate chunk right after splitting buddy
37 * - tree of groups sorted by number of free blocks
38 * - error handling
49 * is larger. If the size is less than sbi->s_mb_stream_request we
59 * ext4_inode_info->i_prealloc_list, which contains list of prealloc
63 * pa_lstart -> the logical start block for this prealloc space
64 * pa_pstart -> the physical start block for this prealloc space
65 * pa_len -> length for this prealloc space (in clusters)
66 * pa_free -> free space available in this prealloc space (in clusters)
78 * have the group allocation flag set then we look at the locality group
83 * The reason for having a per cpu locality group is to reduce the contention
86 * The locality group prealloc space is used looking at whether we have
89 * If we can't allocate blocks via inode prealloc or/and locality group
121 * sbi->s_mb_group_prealloc. The default value of s_mb_group_prealloc is
122 * dependent on the cluster size; for non-bigalloc file systems, it is
125 * terms of number of blocks. If we have mounted the file system with -O
127 * smallest multiple of the stripe value (sbi->s_stripe) which is
137 * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The
140 * stripe size (sbi->s_stripe), we try to search for contiguous block in
142 * not, we search in the specific group using bitmap for best extents. The
160 * - on-disk bitmap
161 * - in-core buddy (actually includes buddy and bitmap)
162 * - preallocation descriptors (PAs)
165 * - inode
166 * assiged to specific inode and can be used for this inode only.
167 * it describes part of inode's space preallocated to specific
174 * - locality group
175 * assigned to specific locality group which does not translate to
181 * in-core buddy = on-disk bitmap + preallocation descriptors
184 * - allocated blocks (persistent)
185 * - preallocated blocks (non-persistent)
189 * literally -- time is discrete and delimited by locks.
192 * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA.
195 * - init buddy: buddy = on-disk + PAs
196 * - new PA: buddy += N; PA = N
197 * - use inode PA: on-disk += N; PA -= N
198 * - discard inode PA buddy -= on-disk - PA; PA = 0
199 * - use locality group PA on-disk += N; PA -= N
200 * - discard locality group PA buddy -= PA; PA = 0
201 * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap
203 * bits from PA, only from on-disk bitmap
207 * killing performance on high-end SMP hardware. let's try to relax it using
211 * nobody can re-allocate that block
212 * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has
214 * on-disk bitmap if buddy has same bit set or/and PA covers corresponded
218 * - init buddy vs.
219 * - new PA
222 * - use inode PA
223 * we need to make sure that either on-disk bitmap or PA has uptodate data
224 * given (3) we care that PA-=N operation doesn't interfere with init
225 * - discard inode PA
227 * - use locality group PA
228 * again PA-=N must be serialized with init
229 * - discard locality group PA
231 * - new PA vs.
232 * - use inode PA
234 * - discard inode PA
236 * - use locality group PA
238 * - discard locality group PA
240 * - use inode PA
241 * - use inode PA
243 * - discard inode PA
245 * - use locality group PA
246 * nothing wrong here -- they're different PAs covering different blocks
247 * - discard locality group PA
251 * - PA is referenced and while it is no discard is possible
252 * - PA is referenced until block isn't marked in on-disk bitmap
253 * - PA changes only after on-disk bitmap
254 * - discard must not compete with init. either init is done before
256 * - buddy init as sum of on-disk bitmap and PAs is done atomically
266 * - allocation:
269 * mark bits in on-disk bitmap
272 * - use preallocation:
273 * find proper PA (per-inode or group)
275 * mark bits in on-disk bitmap
279 * - free:
281 * mark bits in on-disk bitmap
284 * - discard preallocations in group:
287 * load on-disk bitmap
289 * remove PA from object (inode or locality group)
290 * mark free blocks in-core
292 * - discard inode's preallocations:
299 * - bitlock on a group (group)
300 * - object (inode/locality) (object)
301 * - per-pa lock (pa)
304 * - new pa
308 * - find and use pa:
311 * - release consumed pa:
316 * - generate in-core bitmap:
320 * - discard all for given object (inode, locality group):
325 * - discard all for given group:
370 * sure that all the PAs on grp->bb_prealloc_list got freed or if it's empty.
432 ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix; in mb_find_next_zero_bit()
445 ret = ext4_find_next_bit(addr, tmpmax, start) - fix; in mb_find_next_bit()
455 BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); in mb_find_buddy()
458 if (order > e4b->bd_blkbits + 1) { in mb_find_buddy()
465 *max = 1 << (e4b->bd_blkbits + 3); in mb_find_buddy()
466 return e4b->bd_bitmap; in mb_find_buddy()
469 bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order]; in mb_find_buddy()
470 *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order]; in mb_find_buddy()
480 struct super_block *sb = e4b->bd_sb; in mb_free_blocks_double()
482 if (unlikely(e4b->bd_info->bb_bitmap == NULL)) in mb_free_blocks_double()
484 assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); in mb_free_blocks_double()
486 if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) { in mb_free_blocks_double()
489 blocknr = ext4_group_first_block_no(sb, e4b->bd_group); in mb_free_blocks_double()
491 ext4_grp_locked_error(sb, e4b->bd_group, in mb_free_blocks_double()
492 inode ? inode->i_ino : 0, in mb_free_blocks_double()
497 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, in mb_free_blocks_double()
500 mb_clear_bit(first + i, e4b->bd_info->bb_bitmap); in mb_free_blocks_double()
508 if (unlikely(e4b->bd_info->bb_bitmap == NULL)) in mb_mark_used_double()
510 assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); in mb_mark_used_double()
512 BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap)); in mb_mark_used_double()
513 mb_set_bit(first + i, e4b->bd_info->bb_bitmap); in mb_mark_used_double()
519 if (unlikely(e4b->bd_info->bb_bitmap == NULL)) in mb_cmp_bitmaps()
521 if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) { in mb_cmp_bitmaps()
524 b1 = (unsigned char *) e4b->bd_info->bb_bitmap; in mb_cmp_bitmaps()
526 for (i = 0; i < e4b->bd_sb->s_blocksize; i++) { in mb_cmp_bitmaps()
528 ext4_msg(e4b->bd_sb, KERN_ERR, in mb_cmp_bitmaps()
532 e4b->bd_group, i, i * 8, b1[i], b2[i]); in mb_cmp_bitmaps()
544 grp->bb_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS); in mb_group_bb_bitmap_alloc()
545 if (!grp->bb_bitmap) in mb_group_bb_bitmap_alloc()
550 kfree(grp->bb_bitmap); in mb_group_bb_bitmap_alloc()
551 grp->bb_bitmap = NULL; in mb_group_bb_bitmap_alloc()
555 memcpy(grp->bb_bitmap, bh->b_data, sb->s_blocksize); in mb_group_bb_bitmap_alloc()
561 kfree(grp->bb_bitmap); in mb_group_bb_bitmap_free()
607 struct super_block *sb = e4b->bd_sb; in __mb_check_buddy()
608 int order = e4b->bd_blkbits + 1; in __mb_check_buddy()
622 if (e4b->bd_info->bb_check_counter++ % 10) in __mb_check_buddy()
628 buddy2 = mb_find_buddy(e4b, order - 1, &max2); in __mb_check_buddy()
655 !mb_test_bit(k, e4b->bd_bitmap)); in __mb_check_buddy()
659 MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count); in __mb_check_buddy()
660 order--; in __mb_check_buddy()
663 fstart = -1; in __mb_check_buddy()
667 MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free); in __mb_check_buddy()
668 if (fstart == -1) { in __mb_check_buddy()
674 fstart = -1; in __mb_check_buddy()
676 for (j = 0; j < e4b->bd_blkbits + 1; j++) { in __mb_check_buddy()
683 MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info)); in __mb_check_buddy()
684 MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments); in __mb_check_buddy()
686 grp = ext4_get_group_info(sb, e4b->bd_group); in __mb_check_buddy()
689 list_for_each(cur, &grp->bb_prealloc_list) { in __mb_check_buddy()
693 ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k); in __mb_check_buddy()
694 MB_CHECK_ASSERT(groupnr == e4b->bd_group); in __mb_check_buddy()
695 for (i = 0; i < pa->pa_len; i++) in __mb_check_buddy()
725 border = 2 << sb->s_blocksize_bits; in ext4_mb_mark_free_simple()
729 max = ffs(first | border) - 1; in ext4_mb_mark_free_simple()
732 min = fls(len) - 1; in ext4_mb_mark_free_simple()
739 grp->bb_counters[min]++; in ext4_mb_mark_free_simple()
742 buddy + sbi->s_mb_offsets[min]); in ext4_mb_mark_free_simple()
744 len -= chunk; in ext4_mb_mark_free_simple()
759 grp->bb_largest_free_order = -1; /* uninit */ in mb_set_largest_free_order()
761 bits = sb->s_blocksize_bits + 1; in mb_set_largest_free_order()
762 for (i = bits; i >= 0; i--) { in mb_set_largest_free_order()
763 if (grp->bb_counters[i] > 0) { in mb_set_largest_free_order()
764 grp->bb_largest_free_order = i; in mb_set_largest_free_order()
785 * of on-disk bitmap and preallocations */ in ext4_mb_generate_buddy()
787 grp->bb_first_free = i; in ext4_mb_generate_buddy()
792 len = i - first; in ext4_mb_generate_buddy()
797 grp->bb_counters[0]++; in ext4_mb_generate_buddy()
801 grp->bb_fragments = fragments; in ext4_mb_generate_buddy()
803 if (free != grp->bb_free) { in ext4_mb_generate_buddy()
807 free, grp->bb_free); in ext4_mb_generate_buddy()
812 grp->bb_free = free; in ext4_mb_generate_buddy()
818 clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state)); in ext4_mb_generate_buddy()
820 period = get_cycles() - period; in ext4_mb_generate_buddy()
821 atomic_inc(&sbi->s_mb_buddies_generated); in ext4_mb_generate_buddy()
822 atomic64_add(period, &sbi->s_mb_generation_time); in ext4_mb_generate_buddy()
834 e4b->bd_info->bb_fragments = 0; in mb_regenerate_buddy()
835 memset(e4b->bd_info->bb_counters, 0, in mb_regenerate_buddy()
836 sizeof(*e4b->bd_info->bb_counters) * in mb_regenerate_buddy()
837 (e4b->bd_sb->s_blocksize_bits + 2)); in mb_regenerate_buddy()
839 ext4_mb_generate_buddy(e4b->bd_sb, e4b->bd_buddy, in mb_regenerate_buddy()
840 e4b->bd_bitmap, e4b->bd_group, e4b->bd_info); in mb_regenerate_buddy()
881 inode = page->mapping->host; in ext4_mb_init_cache()
882 sb = inode->i_sb; in ext4_mb_init_cache()
887 mb_debug(sb, "init page %lu\n", page->index); in ext4_mb_init_cache()
898 err = -ENOMEM; in ext4_mb_init_cache()
904 first_group = page->index * blocks_per_page / 2; in ext4_mb_init_cache()
944 first_block = page->index * blocks_per_page; in ext4_mb_init_cache()
950 if (!bh[group - first_group]) in ext4_mb_init_cache()
954 if (!buffer_verified(bh[group - first_group])) in ext4_mb_init_cache()
966 bitmap = bh[group - first_group]->b_data; in ext4_mb_init_cache()
976 group, page->index, i * blocksize); in ext4_mb_init_cache()
980 err = -EFSCORRUPTED; in ext4_mb_init_cache()
983 grinfo->bb_fragments = 0; in ext4_mb_init_cache()
984 memset(grinfo->bb_counters, 0, in ext4_mb_init_cache()
985 sizeof(*grinfo->bb_counters) * in ext4_mb_init_cache()
986 (sb->s_blocksize_bits+2)); in ext4_mb_init_cache()
1000 group, page->index, i * blocksize); in ext4_mb_init_cache()
1007 /* mark all preallocated blks used in in-core bitmap */ in ext4_mb_init_cache()
1034 * are on the same page e4b->bd_buddy_page is NULL and return value is 0.
1039 struct inode *inode = EXT4_SB(sb)->s_buddy_cache; in ext4_mb_get_buddy_page_lock()
1044 e4b->bd_buddy_page = NULL; in ext4_mb_get_buddy_page_lock()
1045 e4b->bd_bitmap_page = NULL; in ext4_mb_get_buddy_page_lock()
1047 blocks_per_page = PAGE_SIZE / sb->s_blocksize; in ext4_mb_get_buddy_page_lock()
1056 page = find_or_create_page(inode->i_mapping, pnum, gfp); in ext4_mb_get_buddy_page_lock()
1058 return -ENOMEM; in ext4_mb_get_buddy_page_lock()
1059 BUG_ON(page->mapping != inode->i_mapping); in ext4_mb_get_buddy_page_lock()
1060 e4b->bd_bitmap_page = page; in ext4_mb_get_buddy_page_lock()
1061 e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); in ext4_mb_get_buddy_page_lock()
1070 page = find_or_create_page(inode->i_mapping, pnum, gfp); in ext4_mb_get_buddy_page_lock()
1072 return -ENOMEM; in ext4_mb_get_buddy_page_lock()
1073 BUG_ON(page->mapping != inode->i_mapping); in ext4_mb_get_buddy_page_lock()
1074 e4b->bd_buddy_page = page; in ext4_mb_get_buddy_page_lock()
1080 if (e4b->bd_bitmap_page) { in ext4_mb_put_buddy_page_lock()
1081 unlock_page(e4b->bd_bitmap_page); in ext4_mb_put_buddy_page_lock()
1082 put_page(e4b->bd_bitmap_page); in ext4_mb_put_buddy_page_lock()
1084 if (e4b->bd_buddy_page) { in ext4_mb_put_buddy_page_lock()
1085 unlock_page(e4b->bd_buddy_page); in ext4_mb_put_buddy_page_lock()
1086 put_page(e4b->bd_buddy_page); in ext4_mb_put_buddy_page_lock()
1108 return -EFSCORRUPTED; in ext4_mb_init_group()
1133 ret = -EIO; in ext4_mb_init_group()
1152 ret = -EIO; in ext4_mb_init_group()
1177 struct inode *inode = sbi->s_buddy_cache; in ext4_mb_load_buddy_gfp()
1182 blocks_per_page = PAGE_SIZE / sb->s_blocksize; in ext4_mb_load_buddy_gfp()
1185 return -EFSCORRUPTED; in ext4_mb_load_buddy_gfp()
1187 e4b->bd_blkbits = sb->s_blocksize_bits; in ext4_mb_load_buddy_gfp()
1188 e4b->bd_info = grp; in ext4_mb_load_buddy_gfp()
1189 e4b->bd_sb = sb; in ext4_mb_load_buddy_gfp()
1190 e4b->bd_group = group; in ext4_mb_load_buddy_gfp()
1191 e4b->bd_buddy_page = NULL; in ext4_mb_load_buddy_gfp()
1192 e4b->bd_bitmap_page = NULL; in ext4_mb_load_buddy_gfp()
1215 page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED); in ext4_mb_load_buddy_gfp()
1227 page = find_or_create_page(inode->i_mapping, pnum, gfp); in ext4_mb_load_buddy_gfp()
1229 BUG_ON(page->mapping != inode->i_mapping); in ext4_mb_load_buddy_gfp()
1237 (poff * sb->s_blocksize)); in ext4_mb_load_buddy_gfp()
1243 ret = -ENOMEM; in ext4_mb_load_buddy_gfp()
1247 ret = -EIO; in ext4_mb_load_buddy_gfp()
1252 e4b->bd_bitmap_page = page; in ext4_mb_load_buddy_gfp()
1253 e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); in ext4_mb_load_buddy_gfp()
1259 page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED); in ext4_mb_load_buddy_gfp()
1263 page = find_or_create_page(inode->i_mapping, pnum, gfp); in ext4_mb_load_buddy_gfp()
1265 BUG_ON(page->mapping != inode->i_mapping); in ext4_mb_load_buddy_gfp()
1267 ret = ext4_mb_init_cache(page, e4b->bd_bitmap, in ext4_mb_load_buddy_gfp()
1278 ret = -ENOMEM; in ext4_mb_load_buddy_gfp()
1282 ret = -EIO; in ext4_mb_load_buddy_gfp()
1287 e4b->bd_buddy_page = page; in ext4_mb_load_buddy_gfp()
1288 e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize); in ext4_mb_load_buddy_gfp()
1295 if (e4b->bd_bitmap_page) in ext4_mb_load_buddy_gfp()
1296 put_page(e4b->bd_bitmap_page); in ext4_mb_load_buddy_gfp()
1297 if (e4b->bd_buddy_page) in ext4_mb_load_buddy_gfp()
1298 put_page(e4b->bd_buddy_page); in ext4_mb_load_buddy_gfp()
1299 e4b->bd_buddy = NULL; in ext4_mb_load_buddy_gfp()
1300 e4b->bd_bitmap = NULL; in ext4_mb_load_buddy_gfp()
1312 if (e4b->bd_bitmap_page) in ext4_mb_unload_buddy()
1313 put_page(e4b->bd_bitmap_page); in ext4_mb_unload_buddy()
1314 if (e4b->bd_buddy_page) in ext4_mb_unload_buddy()
1315 put_page(e4b->bd_buddy_page); in ext4_mb_unload_buddy()
1322 int bb_incr = 1 << (e4b->bd_blkbits - 1); in mb_find_order_for_block()
1325 BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); in mb_find_order_for_block()
1326 BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); in mb_find_order_for_block()
1328 bb = e4b->bd_buddy; in mb_find_order_for_block()
1329 while (order <= e4b->bd_blkbits + 1) { in mb_find_order_for_block()
1348 if ((cur & 31) == 0 && (len - cur) >= 32) { in mb_clear_bits()
1361 * will return first found zero bit if any, -1 otherwise
1366 int zero_bit = -1; in mb_test_and_clear_bits()
1370 if ((cur & 31) == 0 && (len - cur) >= 32) { in mb_test_and_clear_bits()
1373 if (*addr != (__u32)(-1) && zero_bit == -1) in mb_test_and_clear_bits()
1379 if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1) in mb_test_and_clear_bits()
1393 if ((cur & 31) == 0 && (len - cur) >= 32) { in ext4_set_bits()
1409 (*bit) -= side; in mb_buddy_adjust_border()
1415 return -1; in mb_buddy_adjust_border()
1438 * --------------------------------- in mb_buddy_mark_free()
1440 * --------------------------------- in mb_buddy_mark_free()
1442 * --------------------------------- in mb_buddy_mark_free()
1458 e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1); in mb_buddy_mark_free()
1460 e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1); in mb_buddy_mark_free()
1466 mb_clear_bits(buddy, first, last - first + 1); in mb_buddy_mark_free()
1467 e4b->bd_info->bb_counters[order - 1] += last - first + 1; in mb_buddy_mark_free()
1482 int last = first + count - 1; in mb_free_blocks()
1483 struct super_block *sb = e4b->bd_sb; in mb_free_blocks()
1487 BUG_ON(last >= (sb->s_blocksize << 3)); in mb_free_blocks()
1488 assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); in mb_free_blocks()
1490 if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) in mb_free_blocks()
1497 e4b->bd_info->bb_free += count; in mb_free_blocks()
1498 if (first < e4b->bd_info->bb_first_free) in mb_free_blocks()
1499 e4b->bd_info->bb_first_free = first; in mb_free_blocks()
1505 left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap); in mb_free_blocks()
1506 block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count); in mb_free_blocks()
1507 if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0]) in mb_free_blocks()
1508 right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap); in mb_free_blocks()
1510 if (unlikely(block != -1)) { in mb_free_blocks()
1514 blocknr = ext4_group_first_block_no(sb, e4b->bd_group); in mb_free_blocks()
1516 if (!(sbi->s_mount_state & EXT4_FC_REPLAY)) { in mb_free_blocks()
1517 ext4_grp_locked_error(sb, e4b->bd_group, in mb_free_blocks()
1518 inode ? inode->i_ino : 0, in mb_free_blocks()
1523 sb, e4b->bd_group, in mb_free_blocks()
1533 e4b->bd_info->bb_fragments--; in mb_free_blocks()
1535 e4b->bd_info->bb_fragments++; in mb_free_blocks()
1545 e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1; in mb_free_blocks()
1548 last -= !right_is_free; in mb_free_blocks()
1549 e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1; in mb_free_blocks()
1556 mb_set_largest_free_order(sb, e4b->bd_info); in mb_free_blocks()
1567 assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); in mb_find_extent()
1574 ex->fe_len = 0; in mb_find_extent()
1575 ex->fe_start = 0; in mb_find_extent()
1576 ex->fe_group = 0; in mb_find_extent()
1584 ex->fe_len = 1 << order; in mb_find_extent()
1585 ex->fe_start = block << order; in mb_find_extent()
1586 ex->fe_group = e4b->bd_group; in mb_find_extent()
1589 next = next - ex->fe_start; in mb_find_extent()
1590 ex->fe_len -= next; in mb_find_extent()
1591 ex->fe_start += next; in mb_find_extent()
1593 while (needed > ex->fe_len && in mb_find_extent()
1600 if (mb_test_bit(next, e4b->bd_bitmap)) in mb_find_extent()
1606 ex->fe_len += 1 << order; in mb_find_extent()
1609 if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) { in mb_find_extent()
1612 ext4_grp_locked_error(e4b->bd_sb, e4b->bd_group, 0, 0, in mb_find_extent()
1615 block, order, needed, ex->fe_group, ex->fe_start, in mb_find_extent()
1616 ex->fe_len, ex->fe_logical); in mb_find_extent()
1617 ex->fe_len = 0; in mb_find_extent()
1618 ex->fe_start = 0; in mb_find_extent()
1619 ex->fe_group = 0; in mb_find_extent()
1621 return ex->fe_len; in mb_find_extent()
1630 int start = ex->fe_start; in mb_mark_used()
1631 int len = ex->fe_len; in mb_mark_used()
1636 BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); in mb_mark_used()
1637 BUG_ON(e4b->bd_group != ex->fe_group); in mb_mark_used()
1638 assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); in mb_mark_used()
1643 e4b->bd_info->bb_free -= len; in mb_mark_used()
1644 if (e4b->bd_info->bb_first_free == start) in mb_mark_used()
1645 e4b->bd_info->bb_first_free += len; in mb_mark_used()
1649 mlen = !mb_test_bit(start - 1, e4b->bd_bitmap); in mb_mark_used()
1650 if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0]) in mb_mark_used()
1651 max = !mb_test_bit(start + len, e4b->bd_bitmap); in mb_mark_used()
1653 e4b->bd_info->bb_fragments++; in mb_mark_used()
1655 e4b->bd_info->bb_fragments--; in mb_mark_used()
1667 e4b->bd_info->bb_counters[ord]--; in mb_mark_used()
1669 len -= mlen; in mb_mark_used()
1682 e4b->bd_info->bb_counters[ord]--; in mb_mark_used()
1684 ord--; in mb_mark_used()
1689 e4b->bd_info->bb_counters[ord]++; in mb_mark_used()
1690 e4b->bd_info->bb_counters[ord]++; in mb_mark_used()
1692 mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info); in mb_mark_used()
1694 ext4_set_bits(e4b->bd_bitmap, ex->fe_start, len0); in mb_mark_used()
1706 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_best_found()
1709 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); in ext4_mb_use_best_found()
1710 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_use_best_found()
1712 ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); in ext4_mb_use_best_found()
1713 ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; in ext4_mb_use_best_found()
1714 ret = mb_mark_used(e4b, &ac->ac_b_ex); in ext4_mb_use_best_found()
1718 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_use_best_found()
1720 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_best_found()
1721 ac->ac_tail = ret & 0xffff; in ext4_mb_use_best_found()
1722 ac->ac_buddy = ret >> 16; in ext4_mb_use_best_found()
1731 ac->ac_bitmap_page = e4b->bd_bitmap_page; in ext4_mb_use_best_found()
1732 get_page(ac->ac_bitmap_page); in ext4_mb_use_best_found()
1733 ac->ac_buddy_page = e4b->bd_buddy_page; in ext4_mb_use_best_found()
1734 get_page(ac->ac_buddy_page); in ext4_mb_use_best_found()
1736 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_use_best_found()
1737 spin_lock(&sbi->s_md_lock); in ext4_mb_use_best_found()
1738 sbi->s_mb_last_group = ac->ac_f_ex.fe_group; in ext4_mb_use_best_found()
1739 sbi->s_mb_last_start = ac->ac_f_ex.fe_start; in ext4_mb_use_best_found()
1740 spin_unlock(&sbi->s_md_lock); in ext4_mb_use_best_found()
1747 if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) in ext4_mb_use_best_found()
1756 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_check_limits()
1757 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_check_limits()
1758 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_check_limits()
1762 if (ac->ac_status == AC_STATUS_FOUND) in ext4_mb_check_limits()
1767 if (ac->ac_found > sbi->s_mb_max_to_scan && in ext4_mb_check_limits()
1768 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_check_limits()
1769 ac->ac_status = AC_STATUS_BREAK; in ext4_mb_check_limits()
1776 if (bex->fe_len < gex->fe_len) in ext4_mb_check_limits()
1779 if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) in ext4_mb_check_limits()
1780 && bex->fe_group == e4b->bd_group) { in ext4_mb_check_limits()
1781 /* recheck chunk's availability - we don't know in ext4_mb_check_limits()
1782 * when it was found (within this lock-unlock in ext4_mb_check_limits()
1784 max = mb_find_extent(e4b, bex->fe_start, gex->fe_len, &ex); in ext4_mb_check_limits()
1785 if (max >= gex->fe_len) { in ext4_mb_check_limits()
1806 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_measure_extent()
1807 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_measure_extent()
1809 BUG_ON(ex->fe_len <= 0); in ext4_mb_measure_extent()
1810 BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
1811 BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
1812 BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); in ext4_mb_measure_extent()
1814 ac->ac_found++; in ext4_mb_measure_extent()
1817 * The special case - take what you catch first in ext4_mb_measure_extent()
1819 if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_measure_extent()
1828 if (ex->fe_len == gex->fe_len) { in ext4_mb_measure_extent()
1837 if (bex->fe_len == 0) { in ext4_mb_measure_extent()
1845 if (bex->fe_len < gex->fe_len) { in ext4_mb_measure_extent()
1848 if (ex->fe_len > bex->fe_len) in ext4_mb_measure_extent()
1850 } else if (ex->fe_len > gex->fe_len) { in ext4_mb_measure_extent()
1854 if (ex->fe_len < bex->fe_len) in ext4_mb_measure_extent()
1865 struct ext4_free_extent ex = ac->ac_b_ex; in ext4_mb_try_best_found()
1871 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_try_best_found()
1875 ext4_lock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
1879 ac->ac_b_ex = ex; in ext4_mb_try_best_found()
1883 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
1893 ext4_group_t group = ac->ac_g_ex.fe_group; in ext4_mb_find_by_goal()
1896 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_find_by_goal()
1897 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_find_by_goal()
1901 return -EFSCORRUPTED; in ext4_mb_find_by_goal()
1902 if (!(ac->ac_flags & (EXT4_MB_HINT_TRY_GOAL | EXT4_MB_HINT_GOAL_ONLY))) in ext4_mb_find_by_goal()
1904 if (grp->bb_free == 0) in ext4_mb_find_by_goal()
1907 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_find_by_goal()
1911 if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) { in ext4_mb_find_by_goal()
1916 ext4_lock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
1917 max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, in ext4_mb_find_by_goal()
1918 ac->ac_g_ex.fe_len, &ex); in ext4_mb_find_by_goal()
1921 if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { in ext4_mb_find_by_goal()
1924 start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) + in ext4_mb_find_by_goal()
1926 /* use do_div to get remainder (would be 64-bit modulo) */ in ext4_mb_find_by_goal()
1927 if (do_div(start, sbi->s_stripe) == 0) { in ext4_mb_find_by_goal()
1928 ac->ac_found++; in ext4_mb_find_by_goal()
1929 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1932 } else if (max >= ac->ac_g_ex.fe_len) { in ext4_mb_find_by_goal()
1934 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
1935 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
1936 ac->ac_found++; in ext4_mb_find_by_goal()
1937 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1939 } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { in ext4_mb_find_by_goal()
1943 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
1944 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
1945 ac->ac_found++; in ext4_mb_find_by_goal()
1946 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1949 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
1963 struct super_block *sb = ac->ac_sb; in ext4_mb_simple_scan_group()
1964 struct ext4_group_info *grp = e4b->bd_info; in ext4_mb_simple_scan_group()
1970 BUG_ON(ac->ac_2order <= 0); in ext4_mb_simple_scan_group()
1971 for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) { in ext4_mb_simple_scan_group()
1972 if (grp->bb_counters[i] == 0) in ext4_mb_simple_scan_group()
1980 ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0, in ext4_mb_simple_scan_group()
1982 grp->bb_counters[i], i); in ext4_mb_simple_scan_group()
1983 ext4_mark_group_bitmap_corrupted(ac->ac_sb, in ext4_mb_simple_scan_group()
1984 e4b->bd_group, in ext4_mb_simple_scan_group()
1988 ac->ac_found++; in ext4_mb_simple_scan_group()
1990 ac->ac_b_ex.fe_len = 1 << i; in ext4_mb_simple_scan_group()
1991 ac->ac_b_ex.fe_start = k << i; in ext4_mb_simple_scan_group()
1992 ac->ac_b_ex.fe_group = e4b->bd_group; in ext4_mb_simple_scan_group()
1996 BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len); in ext4_mb_simple_scan_group()
1998 if (EXT4_SB(sb)->s_mb_stats) in ext4_mb_simple_scan_group()
1999 atomic_inc(&EXT4_SB(sb)->s_bal_2orders); in ext4_mb_simple_scan_group()
2014 struct super_block *sb = ac->ac_sb; in ext4_mb_complex_scan_group()
2015 void *bitmap = e4b->bd_bitmap; in ext4_mb_complex_scan_group()
2020 free = e4b->bd_info->bb_free; in ext4_mb_complex_scan_group()
2024 i = e4b->bd_info->bb_first_free; in ext4_mb_complex_scan_group()
2026 while (free && ac->ac_status == AC_STATUS_CONTINUE) { in ext4_mb_complex_scan_group()
2035 ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, in ext4_mb_complex_scan_group()
2039 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, in ext4_mb_complex_scan_group()
2044 mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); in ext4_mb_complex_scan_group()
2048 ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, in ext4_mb_complex_scan_group()
2052 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, in ext4_mb_complex_scan_group()
2065 free -= ex.fe_len; in ext4_mb_complex_scan_group()
2073 * we try to find stripe-aligned chunks for stripe-size-multiple requests
2079 struct super_block *sb = ac->ac_sb; in ext4_mb_scan_aligned()
2081 void *bitmap = e4b->bd_bitmap; in ext4_mb_scan_aligned()
2088 BUG_ON(sbi->s_stripe == 0); in ext4_mb_scan_aligned()
2090 /* find first stripe-aligned block in group */ in ext4_mb_scan_aligned()
2091 first_group_block = ext4_group_first_block_no(sb, e4b->bd_group); in ext4_mb_scan_aligned()
2093 a = first_group_block + sbi->s_stripe - 1; in ext4_mb_scan_aligned()
2094 do_div(a, sbi->s_stripe); in ext4_mb_scan_aligned()
2095 i = (a * sbi->s_stripe) - first_group_block; in ext4_mb_scan_aligned()
2099 max = mb_find_extent(e4b, i, sbi->s_stripe, &ex); in ext4_mb_scan_aligned()
2100 if (max >= sbi->s_stripe) { in ext4_mb_scan_aligned()
2101 ac->ac_found++; in ext4_mb_scan_aligned()
2103 ac->ac_b_ex = ex; in ext4_mb_scan_aligned()
2108 i += sbi->s_stripe; in ext4_mb_scan_aligned()
2121 int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); in ext4_mb_good_group()
2122 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group()
2129 free = grp->bb_free; in ext4_mb_good_group()
2133 fragments = grp->bb_fragments; in ext4_mb_good_group()
2139 BUG_ON(ac->ac_2order == 0); in ext4_mb_good_group()
2142 if ((ac->ac_flags & EXT4_MB_HINT_DATA) && in ext4_mb_good_group()
2147 if (free < ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2150 if (ac->ac_2order > ac->ac_sb->s_blocksize_bits+1) in ext4_mb_good_group()
2153 if (grp->bb_largest_free_order < ac->ac_2order) in ext4_mb_good_group()
2158 if ((free / fragments) >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2162 if (free >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2182 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group_nolock()
2183 struct super_block *sb = ac->ac_sb; in ext4_mb_good_group_nolock()
2185 bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK; in ext4_mb_good_group_nolock()
2190 return -EFSCORRUPTED; in ext4_mb_good_group_nolock()
2191 if (sbi->s_mb_stats) in ext4_mb_good_group_nolock()
2192 atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]); in ext4_mb_good_group_nolock()
2195 free = grp->bb_free; in ext4_mb_good_group_nolock()
2198 if (cr <= 2 && free < ac->ac_g_ex.fe_len) in ext4_mb_good_group_nolock()
2220 (!sbi->s_log_groups_per_flex || in ext4_mb_good_group_nolock()
2221 ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) && in ext4_mb_good_group_nolock()
2223 (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) in ext4_mb_good_group_nolock()
2251 while (nr-- > 0) { in ext4_mb_prefetch()
2267 (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) { in ext4_mb_prefetch()
2297 while (nr-- > 0) { in ext4_mb_prefetch_fini()
2304 group--; in ext4_mb_prefetch_fini()
2310 (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) { in ext4_mb_prefetch_fini()
2321 int cr = -1; in ext4_mb_regular_allocator()
2329 sb = ac->ac_sb; in ext4_mb_regular_allocator()
2332 /* non-extent files are limited to low blocks/groups */ in ext4_mb_regular_allocator()
2333 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) in ext4_mb_regular_allocator()
2334 ngroups = sbi->s_blockfile_groups; in ext4_mb_regular_allocator()
2336 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_regular_allocator()
2340 if (err || ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
2343 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_regular_allocator()
2347 * ac->ac_2order is set only if the fe_len is a power of 2 in ext4_mb_regular_allocator()
2348 * if ac->ac_2order is set we also set criteria to 0 so that we in ext4_mb_regular_allocator()
2351 i = fls(ac->ac_g_ex.fe_len); in ext4_mb_regular_allocator()
2352 ac->ac_2order = 0; in ext4_mb_regular_allocator()
2357 * We also support searching for power-of-two requests only for in ext4_mb_regular_allocator()
2360 if (i >= sbi->s_mb_order2_reqs && i <= sb->s_blocksize_bits + 2) { in ext4_mb_regular_allocator()
2364 if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) in ext4_mb_regular_allocator()
2365 ac->ac_2order = array_index_nospec(i - 1, in ext4_mb_regular_allocator()
2366 sb->s_blocksize_bits + 2); in ext4_mb_regular_allocator()
2370 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_regular_allocator()
2372 spin_lock(&sbi->s_md_lock); in ext4_mb_regular_allocator()
2373 ac->ac_g_ex.fe_group = sbi->s_mb_last_group; in ext4_mb_regular_allocator()
2374 ac->ac_g_ex.fe_start = sbi->s_mb_last_start; in ext4_mb_regular_allocator()
2375 spin_unlock(&sbi->s_md_lock); in ext4_mb_regular_allocator()
2378 /* Let's just scan groups to find more-less suitable blocks */ in ext4_mb_regular_allocator()
2379 cr = ac->ac_2order ? 0 : 1; in ext4_mb_regular_allocator()
2385 for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { in ext4_mb_regular_allocator()
2386 ac->ac_criteria = cr; in ext4_mb_regular_allocator()
2391 group = ac->ac_g_ex.fe_group; in ext4_mb_regular_allocator()
2398 * Artificially restricted ngroups for non-extent in ext4_mb_regular_allocator()
2412 prefetch_ios < sbi->s_mb_prefetch_limit)) { in ext4_mb_regular_allocator()
2415 nr = sbi->s_mb_prefetch; in ext4_mb_regular_allocator()
2417 nr = 1 << sbi->s_log_groups_per_flex; in ext4_mb_regular_allocator()
2418 nr -= group & (nr - 1); in ext4_mb_regular_allocator()
2419 nr = min(nr, sbi->s_mb_prefetch); in ext4_mb_regular_allocator()
2452 ac->ac_groups_scanned++; in ext4_mb_regular_allocator()
2455 else if (cr == 1 && sbi->s_stripe && in ext4_mb_regular_allocator()
2456 !(ac->ac_g_ex.fe_len % sbi->s_stripe)) in ext4_mb_regular_allocator()
2464 if (ac->ac_status != AC_STATUS_CONTINUE) in ext4_mb_regular_allocator()
2468 if (sbi->s_mb_stats && i == ngroups) in ext4_mb_regular_allocator()
2469 atomic64_inc(&sbi->s_bal_cX_failed[cr]); in ext4_mb_regular_allocator()
2472 if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && in ext4_mb_regular_allocator()
2473 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_regular_allocator()
2479 if (ac->ac_status != AC_STATUS_FOUND) { in ext4_mb_regular_allocator()
2485 lost = atomic_inc_return(&sbi->s_mb_lost_chunks); in ext4_mb_regular_allocator()
2487 ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start, in ext4_mb_regular_allocator()
2488 ac->ac_b_ex.fe_len, lost); in ext4_mb_regular_allocator()
2490 ac->ac_b_ex.fe_group = 0; in ext4_mb_regular_allocator()
2491 ac->ac_b_ex.fe_start = 0; in ext4_mb_regular_allocator()
2492 ac->ac_b_ex.fe_len = 0; in ext4_mb_regular_allocator()
2493 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_regular_allocator()
2494 ac->ac_flags |= EXT4_MB_HINT_FIRST; in ext4_mb_regular_allocator()
2500 if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
2501 atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]); in ext4_mb_regular_allocator()
2503 if (!err && ac->ac_status != AC_STATUS_FOUND && first_err) in ext4_mb_regular_allocator()
2507 ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status, in ext4_mb_regular_allocator()
2508 ac->ac_flags, cr, err); in ext4_mb_regular_allocator()
2518 struct super_block *sb = PDE_DATA(file_inode(seq->file)); in ext4_mb_seq_groups_start()
2529 struct super_block *sb = PDE_DATA(file_inode(seq->file)); in ext4_mb_seq_groups_next()
2541 struct super_block *sb = PDE_DATA(file_inode(seq->file)); in ext4_mb_seq_groups_show()
2548 sb->s_blocksize_bits, in ext4_mb_seq_groups_show()
2555 group--; in ext4_mb_seq_groups_show()
2571 seq_printf(seq, "#%-5u: I/O error\n", group); in ext4_mb_seq_groups_show()
2582 seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free, in ext4_mb_seq_groups_show()
2585 seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ? in ext4_mb_seq_groups_show()
2605 struct super_block *sb = (struct super_block *)seq->private; in ext4_seq_mb_stats_show()
2609 if (!sbi->s_mb_stats) { in ext4_seq_mb_stats_show()
2614 seq_printf(seq, "\treqs: %u\n", atomic_read(&sbi->s_bal_reqs)); in ext4_seq_mb_stats_show()
2615 seq_printf(seq, "\tsuccess: %u\n", atomic_read(&sbi->s_bal_success)); in ext4_seq_mb_stats_show()
2617 seq_printf(seq, "\tgroups_scanned: %u\n", atomic_read(&sbi->s_bal_groups_scanned)); in ext4_seq_mb_stats_show()
2620 seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[0])); in ext4_seq_mb_stats_show()
2622 atomic64_read(&sbi->s_bal_cX_groups_considered[0])); in ext4_seq_mb_stats_show()
2624 atomic64_read(&sbi->s_bal_cX_failed[0])); in ext4_seq_mb_stats_show()
2627 seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[1])); in ext4_seq_mb_stats_show()
2629 atomic64_read(&sbi->s_bal_cX_groups_considered[1])); in ext4_seq_mb_stats_show()
2631 atomic64_read(&sbi->s_bal_cX_failed[1])); in ext4_seq_mb_stats_show()
2634 seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[2])); in ext4_seq_mb_stats_show()
2636 atomic64_read(&sbi->s_bal_cX_groups_considered[2])); in ext4_seq_mb_stats_show()
2638 atomic64_read(&sbi->s_bal_cX_failed[2])); in ext4_seq_mb_stats_show()
2641 seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[3])); in ext4_seq_mb_stats_show()
2643 atomic64_read(&sbi->s_bal_cX_groups_considered[3])); in ext4_seq_mb_stats_show()
2645 atomic64_read(&sbi->s_bal_cX_failed[3])); in ext4_seq_mb_stats_show()
2646 seq_printf(seq, "\textents_scanned: %u\n", atomic_read(&sbi->s_bal_ex_scanned)); in ext4_seq_mb_stats_show()
2647 seq_printf(seq, "\t\tgoal_hits: %u\n", atomic_read(&sbi->s_bal_goals)); in ext4_seq_mb_stats_show()
2648 seq_printf(seq, "\t\t2^n_hits: %u\n", atomic_read(&sbi->s_bal_2orders)); in ext4_seq_mb_stats_show()
2649 seq_printf(seq, "\t\tbreaks: %u\n", atomic_read(&sbi->s_bal_breaks)); in ext4_seq_mb_stats_show()
2650 seq_printf(seq, "\t\tlost: %u\n", atomic_read(&sbi->s_mb_lost_chunks)); in ext4_seq_mb_stats_show()
2653 atomic_read(&sbi->s_mb_buddies_generated), in ext4_seq_mb_stats_show()
2656 atomic64_read(&sbi->s_mb_generation_time)); in ext4_seq_mb_stats_show()
2658 atomic_read(&sbi->s_mb_preallocated)); in ext4_seq_mb_stats_show()
2660 atomic_read(&sbi->s_mb_discarded)); in ext4_seq_mb_stats_show()
2666 int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; in get_groupinfo_cache()
2674 * Allocate the top-level s_group_info array for the specified number
2683 size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> in ext4_mb_alloc_groupinfo()
2685 if (size <= sbi->s_group_info_size) in ext4_mb_alloc_groupinfo()
2688 size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size); in ext4_mb_alloc_groupinfo()
2692 return -ENOMEM; in ext4_mb_alloc_groupinfo()
2695 old_groupinfo = rcu_dereference(sbi->s_group_info); in ext4_mb_alloc_groupinfo()
2698 sbi->s_group_info_size * sizeof(*sbi->s_group_info)); in ext4_mb_alloc_groupinfo()
2700 rcu_assign_pointer(sbi->s_group_info, new_groupinfo); in ext4_mb_alloc_groupinfo()
2701 sbi->s_group_info_size = size / sizeof(*sbi->s_group_info); in ext4_mb_alloc_groupinfo()
2705 sbi->s_group_info_size); in ext4_mb_alloc_groupinfo()
2718 struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); in ext4_mb_add_groupinfo()
2735 rcu_dereference(sbi->s_group_info)[idx] = meta_group_info; in ext4_mb_add_groupinfo()
2740 i = group & (EXT4_DESC_PER_BLOCK(sb) - 1); in ext4_mb_add_groupinfo()
2748 &(meta_group_info[i]->bb_state)); in ext4_mb_add_groupinfo()
2755 (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { in ext4_mb_add_groupinfo()
2756 meta_group_info[i]->bb_free = in ext4_mb_add_groupinfo()
2759 meta_group_info[i]->bb_free = in ext4_mb_add_groupinfo()
2763 INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list); in ext4_mb_add_groupinfo()
2764 init_rwsem(&meta_group_info[i]->alloc_sem); in ext4_mb_add_groupinfo()
2765 meta_group_info[i]->bb_free_root = RB_ROOT; in ext4_mb_add_groupinfo()
2766 meta_group_info[i]->bb_largest_free_order = -1; /* uninit */ in ext4_mb_add_groupinfo()
2777 group_info = rcu_dereference(sbi->s_group_info); in ext4_mb_add_groupinfo()
2783 return -ENOMEM; in ext4_mb_add_groupinfo()
2800 sbi->s_buddy_cache = new_inode(sb); in ext4_mb_init_backend()
2801 if (sbi->s_buddy_cache == NULL) { in ext4_mb_init_backend()
2805 /* To avoid potentially colliding with an valid on-disk inode number, in ext4_mb_init_backend()
2809 sbi->s_buddy_cache->i_ino = EXT4_BAD_INO; in ext4_mb_init_backend()
2810 EXT4_I(sbi->s_buddy_cache)->i_disksize = 0; in ext4_mb_init_backend()
2827 if (sbi->s_es->s_log_groups_per_flex >= 32) { in ext4_mb_init_backend()
2831 sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex, in ext4_mb_init_backend()
2832 BLK_MAX_SEGMENT_SIZE >> (sb->s_blocksize_bits - 9)); in ext4_mb_init_backend()
2833 sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */ in ext4_mb_init_backend()
2835 sbi->s_mb_prefetch = 32; in ext4_mb_init_backend()
2837 if (sbi->s_mb_prefetch > ext4_get_groups_count(sb)) in ext4_mb_init_backend()
2838 sbi->s_mb_prefetch = ext4_get_groups_count(sb); in ext4_mb_init_backend()
2840 * given cr=0 is an CPU-related optimization we shouldn't try to in ext4_mb_init_backend()
2846 sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4; in ext4_mb_init_backend()
2847 if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb)) in ext4_mb_init_backend()
2848 sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb); in ext4_mb_init_backend()
2853 cachep = get_groupinfo_cache(sb->s_blocksize_bits); in ext4_mb_init_backend()
2854 while (i-- > 0) { in ext4_mb_init_backend()
2860 i = sbi->s_group_info_size; in ext4_mb_init_backend()
2862 group_info = rcu_dereference(sbi->s_group_info); in ext4_mb_init_backend()
2863 while (i-- > 0) in ext4_mb_init_backend()
2866 iput(sbi->s_buddy_cache); in ext4_mb_init_backend()
2869 kvfree(rcu_dereference(sbi->s_group_info)); in ext4_mb_init_backend()
2871 return -ENOMEM; in ext4_mb_init_backend()
2889 int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; in ext4_groupinfo_create_slab()
2893 return -EINVAL; in ext4_groupinfo_create_slab()
2916 "EXT4-fs: no memory for groupinfo slab cache\n"); in ext4_groupinfo_create_slab()
2917 return -ENOMEM; in ext4_groupinfo_create_slab()
2931 i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_offsets); in ext4_mb_init()
2933 sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL); in ext4_mb_init()
2934 if (sbi->s_mb_offsets == NULL) { in ext4_mb_init()
2935 ret = -ENOMEM; in ext4_mb_init()
2939 i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_maxs); in ext4_mb_init()
2940 sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL); in ext4_mb_init()
2941 if (sbi->s_mb_maxs == NULL) { in ext4_mb_init()
2942 ret = -ENOMEM; in ext4_mb_init()
2946 ret = ext4_groupinfo_create_slab(sb->s_blocksize); in ext4_mb_init()
2951 sbi->s_mb_maxs[0] = sb->s_blocksize << 3; in ext4_mb_init()
2952 sbi->s_mb_offsets[0] = 0; in ext4_mb_init()
2956 offset_incr = 1 << (sb->s_blocksize_bits - 1); in ext4_mb_init()
2957 max = sb->s_blocksize << 2; in ext4_mb_init()
2959 sbi->s_mb_offsets[i] = offset; in ext4_mb_init()
2960 sbi->s_mb_maxs[i] = max; in ext4_mb_init()
2965 } while (i <= sb->s_blocksize_bits + 1); in ext4_mb_init()
2967 spin_lock_init(&sbi->s_md_lock); in ext4_mb_init()
2968 sbi->s_mb_free_pending = 0; in ext4_mb_init()
2969 INIT_LIST_HEAD(&sbi->s_freed_data_list); in ext4_mb_init()
2971 sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN; in ext4_mb_init()
2972 sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN; in ext4_mb_init()
2973 sbi->s_mb_stats = MB_DEFAULT_STATS; in ext4_mb_init()
2974 sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; in ext4_mb_init()
2975 sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; in ext4_mb_init()
2976 sbi->s_mb_max_inode_prealloc = MB_DEFAULT_MAX_INODE_PREALLOC; in ext4_mb_init()
2989 sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >> in ext4_mb_init()
2990 sbi->s_cluster_bits, 32); in ext4_mb_init()
2999 if (sbi->s_stripe > 1) { in ext4_mb_init()
3000 sbi->s_mb_group_prealloc = roundup( in ext4_mb_init()
3001 sbi->s_mb_group_prealloc, sbi->s_stripe); in ext4_mb_init()
3004 sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group); in ext4_mb_init()
3005 if (sbi->s_locality_groups == NULL) { in ext4_mb_init()
3006 ret = -ENOMEM; in ext4_mb_init()
3011 lg = per_cpu_ptr(sbi->s_locality_groups, i); in ext4_mb_init()
3012 mutex_init(&lg->lg_mutex); in ext4_mb_init()
3014 INIT_LIST_HEAD(&lg->lg_prealloc_list[j]); in ext4_mb_init()
3015 spin_lock_init(&lg->lg_prealloc_lock); in ext4_mb_init()
3026 free_percpu(sbi->s_locality_groups); in ext4_mb_init()
3027 sbi->s_locality_groups = NULL; in ext4_mb_init()
3029 kfree(sbi->s_mb_offsets); in ext4_mb_init()
3030 sbi->s_mb_offsets = NULL; in ext4_mb_init()
3031 kfree(sbi->s_mb_maxs); in ext4_mb_init()
3032 sbi->s_mb_maxs = NULL; in ext4_mb_init()
3043 list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) { in ext4_mb_cleanup_pa()
3045 list_del(&pa->pa_group_list); in ext4_mb_cleanup_pa()
3059 struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); in ext4_mb_release()
3062 if (sbi->s_group_info) { in ext4_mb_release()
3078 EXT4_DESC_PER_BLOCK(sb) - 1) >> in ext4_mb_release()
3081 group_info = rcu_dereference(sbi->s_group_info); in ext4_mb_release()
3087 kfree(sbi->s_mb_offsets); in ext4_mb_release()
3088 kfree(sbi->s_mb_maxs); in ext4_mb_release()
3089 iput(sbi->s_buddy_cache); in ext4_mb_release()
3090 if (sbi->s_mb_stats) { in ext4_mb_release()
3093 atomic_read(&sbi->s_bal_allocated), in ext4_mb_release()
3094 atomic_read(&sbi->s_bal_reqs), in ext4_mb_release()
3095 atomic_read(&sbi->s_bal_success)); in ext4_mb_release()
3099 atomic_read(&sbi->s_bal_ex_scanned), in ext4_mb_release()
3100 atomic_read(&sbi->s_bal_groups_scanned), in ext4_mb_release()
3101 atomic_read(&sbi->s_bal_goals), in ext4_mb_release()
3102 atomic_read(&sbi->s_bal_2orders), in ext4_mb_release()
3103 atomic_read(&sbi->s_bal_breaks), in ext4_mb_release()
3104 atomic_read(&sbi->s_mb_lost_chunks)); in ext4_mb_release()
3107 atomic_read(&sbi->s_mb_buddies_generated), in ext4_mb_release()
3108 atomic64_read(&sbi->s_mb_generation_time)); in ext4_mb_release()
3111 atomic_read(&sbi->s_mb_preallocated), in ext4_mb_release()
3112 atomic_read(&sbi->s_mb_discarded)); in ext4_mb_release()
3115 free_percpu(sbi->s_locality_groups); in ext4_mb_release()
3132 return __blkdev_issue_discard(sb->s_bdev, in ext4_issue_discard()
3133 (sector_t)discard_block << (sb->s_blocksize_bits - 9), in ext4_issue_discard()
3134 (sector_t)count << (sb->s_blocksize_bits - 9), in ext4_issue_discard()
3148 entry->efd_count, entry->efd_group, entry); in ext4_free_data_in_buddy()
3150 err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b); in ext4_free_data_in_buddy()
3154 spin_lock(&EXT4_SB(sb)->s_md_lock); in ext4_free_data_in_buddy()
3155 EXT4_SB(sb)->s_mb_free_pending -= entry->efd_count; in ext4_free_data_in_buddy()
3156 spin_unlock(&EXT4_SB(sb)->s_md_lock); in ext4_free_data_in_buddy()
3160 count += entry->efd_count; in ext4_free_data_in_buddy()
3162 ext4_lock_group(sb, entry->efd_group); in ext4_free_data_in_buddy()
3164 rb_erase(&entry->efd_node, &(db->bb_free_root)); in ext4_free_data_in_buddy()
3165 mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count); in ext4_free_data_in_buddy()
3170 * If the volume is mounted with -o discard, online discard in ext4_free_data_in_buddy()
3176 if (!db->bb_free_root.rb_node) { in ext4_free_data_in_buddy()
3183 ext4_unlock_group(sb, entry->efd_group); in ext4_free_data_in_buddy()
3206 spin_lock(&sbi->s_md_lock); in ext4_process_freed_data()
3207 list_for_each_entry(entry, &sbi->s_freed_data_list, efd_list) { in ext4_process_freed_data()
3208 if (entry->efd_tid != commit_tid) in ext4_process_freed_data()
3210 cut_pos = &entry->efd_list; in ext4_process_freed_data()
3213 list_cut_position(&freed_data_list, &sbi->s_freed_data_list, in ext4_process_freed_data()
3215 spin_unlock(&sbi->s_md_lock); in ext4_process_freed_data()
3219 err = ext4_issue_discard(sb, entry->efd_group, in ext4_process_freed_data()
3220 entry->efd_start_cluster, in ext4_process_freed_data()
3221 entry->efd_count, in ext4_process_freed_data()
3223 if (err && err != -EOPNOTSUPP) { in ext4_process_freed_data()
3226 " with %d", entry->efd_group, in ext4_process_freed_data()
3227 entry->efd_start_cluster, in ext4_process_freed_data()
3228 entry->efd_count, err); in ext4_process_freed_data()
3229 } else if (err == -EOPNOTSUPP) in ext4_process_freed_data()
3267 return -ENOMEM; in ext4_init_mballoc()
3285 * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
3300 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_mark_diskspace_used()
3301 BUG_ON(ac->ac_b_ex.fe_len <= 0); in ext4_mb_mark_diskspace_used()
3303 sb = ac->ac_sb; in ext4_mb_mark_diskspace_used()
3306 bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3318 err = -EIO; in ext4_mb_mark_diskspace_used()
3319 gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); in ext4_mb_mark_diskspace_used()
3323 ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, in ext4_mb_mark_diskspace_used()
3331 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_mark_diskspace_used()
3333 len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3334 if (!ext4_inode_block_valid(ac->ac_inode, block, len)) { in ext4_mb_mark_diskspace_used()
3335 ext4_error(sb, "Allocating blocks %llu-%llu which overlap " in ext4_mb_mark_diskspace_used()
3341 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3342 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
3343 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3344 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3347 err = -EFSCORRUPTED; in ext4_mb_mark_diskspace_used()
3351 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3355 for (i = 0; i < ac->ac_b_ex.fe_len; i++) { in ext4_mb_mark_diskspace_used()
3356 BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, in ext4_mb_mark_diskspace_used()
3357 bitmap_bh->b_data)); in ext4_mb_mark_diskspace_used()
3361 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
3362 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3364 (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { in ext4_mb_mark_diskspace_used()
3365 gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); in ext4_mb_mark_diskspace_used()
3368 ac->ac_b_ex.fe_group, gdp)); in ext4_mb_mark_diskspace_used()
3370 len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; in ext4_mb_mark_diskspace_used()
3372 ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh); in ext4_mb_mark_diskspace_used()
3373 ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp); in ext4_mb_mark_diskspace_used()
3375 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3376 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3380 if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) in ext4_mb_mark_diskspace_used()
3382 percpu_counter_sub(&sbi->s_dirtyclusters_counter, in ext4_mb_mark_diskspace_used()
3385 if (sbi->s_log_groups_per_flex) { in ext4_mb_mark_diskspace_used()
3387 ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3388 atomic64_sub(ac->ac_b_ex.fe_len, in ext4_mb_mark_diskspace_used()
3390 flex_group)->free_clusters); in ext4_mb_mark_diskspace_used()
3432 EXT4_BLOCKS_PER_GROUP(sb) - EXT4_C2B(sbi, blkoff)); in ext4_mb_mark_bb()
3442 err = -EIO; in ext4_mb_mark_bb()
3450 if (!mb_test_bit(blkoff + i, bitmap_bh->b_data) == in ext4_mb_mark_bb()
3454 clen_changed = clen - already; in ext4_mb_mark_bb()
3456 ext4_set_bits(bitmap_bh->b_data, blkoff, clen); in ext4_mb_mark_bb()
3458 mb_test_and_clear_bits(bitmap_bh->b_data, blkoff, clen); in ext4_mb_mark_bb()
3460 (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { in ext4_mb_mark_bb()
3461 gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); in ext4_mb_mark_bb()
3466 clen = ext4_free_group_clusters(sb, gdp) - clen_changed; in ext4_mb_mark_bb()
3476 if (sbi->s_log_groups_per_flex) { in ext4_mb_mark_bb()
3482 atomic64_sub(clen_changed, &fg->free_clusters); in ext4_mb_mark_bb()
3484 atomic64_add(clen_changed, &fg->free_clusters); in ext4_mb_mark_bb()
3498 len -= thisgrp_len; in ext4_mb_mark_bb()
3508 * here we normalize request for locality group
3518 struct super_block *sb = ac->ac_sb; in ext4_mb_normalize_group_request()
3519 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_normalize_group_request()
3522 ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; in ext4_mb_normalize_group_request()
3523 mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len); in ext4_mb_normalize_group_request()
3534 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_normalize_request()
3535 struct ext4_super_block *es = sbi->s_es; in ext4_mb_normalize_request()
3541 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_normalize_request()
3546 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_normalize_request()
3550 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_normalize_request()
3555 if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) in ext4_mb_normalize_request()
3558 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { in ext4_mb_normalize_request()
3563 bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_normalize_request()
3567 size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); in ext4_mb_normalize_request()
3569 if (size < i_size_read(ac->ac_inode)) in ext4_mb_normalize_request()
3570 size = i_size_read(ac->ac_inode); in ext4_mb_normalize_request()
3597 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
3598 (21 - bsbits)) << 21; in ext4_mb_normalize_request()
3601 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
3602 (22 - bsbits)) << 22; in ext4_mb_normalize_request()
3604 } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, in ext4_mb_normalize_request()
3606 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
3607 (23 - bsbits)) << 23; in ext4_mb_normalize_request()
3610 start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; in ext4_mb_normalize_request()
3611 size = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
3612 ac->ac_o_ex.fe_len) << bsbits; in ext4_mb_normalize_request()
3623 start = max(start, rounddown(ac->ac_o_ex.fe_logical, in ext4_mb_normalize_request()
3624 (ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb))); in ext4_mb_normalize_request()
3627 if (ar->pleft && start <= ar->lleft) { in ext4_mb_normalize_request()
3628 size -= ar->lleft + 1 - start; in ext4_mb_normalize_request()
3629 start = ar->lleft + 1; in ext4_mb_normalize_request()
3631 if (ar->pright && start + size - 1 >= ar->lright) in ext4_mb_normalize_request()
3632 size -= start + size - ar->lright; in ext4_mb_normalize_request()
3638 if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) in ext4_mb_normalize_request()
3639 size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb); in ext4_mb_normalize_request()
3645 list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { in ext4_mb_normalize_request()
3648 if (pa->pa_deleted) in ext4_mb_normalize_request()
3650 spin_lock(&pa->pa_lock); in ext4_mb_normalize_request()
3651 if (pa->pa_deleted) { in ext4_mb_normalize_request()
3652 spin_unlock(&pa->pa_lock); in ext4_mb_normalize_request()
3656 pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
3657 pa->pa_len); in ext4_mb_normalize_request()
3660 BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || in ext4_mb_normalize_request()
3661 ac->ac_o_ex.fe_logical < pa->pa_lstart)); in ext4_mb_normalize_request()
3664 if (pa->pa_lstart >= end || pa_end <= start) { in ext4_mb_normalize_request()
3665 spin_unlock(&pa->pa_lock); in ext4_mb_normalize_request()
3668 BUG_ON(pa->pa_lstart <= start && pa_end >= end); in ext4_mb_normalize_request()
3671 if (pa_end <= ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3674 } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3675 BUG_ON(pa->pa_lstart > end); in ext4_mb_normalize_request()
3676 end = pa->pa_lstart; in ext4_mb_normalize_request()
3678 spin_unlock(&pa->pa_lock); in ext4_mb_normalize_request()
3681 size = end - start; in ext4_mb_normalize_request()
3685 list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { in ext4_mb_normalize_request()
3688 spin_lock(&pa->pa_lock); in ext4_mb_normalize_request()
3689 if (pa->pa_deleted == 0) { in ext4_mb_normalize_request()
3690 pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
3691 pa->pa_len); in ext4_mb_normalize_request()
3692 BUG_ON(!(start >= pa_end || end <= pa->pa_lstart)); in ext4_mb_normalize_request()
3694 spin_unlock(&pa->pa_lock); in ext4_mb_normalize_request()
3698 if (start + size <= ac->ac_o_ex.fe_logical && in ext4_mb_normalize_request()
3699 start > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3700 ext4_msg(ac->ac_sb, KERN_ERR, in ext4_mb_normalize_request()
3703 (unsigned long) ac->ac_o_ex.fe_logical); in ext4_mb_normalize_request()
3706 BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); in ext4_mb_normalize_request()
3712 ac->ac_g_ex.fe_logical = start; in ext4_mb_normalize_request()
3713 ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); in ext4_mb_normalize_request()
3716 if (ar->pright && (ar->lright == (start + size)) && in ext4_mb_normalize_request()
3717 ar->pright >= size && in ext4_mb_normalize_request()
3718 ar->pright - size >= le32_to_cpu(es->s_first_data_block)) { in ext4_mb_normalize_request()
3720 ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, in ext4_mb_normalize_request()
3721 &ac->ac_g_ex.fe_group, in ext4_mb_normalize_request()
3722 &ac->ac_g_ex.fe_start); in ext4_mb_normalize_request()
3723 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
3725 if (ar->pleft && (ar->lleft + 1 == start) && in ext4_mb_normalize_request()
3726 ar->pleft + 1 < ext4_blocks_count(es)) { in ext4_mb_normalize_request()
3728 ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, in ext4_mb_normalize_request()
3729 &ac->ac_g_ex.fe_group, in ext4_mb_normalize_request()
3730 &ac->ac_g_ex.fe_start); in ext4_mb_normalize_request()
3731 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
3734 mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size, in ext4_mb_normalize_request()
3740 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_collect_stats()
3742 if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) { in ext4_mb_collect_stats()
3743 atomic_inc(&sbi->s_bal_reqs); in ext4_mb_collect_stats()
3744 atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); in ext4_mb_collect_stats()
3745 if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) in ext4_mb_collect_stats()
3746 atomic_inc(&sbi->s_bal_success); in ext4_mb_collect_stats()
3747 atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); in ext4_mb_collect_stats()
3748 atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned); in ext4_mb_collect_stats()
3749 if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && in ext4_mb_collect_stats()
3750 ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) in ext4_mb_collect_stats()
3751 atomic_inc(&sbi->s_bal_goals); in ext4_mb_collect_stats()
3752 if (ac->ac_found > sbi->s_mb_max_to_scan) in ext4_mb_collect_stats()
3753 atomic_inc(&sbi->s_bal_breaks); in ext4_mb_collect_stats()
3756 if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) in ext4_mb_collect_stats()
3766 * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
3770 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_discard_allocated_blocks()
3775 if (ac->ac_f_ex.fe_len == 0) in ext4_discard_allocated_blocks()
3777 err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b); in ext4_discard_allocated_blocks()
3787 ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
3788 mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start, in ext4_discard_allocated_blocks()
3789 ac->ac_f_ex.fe_len); in ext4_discard_allocated_blocks()
3790 ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
3794 if (pa->pa_type == MB_INODE_PA) in ext4_discard_allocated_blocks()
3795 pa->pa_free += ac->ac_b_ex.fe_len; in ext4_discard_allocated_blocks()
3804 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_inode_pa()
3810 start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); in ext4_mb_use_inode_pa()
3811 end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len), in ext4_mb_use_inode_pa()
3812 start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); in ext4_mb_use_inode_pa()
3813 len = EXT4_NUM_B2C(sbi, end - start); in ext4_mb_use_inode_pa()
3814 ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, in ext4_mb_use_inode_pa()
3815 &ac->ac_b_ex.fe_start); in ext4_mb_use_inode_pa()
3816 ac->ac_b_ex.fe_len = len; in ext4_mb_use_inode_pa()
3817 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_inode_pa()
3818 ac->ac_pa = pa; in ext4_mb_use_inode_pa()
3820 BUG_ON(start < pa->pa_pstart); in ext4_mb_use_inode_pa()
3821 BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len)); in ext4_mb_use_inode_pa()
3822 BUG_ON(pa->pa_free < len); in ext4_mb_use_inode_pa()
3823 BUG_ON(ac->ac_b_ex.fe_len <= 0); in ext4_mb_use_inode_pa()
3824 pa->pa_free -= len; in ext4_mb_use_inode_pa()
3826 mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa); in ext4_mb_use_inode_pa()
3830 * use blocks preallocated to locality group
3835 unsigned int len = ac->ac_o_ex.fe_len; in ext4_mb_use_group_pa()
3837 ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, in ext4_mb_use_group_pa()
3838 &ac->ac_b_ex.fe_group, in ext4_mb_use_group_pa()
3839 &ac->ac_b_ex.fe_start); in ext4_mb_use_group_pa()
3840 ac->ac_b_ex.fe_len = len; in ext4_mb_use_group_pa()
3841 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_group_pa()
3842 ac->ac_pa = pa; in ext4_mb_use_group_pa()
3847 * in on-disk bitmap -- see ext4_mb_release_context() in ext4_mb_use_group_pa()
3850 mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n", in ext4_mb_use_group_pa()
3851 pa->pa_lstart-len, len, pa); in ext4_mb_use_group_pa()
3868 atomic_inc(&pa->pa_count); in ext4_mb_check_group_pa()
3871 cur_distance = abs(goal_block - cpa->pa_pstart); in ext4_mb_check_group_pa()
3872 new_distance = abs(goal_block - pa->pa_pstart); in ext4_mb_check_group_pa()
3878 atomic_dec(&cpa->pa_count); in ext4_mb_check_group_pa()
3879 atomic_inc(&pa->pa_count); in ext4_mb_check_group_pa()
3889 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_preallocated()
3891 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_use_preallocated()
3897 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_use_preallocated()
3900 /* first, try per-file preallocation */ in ext4_mb_use_preallocated()
3902 list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { in ext4_mb_use_preallocated()
3906 if (ac->ac_o_ex.fe_logical < pa->pa_lstart || in ext4_mb_use_preallocated()
3907 ac->ac_o_ex.fe_logical >= (pa->pa_lstart + in ext4_mb_use_preallocated()
3908 EXT4_C2B(sbi, pa->pa_len))) in ext4_mb_use_preallocated()
3911 /* non-extent files can't have physical blocks past 2^32 */ in ext4_mb_use_preallocated()
3912 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && in ext4_mb_use_preallocated()
3913 (pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len) > in ext4_mb_use_preallocated()
3918 spin_lock(&pa->pa_lock); in ext4_mb_use_preallocated()
3919 if (pa->pa_deleted == 0 && pa->pa_free) { in ext4_mb_use_preallocated()
3920 atomic_inc(&pa->pa_count); in ext4_mb_use_preallocated()
3922 spin_unlock(&pa->pa_lock); in ext4_mb_use_preallocated()
3923 ac->ac_criteria = 10; in ext4_mb_use_preallocated()
3927 spin_unlock(&pa->pa_lock); in ext4_mb_use_preallocated()
3932 if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) in ext4_mb_use_preallocated()
3935 /* inode may have no locality group for some reason */ in ext4_mb_use_preallocated()
3936 lg = ac->ac_lg; in ext4_mb_use_preallocated()
3939 order = fls(ac->ac_o_ex.fe_len) - 1; in ext4_mb_use_preallocated()
3940 if (order > PREALLOC_TB_SIZE - 1) in ext4_mb_use_preallocated()
3942 order = PREALLOC_TB_SIZE - 1; in ext4_mb_use_preallocated()
3944 goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); in ext4_mb_use_preallocated()
3951 list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[i], in ext4_mb_use_preallocated()
3953 spin_lock(&pa->pa_lock); in ext4_mb_use_preallocated()
3954 if (pa->pa_deleted == 0 && in ext4_mb_use_preallocated()
3955 pa->pa_free >= ac->ac_o_ex.fe_len) { in ext4_mb_use_preallocated()
3960 spin_unlock(&pa->pa_lock); in ext4_mb_use_preallocated()
3966 ac->ac_criteria = 20; in ext4_mb_use_preallocated()
3974 * but not yet committed and marks them used in in-core bitmap.
3988 n = rb_first(&(grp->bb_free_root)); in ext4_mb_generate_from_freelist()
3992 ext4_set_bits(bitmap, entry->efd_start_cluster, entry->efd_count); in ext4_mb_generate_from_freelist()
4000 * used in in-core bitmap. buddy must be generated from this bitmap
4026 list_for_each(cur, &grp->bb_prealloc_list) { in ext4_mb_generate_from_pa()
4028 spin_lock(&pa->pa_lock); in ext4_mb_generate_from_pa()
4029 ext4_get_group_no_and_offset(sb, pa->pa_pstart, in ext4_mb_generate_from_pa()
4031 len = pa->pa_len; in ext4_mb_generate_from_pa()
4032 spin_unlock(&pa->pa_lock); in ext4_mb_generate_from_pa()
4047 if (pa->pa_deleted) { in ext4_mb_mark_pa_deleted()
4049 pa->pa_type, pa->pa_pstart, pa->pa_lstart, in ext4_mb_mark_pa_deleted()
4050 pa->pa_len); in ext4_mb_mark_pa_deleted()
4054 pa->pa_deleted = 1; in ext4_mb_mark_pa_deleted()
4056 if (pa->pa_type == MB_INODE_PA) { in ext4_mb_mark_pa_deleted()
4057 ei = EXT4_I(pa->pa_inode); in ext4_mb_mark_pa_deleted()
4058 atomic_dec(&ei->i_prealloc_active); in ext4_mb_mark_pa_deleted()
4067 BUG_ON(atomic_read(&pa->pa_count)); in ext4_mb_pa_callback()
4068 BUG_ON(pa->pa_deleted == 0); in ext4_mb_pa_callback()
4083 spin_lock(&pa->pa_lock); in ext4_mb_put_pa()
4084 if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) { in ext4_mb_put_pa()
4085 spin_unlock(&pa->pa_lock); in ext4_mb_put_pa()
4089 if (pa->pa_deleted == 1) { in ext4_mb_put_pa()
4090 spin_unlock(&pa->pa_lock); in ext4_mb_put_pa()
4095 spin_unlock(&pa->pa_lock); in ext4_mb_put_pa()
4097 grp_blk = pa->pa_pstart; in ext4_mb_put_pa()
4099 * If doing group-based preallocation, pa_pstart may be in the in ext4_mb_put_pa()
4102 if (pa->pa_type == MB_GROUP_PA) in ext4_mb_put_pa()
4103 grp_blk--; in ext4_mb_put_pa()
4112 * copy on-disk bitmap to buddy in ext4_mb_put_pa()
4113 * mark B in on-disk bitmap in ext4_mb_put_pa()
4122 list_del(&pa->pa_group_list); in ext4_mb_put_pa()
4125 spin_lock(pa->pa_obj_lock); in ext4_mb_put_pa()
4126 list_del_rcu(&pa->pa_inode_list); in ext4_mb_put_pa()
4127 spin_unlock(pa->pa_obj_lock); in ext4_mb_put_pa()
4129 call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); in ext4_mb_put_pa()
4138 struct super_block *sb = ac->ac_sb; in ext4_mb_new_inode_pa()
4145 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4146 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_inode_pa()
4147 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_inode_pa()
4148 BUG_ON(ac->ac_pa == NULL); in ext4_mb_new_inode_pa()
4150 pa = ac->ac_pa; in ext4_mb_new_inode_pa()
4152 if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { in ext4_mb_new_inode_pa()
4159 BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); in ext4_mb_new_inode_pa()
4160 BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); in ext4_mb_new_inode_pa()
4173 new_bex_end = ac->ac_g_ex.fe_logical + in ext4_mb_new_inode_pa()
4174 EXT4_C2B(sbi, ac->ac_g_ex.fe_len); in ext4_mb_new_inode_pa()
4175 new_bex_start = new_bex_end - EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4176 if (ac->ac_o_ex.fe_logical >= new_bex_start) in ext4_mb_new_inode_pa()
4179 new_bex_start = ac->ac_g_ex.fe_logical; in ext4_mb_new_inode_pa()
4181 new_bex_start + EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4182 if (ac->ac_o_ex.fe_logical < new_bex_end) in ext4_mb_new_inode_pa()
4185 new_bex_start = ac->ac_o_ex.fe_logical; in ext4_mb_new_inode_pa()
4187 new_bex_start + EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4190 ac->ac_b_ex.fe_logical = new_bex_start; in ext4_mb_new_inode_pa()
4192 BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); in ext4_mb_new_inode_pa()
4193 BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4194 BUG_ON(new_bex_end > (ac->ac_g_ex.fe_logical + in ext4_mb_new_inode_pa()
4195 EXT4_C2B(sbi, ac->ac_g_ex.fe_len))); in ext4_mb_new_inode_pa()
4200 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_new_inode_pa()
4202 pa->pa_lstart = ac->ac_b_ex.fe_logical; in ext4_mb_new_inode_pa()
4203 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_inode_pa()
4204 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_inode_pa()
4205 pa->pa_free = pa->pa_len; in ext4_mb_new_inode_pa()
4206 spin_lock_init(&pa->pa_lock); in ext4_mb_new_inode_pa()
4207 INIT_LIST_HEAD(&pa->pa_inode_list); in ext4_mb_new_inode_pa()
4208 INIT_LIST_HEAD(&pa->pa_group_list); in ext4_mb_new_inode_pa()
4209 pa->pa_deleted = 0; in ext4_mb_new_inode_pa()
4210 pa->pa_type = MB_INODE_PA; in ext4_mb_new_inode_pa()
4212 mb_debug(sb, "new inode pa %p: %llu/%d for %u\n", pa, pa->pa_pstart, in ext4_mb_new_inode_pa()
4213 pa->pa_len, pa->pa_lstart); in ext4_mb_new_inode_pa()
4217 atomic_add(pa->pa_free, &sbi->s_mb_preallocated); in ext4_mb_new_inode_pa()
4219 ei = EXT4_I(ac->ac_inode); in ext4_mb_new_inode_pa()
4220 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_inode_pa()
4224 pa->pa_obj_lock = &ei->i_prealloc_lock; in ext4_mb_new_inode_pa()
4225 pa->pa_inode = ac->ac_inode; in ext4_mb_new_inode_pa()
4227 list_add(&pa->pa_group_list, &grp->bb_prealloc_list); in ext4_mb_new_inode_pa()
4229 spin_lock(pa->pa_obj_lock); in ext4_mb_new_inode_pa()
4230 list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list); in ext4_mb_new_inode_pa()
4231 spin_unlock(pa->pa_obj_lock); in ext4_mb_new_inode_pa()
4232 atomic_inc(&ei->i_prealloc_active); in ext4_mb_new_inode_pa()
4236 * creates new preallocated space for locality group inodes belongs to
4241 struct super_block *sb = ac->ac_sb; in ext4_mb_new_group_pa()
4247 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_group_pa()
4248 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_group_pa()
4249 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_group_pa()
4250 BUG_ON(ac->ac_pa == NULL); in ext4_mb_new_group_pa()
4252 pa = ac->ac_pa; in ext4_mb_new_group_pa()
4256 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_new_group_pa()
4258 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_group_pa()
4259 pa->pa_lstart = pa->pa_pstart; in ext4_mb_new_group_pa()
4260 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_group_pa()
4261 pa->pa_free = pa->pa_len; in ext4_mb_new_group_pa()
4262 spin_lock_init(&pa->pa_lock); in ext4_mb_new_group_pa()
4263 INIT_LIST_HEAD(&pa->pa_inode_list); in ext4_mb_new_group_pa()
4264 INIT_LIST_HEAD(&pa->pa_group_list); in ext4_mb_new_group_pa()
4265 pa->pa_deleted = 0; in ext4_mb_new_group_pa()
4266 pa->pa_type = MB_GROUP_PA; in ext4_mb_new_group_pa()
4268 mb_debug(sb, "new group pa %p: %llu/%d for %u\n", pa, pa->pa_pstart, in ext4_mb_new_group_pa()
4269 pa->pa_len, pa->pa_lstart); in ext4_mb_new_group_pa()
4273 atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated); in ext4_mb_new_group_pa()
4275 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_group_pa()
4278 lg = ac->ac_lg; in ext4_mb_new_group_pa()
4281 pa->pa_obj_lock = &lg->lg_prealloc_lock; in ext4_mb_new_group_pa()
4282 pa->pa_inode = NULL; in ext4_mb_new_group_pa()
4284 list_add(&pa->pa_group_list, &grp->bb_prealloc_list); in ext4_mb_new_group_pa()
4294 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_new_preallocation()
4301 * finds all unused blocks in on-disk bitmap, frees them in
4302 * in-core bitmap and buddy.
4306 * TODO: optimize the case when there are no in-core structures yet
4312 struct super_block *sb = e4b->bd_sb; in ext4_mb_release_inode_pa()
4321 BUG_ON(pa->pa_deleted == 0); in ext4_mb_release_inode_pa()
4322 ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); in ext4_mb_release_inode_pa()
4323 grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit); in ext4_mb_release_inode_pa()
4324 BUG_ON(group != e4b->bd_group && pa->pa_len != 0); in ext4_mb_release_inode_pa()
4325 end = bit + pa->pa_len; in ext4_mb_release_inode_pa()
4328 bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit); in ext4_mb_release_inode_pa()
4331 next = mb_find_next_bit(bitmap_bh->b_data, end, bit); in ext4_mb_release_inode_pa()
4334 (unsigned) next - bit, (unsigned) group); in ext4_mb_release_inode_pa()
4335 free += next - bit; in ext4_mb_release_inode_pa()
4337 trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit); in ext4_mb_release_inode_pa()
4340 next - bit); in ext4_mb_release_inode_pa()
4341 mb_free_blocks(pa->pa_inode, e4b, bit, next - bit); in ext4_mb_release_inode_pa()
4344 if (free != pa->pa_free) { in ext4_mb_release_inode_pa()
4345 ext4_msg(e4b->bd_sb, KERN_CRIT, in ext4_mb_release_inode_pa()
4347 pa, (unsigned long) pa->pa_lstart, in ext4_mb_release_inode_pa()
4348 (unsigned long) pa->pa_pstart, in ext4_mb_release_inode_pa()
4349 pa->pa_len); in ext4_mb_release_inode_pa()
4351 free, pa->pa_free); in ext4_mb_release_inode_pa()
4357 atomic_add(free, &sbi->s_mb_discarded); in ext4_mb_release_inode_pa()
4366 struct super_block *sb = e4b->bd_sb; in ext4_mb_release_group_pa()
4371 BUG_ON(pa->pa_deleted == 0); in ext4_mb_release_group_pa()
4372 ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); in ext4_mb_release_group_pa()
4373 if (unlikely(group != e4b->bd_group && pa->pa_len != 0)) { in ext4_mb_release_group_pa()
4375 e4b->bd_group, group, pa->pa_pstart); in ext4_mb_release_group_pa()
4378 mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len); in ext4_mb_release_group_pa()
4379 atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded); in ext4_mb_release_group_pa()
4380 trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len); in ext4_mb_release_group_pa()
4389 * - when do we discard
4391 * - how many do we discard
4409 if (list_empty(&grp->bb_prealloc_list)) in ext4_mb_discard_group_preallocations()
4415 ext4_error_err(sb, -err, in ext4_mb_discard_group_preallocations()
4432 &grp->bb_prealloc_list, pa_group_list) { in ext4_mb_discard_group_preallocations()
4433 spin_lock(&pa->pa_lock); in ext4_mb_discard_group_preallocations()
4434 if (atomic_read(&pa->pa_count)) { in ext4_mb_discard_group_preallocations()
4435 spin_unlock(&pa->pa_lock); in ext4_mb_discard_group_preallocations()
4439 if (pa->pa_deleted) { in ext4_mb_discard_group_preallocations()
4440 spin_unlock(&pa->pa_lock); in ext4_mb_discard_group_preallocations()
4451 free += pa->pa_free; in ext4_mb_discard_group_preallocations()
4453 spin_unlock(&pa->pa_lock); in ext4_mb_discard_group_preallocations()
4455 list_del(&pa->pa_group_list); in ext4_mb_discard_group_preallocations()
4456 list_add(&pa->u.pa_tmp_list, &list); in ext4_mb_discard_group_preallocations()
4462 /* remove from object (inode or locality group) */ in ext4_mb_discard_group_preallocations()
4463 spin_lock(pa->pa_obj_lock); in ext4_mb_discard_group_preallocations()
4464 list_del_rcu(&pa->pa_inode_list); in ext4_mb_discard_group_preallocations()
4465 spin_unlock(pa->pa_obj_lock); in ext4_mb_discard_group_preallocations()
4467 if (pa->pa_type == MB_GROUP_PA) in ext4_mb_discard_group_preallocations()
4472 list_del(&pa->u.pa_tmp_list); in ext4_mb_discard_group_preallocations()
4473 call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); in ext4_mb_discard_group_preallocations()
4481 free, group, grp->bb_free); in ext4_mb_discard_group_preallocations()
4486 * releases all non-used preallocated blocks for given inode
4497 struct super_block *sb = inode->i_sb; in ext4_discard_preallocations()
4505 if (!S_ISREG(inode->i_mode)) { in ext4_discard_preallocations()
4506 /*BUG_ON(!list_empty(&ei->i_prealloc_list));*/ in ext4_discard_preallocations()
4510 if (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) in ext4_discard_preallocations()
4514 inode->i_ino); in ext4_discard_preallocations()
4516 atomic_read(&ei->i_prealloc_active), needed); in ext4_discard_preallocations()
4525 spin_lock(&ei->i_prealloc_lock); in ext4_discard_preallocations()
4526 while (!list_empty(&ei->i_prealloc_list) && needed) { in ext4_discard_preallocations()
4527 pa = list_entry(ei->i_prealloc_list.prev, in ext4_discard_preallocations()
4529 BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock); in ext4_discard_preallocations()
4530 spin_lock(&pa->pa_lock); in ext4_discard_preallocations()
4531 if (atomic_read(&pa->pa_count)) { in ext4_discard_preallocations()
4532 /* this shouldn't happen often - nobody should in ext4_discard_preallocations()
4534 spin_unlock(&pa->pa_lock); in ext4_discard_preallocations()
4535 spin_unlock(&ei->i_prealloc_lock); in ext4_discard_preallocations()
4537 "uh-oh! used pa while discarding"); in ext4_discard_preallocations()
4543 if (pa->pa_deleted == 0) { in ext4_discard_preallocations()
4545 spin_unlock(&pa->pa_lock); in ext4_discard_preallocations()
4546 list_del_rcu(&pa->pa_inode_list); in ext4_discard_preallocations()
4547 list_add(&pa->u.pa_tmp_list, &list); in ext4_discard_preallocations()
4548 needed--; in ext4_discard_preallocations()
4553 spin_unlock(&pa->pa_lock); in ext4_discard_preallocations()
4554 spin_unlock(&ei->i_prealloc_lock); in ext4_discard_preallocations()
4559 * ->clear_inode() the inode will get freed in ext4_discard_preallocations()
4562 * freed memory, bad-bad-bad */ in ext4_discard_preallocations()
4566 * of ->clear_inode(), but not in case of in ext4_discard_preallocations()
4571 spin_unlock(&ei->i_prealloc_lock); in ext4_discard_preallocations()
4574 BUG_ON(pa->pa_type != MB_INODE_PA); in ext4_discard_preallocations()
4575 group = ext4_get_group_number(sb, pa->pa_pstart); in ext4_discard_preallocations()
4580 ext4_error_err(sb, -err, "Error %d loading buddy information for %u", in ext4_discard_preallocations()
4588 ext4_error_err(sb, -err, "Error %d reading block bitmap for %u", in ext4_discard_preallocations()
4595 list_del(&pa->pa_group_list); in ext4_discard_preallocations()
4602 list_del(&pa->u.pa_tmp_list); in ext4_discard_preallocations()
4603 call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); in ext4_discard_preallocations()
4614 return -ENOMEM; in ext4_mb_pa_alloc()
4615 atomic_set(&pa->pa_count, 1); in ext4_mb_pa_alloc()
4616 ac->ac_pa = pa; in ext4_mb_pa_alloc()
4622 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_pa_free()
4625 ac->ac_pa = NULL; in ext4_mb_pa_free()
4626 WARN_ON(!atomic_dec_and_test(&pa->pa_count)); in ext4_mb_pa_free()
4649 list_for_each(cur, &grp->bb_prealloc_list) { in ext4_mb_show_pa()
4652 spin_lock(&pa->pa_lock); in ext4_mb_show_pa()
4653 ext4_get_group_no_and_offset(sb, pa->pa_pstart, in ext4_mb_show_pa()
4655 spin_unlock(&pa->pa_lock); in ext4_mb_show_pa()
4657 pa->pa_len); in ext4_mb_show_pa()
4660 mb_debug(sb, "%u: %d/%d\n", i, grp->bb_free, in ext4_mb_show_pa()
4661 grp->bb_fragments); in ext4_mb_show_pa()
4667 struct super_block *sb = ac->ac_sb; in ext4_mb_show_ac()
4675 ac->ac_status, ac->ac_flags); in ext4_mb_show_ac()
4679 (unsigned long)ac->ac_o_ex.fe_group, in ext4_mb_show_ac()
4680 (unsigned long)ac->ac_o_ex.fe_start, in ext4_mb_show_ac()
4681 (unsigned long)ac->ac_o_ex.fe_len, in ext4_mb_show_ac()
4682 (unsigned long)ac->ac_o_ex.fe_logical, in ext4_mb_show_ac()
4683 (unsigned long)ac->ac_g_ex.fe_group, in ext4_mb_show_ac()
4684 (unsigned long)ac->ac_g_ex.fe_start, in ext4_mb_show_ac()
4685 (unsigned long)ac->ac_g_ex.fe_len, in ext4_mb_show_ac()
4686 (unsigned long)ac->ac_g_ex.fe_logical, in ext4_mb_show_ac()
4687 (unsigned long)ac->ac_b_ex.fe_group, in ext4_mb_show_ac()
4688 (unsigned long)ac->ac_b_ex.fe_start, in ext4_mb_show_ac()
4689 (unsigned long)ac->ac_b_ex.fe_len, in ext4_mb_show_ac()
4690 (unsigned long)ac->ac_b_ex.fe_logical, in ext4_mb_show_ac()
4691 (int)ac->ac_criteria); in ext4_mb_show_ac()
4692 mb_debug(sb, "%u found", ac->ac_found); in ext4_mb_show_ac()
4702 ext4_mb_show_pa(ac->ac_sb); in ext4_mb_show_ac()
4708 * We use locality group preallocation for small size file. The size of the
4716 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_group_or_file()
4717 int bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_group_or_file()
4720 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_group_or_file()
4723 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_group_or_file()
4726 size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); in ext4_mb_group_or_file()
4727 isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) in ext4_mb_group_or_file()
4731 !inode_is_open_for_write(ac->ac_inode)) { in ext4_mb_group_or_file()
4732 ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; in ext4_mb_group_or_file()
4736 if (sbi->s_mb_group_prealloc <= 0) { in ext4_mb_group_or_file()
4737 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
4743 if (size > sbi->s_mb_stream_request) { in ext4_mb_group_or_file()
4744 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
4748 BUG_ON(ac->ac_lg != NULL); in ext4_mb_group_or_file()
4750 * locality group prealloc space are per cpu. The reason for having in ext4_mb_group_or_file()
4751 * per cpu locality group is to reduce the contention between block in ext4_mb_group_or_file()
4754 ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); in ext4_mb_group_or_file()
4757 ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; in ext4_mb_group_or_file()
4760 mutex_lock(&ac->ac_lg->lg_mutex); in ext4_mb_group_or_file()
4767 struct super_block *sb = ar->inode->i_sb; in ext4_mb_initialize_context()
4769 struct ext4_super_block *es = sbi->s_es; in ext4_mb_initialize_context()
4776 len = ar->len; in ext4_mb_initialize_context()
4783 goal = ar->goal; in ext4_mb_initialize_context()
4784 if (goal < le32_to_cpu(es->s_first_data_block) || in ext4_mb_initialize_context()
4786 goal = le32_to_cpu(es->s_first_data_block); in ext4_mb_initialize_context()
4790 ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); in ext4_mb_initialize_context()
4791 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_initialize_context()
4792 ac->ac_sb = sb; in ext4_mb_initialize_context()
4793 ac->ac_inode = ar->inode; in ext4_mb_initialize_context()
4794 ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; in ext4_mb_initialize_context()
4795 ac->ac_o_ex.fe_group = group; in ext4_mb_initialize_context()
4796 ac->ac_o_ex.fe_start = block; in ext4_mb_initialize_context()
4797 ac->ac_o_ex.fe_len = len; in ext4_mb_initialize_context()
4798 ac->ac_g_ex = ac->ac_o_ex; in ext4_mb_initialize_context()
4799 ac->ac_flags = ar->flags; in ext4_mb_initialize_context()
4802 * locality group. this is a policy, actually */ in ext4_mb_initialize_context()
4807 (unsigned) ar->len, (unsigned) ar->logical, in ext4_mb_initialize_context()
4808 (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, in ext4_mb_initialize_context()
4809 (unsigned) ar->lleft, (unsigned) ar->pleft, in ext4_mb_initialize_context()
4810 (unsigned) ar->lright, (unsigned) ar->pright, in ext4_mb_initialize_context()
4811 inode_is_open_for_write(ar->inode) ? "" : "non-"); in ext4_mb_initialize_context()
4826 mb_debug(sb, "discard locality group preallocation\n"); in ext4_mb_discard_lg_preallocations()
4830 spin_lock(&lg->lg_prealloc_lock); in ext4_mb_discard_lg_preallocations()
4831 list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order], in ext4_mb_discard_lg_preallocations()
4833 lockdep_is_held(&lg->lg_prealloc_lock)) { in ext4_mb_discard_lg_preallocations()
4834 spin_lock(&pa->pa_lock); in ext4_mb_discard_lg_preallocations()
4835 if (atomic_read(&pa->pa_count)) { in ext4_mb_discard_lg_preallocations()
4841 spin_unlock(&pa->pa_lock); in ext4_mb_discard_lg_preallocations()
4844 if (pa->pa_deleted) { in ext4_mb_discard_lg_preallocations()
4845 spin_unlock(&pa->pa_lock); in ext4_mb_discard_lg_preallocations()
4849 BUG_ON(pa->pa_type != MB_GROUP_PA); in ext4_mb_discard_lg_preallocations()
4853 spin_unlock(&pa->pa_lock); in ext4_mb_discard_lg_preallocations()
4855 list_del_rcu(&pa->pa_inode_list); in ext4_mb_discard_lg_preallocations()
4856 list_add(&pa->u.pa_tmp_list, &discard_list); in ext4_mb_discard_lg_preallocations()
4858 total_entries--; in ext4_mb_discard_lg_preallocations()
4869 spin_unlock(&lg->lg_prealloc_lock); in ext4_mb_discard_lg_preallocations()
4874 group = ext4_get_group_number(sb, pa->pa_pstart); in ext4_mb_discard_lg_preallocations()
4878 ext4_error_err(sb, -err, "Error %d loading buddy information for %u", in ext4_mb_discard_lg_preallocations()
4883 list_del(&pa->pa_group_list); in ext4_mb_discard_lg_preallocations()
4888 list_del(&pa->u.pa_tmp_list); in ext4_mb_discard_lg_preallocations()
4889 call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); in ext4_mb_discard_lg_preallocations()
4905 struct super_block *sb = ac->ac_sb; in ext4_mb_add_n_trim()
4906 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_add_n_trim()
4907 struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; in ext4_mb_add_n_trim()
4909 order = fls(pa->pa_free) - 1; in ext4_mb_add_n_trim()
4910 if (order > PREALLOC_TB_SIZE - 1) in ext4_mb_add_n_trim()
4912 order = PREALLOC_TB_SIZE - 1; in ext4_mb_add_n_trim()
4914 spin_lock(&lg->lg_prealloc_lock); in ext4_mb_add_n_trim()
4915 list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order], in ext4_mb_add_n_trim()
4917 lockdep_is_held(&lg->lg_prealloc_lock)) { in ext4_mb_add_n_trim()
4918 spin_lock(&tmp_pa->pa_lock); in ext4_mb_add_n_trim()
4919 if (tmp_pa->pa_deleted) { in ext4_mb_add_n_trim()
4920 spin_unlock(&tmp_pa->pa_lock); in ext4_mb_add_n_trim()
4923 if (!added && pa->pa_free < tmp_pa->pa_free) { in ext4_mb_add_n_trim()
4925 list_add_tail_rcu(&pa->pa_inode_list, in ext4_mb_add_n_trim()
4926 &tmp_pa->pa_inode_list); in ext4_mb_add_n_trim()
4933 spin_unlock(&tmp_pa->pa_lock); in ext4_mb_add_n_trim()
4937 list_add_tail_rcu(&pa->pa_inode_list, in ext4_mb_add_n_trim()
4938 &lg->lg_prealloc_list[order]); in ext4_mb_add_n_trim()
4939 spin_unlock(&lg->lg_prealloc_lock); in ext4_mb_add_n_trim()
4951 * if per-inode prealloc list is too long, trim some PA
4956 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); in ext4_mb_trim_inode_pa()
4959 count = atomic_read(&ei->i_prealloc_active); in ext4_mb_trim_inode_pa()
4960 delta = (sbi->s_mb_max_inode_prealloc >> 2) + 1; in ext4_mb_trim_inode_pa()
4961 if (count > sbi->s_mb_max_inode_prealloc + delta) { in ext4_mb_trim_inode_pa()
4962 count -= sbi->s_mb_max_inode_prealloc; in ext4_mb_trim_inode_pa()
4972 struct inode *inode = ac->ac_inode; in ext4_mb_release_context()
4974 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_release_context()
4975 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_release_context()
4977 if (pa->pa_type == MB_GROUP_PA) { in ext4_mb_release_context()
4979 spin_lock(&pa->pa_lock); in ext4_mb_release_context()
4980 pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
4981 pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
4982 pa->pa_free -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
4983 pa->pa_len -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
4984 spin_unlock(&pa->pa_lock); in ext4_mb_release_context()
4992 if (likely(pa->pa_free)) { in ext4_mb_release_context()
4993 spin_lock(pa->pa_obj_lock); in ext4_mb_release_context()
4994 list_del_rcu(&pa->pa_inode_list); in ext4_mb_release_context()
4995 spin_unlock(pa->pa_obj_lock); in ext4_mb_release_context()
5000 if (pa->pa_type == MB_INODE_PA) { in ext4_mb_release_context()
5002 * treat per-inode prealloc list as a lru list, then try in ext4_mb_release_context()
5005 spin_lock(pa->pa_obj_lock); in ext4_mb_release_context()
5006 list_move(&pa->pa_inode_list, &ei->i_prealloc_list); in ext4_mb_release_context()
5007 spin_unlock(pa->pa_obj_lock); in ext4_mb_release_context()
5010 ext4_mb_put_pa(ac, ac->ac_sb, pa); in ext4_mb_release_context()
5012 if (ac->ac_bitmap_page) in ext4_mb_release_context()
5013 put_page(ac->ac_bitmap_page); in ext4_mb_release_context()
5014 if (ac->ac_buddy_page) in ext4_mb_release_context()
5015 put_page(ac->ac_buddy_page); in ext4_mb_release_context()
5016 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_release_context()
5017 mutex_unlock(&ac->ac_lg->lg_mutex); in ext4_mb_release_context()
5038 needed -= ret; in ext4_mb_discard_preallocations()
5057 freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); in ext4_mb_discard_preallocations_should_retry()
5063 if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) { in ext4_mb_discard_preallocations_should_retry()
5064 ac->ac_flags |= EXT4_MB_STRICT_CHECK; in ext4_mb_discard_preallocations_should_retry()
5095 sb = ar->inode->i_sb; in ext4_mb_new_blocks()
5099 if (sbi->s_mount_state & EXT4_FC_REPLAY) in ext4_mb_new_blocks()
5103 if (ext4_is_quota_file(ar->inode)) in ext4_mb_new_blocks()
5104 ar->flags |= EXT4_MB_USE_ROOT_BLOCKS; in ext4_mb_new_blocks()
5106 if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) { in ext4_mb_new_blocks()
5111 while (ar->len && in ext4_mb_new_blocks()
5112 ext4_claim_free_clusters(sbi, ar->len, ar->flags)) { in ext4_mb_new_blocks()
5116 ar->len = ar->len >> 1; in ext4_mb_new_blocks()
5118 if (!ar->len) { in ext4_mb_new_blocks()
5120 *errp = -ENOSPC; in ext4_mb_new_blocks()
5123 reserv_clstrs = ar->len; in ext4_mb_new_blocks()
5124 if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) { in ext4_mb_new_blocks()
5125 dquot_alloc_block_nofail(ar->inode, in ext4_mb_new_blocks()
5126 EXT4_C2B(sbi, ar->len)); in ext4_mb_new_blocks()
5128 while (ar->len && in ext4_mb_new_blocks()
5129 dquot_alloc_block(ar->inode, in ext4_mb_new_blocks()
5130 EXT4_C2B(sbi, ar->len))) { in ext4_mb_new_blocks()
5132 ar->flags |= EXT4_MB_HINT_NOPREALLOC; in ext4_mb_new_blocks()
5133 ar->len--; in ext4_mb_new_blocks()
5136 inquota = ar->len; in ext4_mb_new_blocks()
5137 if (ar->len == 0) { in ext4_mb_new_blocks()
5138 *errp = -EDQUOT; in ext4_mb_new_blocks()
5145 ar->len = 0; in ext4_mb_new_blocks()
5146 *errp = -ENOMEM; in ext4_mb_new_blocks()
5152 ar->len = 0; in ext4_mb_new_blocks()
5156 ac->ac_op = EXT4_MB_HISTORY_PREALLOC; in ext4_mb_new_blocks()
5159 ac->ac_op = EXT4_MB_HISTORY_ALLOC; in ext4_mb_new_blocks()
5169 * pa allocated above is added to grp->bb_prealloc_list only in ext4_mb_new_blocks()
5171 * ac->ac_status == AC_STATUS_FOUND. in ext4_mb_new_blocks()
5172 * And error from above mean ac->ac_status != AC_STATUS_FOUND in ext4_mb_new_blocks()
5180 if (ac->ac_status == AC_STATUS_FOUND && in ext4_mb_new_blocks()
5181 ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len) in ext4_mb_new_blocks()
5184 if (likely(ac->ac_status == AC_STATUS_FOUND)) { in ext4_mb_new_blocks()
5190 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_blocks()
5191 ar->len = ac->ac_b_ex.fe_len; in ext4_mb_new_blocks()
5202 *errp = -ENOSPC; in ext4_mb_new_blocks()
5207 ac->ac_b_ex.fe_len = 0; in ext4_mb_new_blocks()
5208 ar->len = 0; in ext4_mb_new_blocks()
5215 if (inquota && ar->len < inquota) in ext4_mb_new_blocks()
5216 dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); in ext4_mb_new_blocks()
5217 if (!ar->len) { in ext4_mb_new_blocks()
5218 if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) in ext4_mb_new_blocks()
5220 percpu_counter_sub(&sbi->s_dirtyclusters_counter, in ext4_mb_new_blocks()
5239 if ((entry->efd_tid != new_entry->efd_tid) || in ext4_try_merge_freed_extent()
5240 (entry->efd_group != new_entry->efd_group)) in ext4_try_merge_freed_extent()
5242 if (entry->efd_start_cluster + entry->efd_count == in ext4_try_merge_freed_extent()
5243 new_entry->efd_start_cluster) { in ext4_try_merge_freed_extent()
5244 new_entry->efd_start_cluster = entry->efd_start_cluster; in ext4_try_merge_freed_extent()
5245 new_entry->efd_count += entry->efd_count; in ext4_try_merge_freed_extent()
5246 } else if (new_entry->efd_start_cluster + new_entry->efd_count == in ext4_try_merge_freed_extent()
5247 entry->efd_start_cluster) { in ext4_try_merge_freed_extent()
5248 new_entry->efd_count += entry->efd_count; in ext4_try_merge_freed_extent()
5251 spin_lock(&sbi->s_md_lock); in ext4_try_merge_freed_extent()
5252 list_del(&entry->efd_list); in ext4_try_merge_freed_extent()
5253 spin_unlock(&sbi->s_md_lock); in ext4_try_merge_freed_extent()
5254 rb_erase(&entry->efd_node, entry_rb_root); in ext4_try_merge_freed_extent()
5262 ext4_group_t group = e4b->bd_group; in ext4_mb_free_metadata()
5264 ext4_grpblk_t clusters = new_entry->efd_count; in ext4_mb_free_metadata()
5266 struct ext4_group_info *db = e4b->bd_info; in ext4_mb_free_metadata()
5267 struct super_block *sb = e4b->bd_sb; in ext4_mb_free_metadata()
5269 struct rb_node **n = &db->bb_free_root.rb_node, *node; in ext4_mb_free_metadata()
5273 BUG_ON(e4b->bd_bitmap_page == NULL); in ext4_mb_free_metadata()
5274 BUG_ON(e4b->bd_buddy_page == NULL); in ext4_mb_free_metadata()
5276 new_node = &new_entry->efd_node; in ext4_mb_free_metadata()
5277 cluster = new_entry->efd_start_cluster; in ext4_mb_free_metadata()
5283 * on-disk bitmap and lose not-yet-available in ext4_mb_free_metadata()
5285 get_page(e4b->bd_buddy_page); in ext4_mb_free_metadata()
5286 get_page(e4b->bd_bitmap_page); in ext4_mb_free_metadata()
5291 if (cluster < entry->efd_start_cluster) in ext4_mb_free_metadata()
5292 n = &(*n)->rb_left; in ext4_mb_free_metadata()
5293 else if (cluster >= (entry->efd_start_cluster + entry->efd_count)) in ext4_mb_free_metadata()
5294 n = &(*n)->rb_right; in ext4_mb_free_metadata()
5299 "Block already on to-be-freed list"); in ext4_mb_free_metadata()
5306 rb_insert_color(new_node, &db->bb_free_root); in ext4_mb_free_metadata()
5313 &(db->bb_free_root)); in ext4_mb_free_metadata()
5320 &(db->bb_free_root)); in ext4_mb_free_metadata()
5323 spin_lock(&sbi->s_md_lock); in ext4_mb_free_metadata()
5324 list_add_tail(&new_entry->efd_list, &sbi->s_freed_data_list); in ext4_mb_free_metadata()
5325 sbi->s_mb_free_pending += clusters; in ext4_mb_free_metadata()
5326 spin_unlock(&sbi->s_md_lock); in ext4_mb_free_metadata()
5339 struct super_block *sb = ar->inode->i_sb; in ext4_mb_new_blocks_simple()
5345 struct ext4_super_block *es = EXT4_SB(sb)->s_es; in ext4_mb_new_blocks_simple()
5347 goal = ar->goal; in ext4_mb_new_blocks_simple()
5348 if (goal < le32_to_cpu(es->s_first_data_block) || in ext4_mb_new_blocks_simple()
5350 goal = le32_to_cpu(es->s_first_data_block); in ext4_mb_new_blocks_simple()
5352 ar->len = 0; in ext4_mb_new_blocks_simple()
5366 i = mb_find_next_zero_bit(bitmap_bh->b_data, max, in ext4_mb_new_blocks_simple()
5382 *errp = -ENOSPC; in ext4_mb_new_blocks_simple()
5388 ar->len = 1; in ext4_mb_new_blocks_simple()
5397 struct super_block *sb = inode->i_sb; in ext4_free_blocks_simple()
5416 if (!mb_test_bit(blkoff + i, bitmap_bh->b_data)) in ext4_free_blocks_simple()
5419 mb_clear_bits(bitmap_bh->b_data, blkoff, count); in ext4_free_blocks_simple()
5425 count - already_freed); in ext4_free_blocks_simple()
5435 * ext4_mb_clear_bb() -- helper function for freeing blocks.
5449 struct super_block *sb = inode->i_sb; in ext4_mb_clear_bb()
5466 ext4_error(sb, "Freeing blocks in system zone - " in ext4_mb_clear_bb()
5486 overflow = EXT4_C2B(sbi, bit) + count - in ext4_mb_clear_bb()
5488 count -= overflow; in ext4_mb_clear_bb()
5501 err = -EIO; in ext4_mb_clear_bb()
5507 ext4_error(sb, "Freeing blocks in system zone - " in ext4_mb_clear_bb()
5520 * to unshare ->b_data if a currently-committing transaction is in ext4_mb_clear_bb()
5531 BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data)); in ext4_mb_clear_bb()
5558 new_entry->efd_start_cluster = bit; in ext4_mb_clear_bb()
5559 new_entry->efd_group = block_group; in ext4_mb_clear_bb()
5560 new_entry->efd_count = count_clusters; in ext4_mb_clear_bb()
5561 new_entry->efd_tid = handle->h_transaction->t_tid; in ext4_mb_clear_bb()
5564 mb_clear_bits(bitmap_bh->b_data, bit, count_clusters); in ext4_mb_clear_bb()
5567 /* need to update group_info->bb_free and bitmap in ext4_mb_clear_bb()
5574 if (err && err != -EOPNOTSUPP) in ext4_mb_clear_bb()
5583 mb_clear_bits(bitmap_bh->b_data, bit, count_clusters); in ext4_mb_clear_bb()
5593 if (sbi->s_log_groups_per_flex) { in ext4_mb_clear_bb()
5597 flex_group)->free_clusters); in ext4_mb_clear_bb()
5608 percpu_counter_add(&sbi->s_freeclusters_counter, in ext4_mb_clear_bb()
5639 * ext4_free_blocks() -- Free given blocks and update quota
5651 struct super_block *sb = inode->i_sb; in ext4_free_blocks()
5657 if (sbi->s_mount_state & EXT4_FC_REPLAY) { in ext4_free_blocks()
5665 BUG_ON(block != bh->b_blocknr); in ext4_free_blocks()
5667 block = bh->b_blocknr; in ext4_free_blocks()
5672 ext4_error(sb, "Freeing blocks not in datazone - " in ext4_free_blocks()
5698 overflow = sbi->s_cluster_ratio - overflow; in ext4_free_blocks()
5701 count -= overflow; in ext4_free_blocks()
5705 block -= overflow; in ext4_free_blocks()
5715 count -= overflow; in ext4_free_blocks()
5719 count += sbi->s_cluster_ratio - overflow; in ext4_free_blocks()
5731 bh = sb_find_get_block(inode->i_sb, block + i); in ext4_free_blocks()
5741 * ext4_group_add_blocks() -- Add given blocks to an existing group
5763 ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1); in ext4_group_add_blocks()
5764 unsigned long cluster_count = last_cluster - first_cluster + 1; in ext4_group_add_blocks()
5766 ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1); in ext4_group_add_blocks()
5779 err = -EINVAL; in ext4_group_add_blocks()
5792 err = -EIO; in ext4_group_add_blocks()
5797 ext4_error(sb, "Adding blocks in system zones - " in ext4_group_add_blocks()
5800 err = -EINVAL; in ext4_group_add_blocks()
5811 * to unshare ->b_data if a currently-committing transaction is in ext4_group_add_blocks()
5821 if (!mb_test_bit(bit + i, bitmap_bh->b_data)) { in ext4_group_add_blocks()
5835 * need to update group_info->bb_free and bitmap in ext4_group_add_blocks()
5840 mb_clear_bits(bitmap_bh->b_data, bit, cluster_count); in ext4_group_add_blocks()
5848 percpu_counter_add(&sbi->s_freeclusters_counter, in ext4_group_add_blocks()
5851 if (sbi->s_log_groups_per_flex) { in ext4_group_add_blocks()
5855 flex_group)->free_clusters); in ext4_group_add_blocks()
5877 * ext4_trim_extent -- function to TRIM one single free extent in the group
5917 * ext4_trim_all_free -- function to trim all free space in alloc. group
5956 minblocks >= atomic_read(&EXT4_SB(sb)->s_last_trim_minblks)) in ext4_trim_all_free()
5959 start = (e4b.bd_info->bb_first_free > start) ? in ext4_trim_all_free()
5960 e4b.bd_info->bb_first_free : start; in ext4_trim_all_free()
5968 if ((next - start) >= minblocks) { in ext4_trim_all_free()
5970 next - start, group, &e4b); in ext4_trim_all_free()
5971 if (ret && ret != -EOPNOTSUPP) in ext4_trim_all_free()
5974 count += next - start; in ext4_trim_all_free()
5976 free_count += next - start; in ext4_trim_all_free()
5980 count = -ERESTARTSYS; in ext4_trim_all_free()
5990 if ((e4b.bd_info->bb_free - free_count) < minblocks) in ext4_trim_all_free()
6009 * ext4_trim_fs() -- trim ioctl handle function
6022 struct request_queue *q = bdev_get_queue(sb->s_bdev); in ext4_trim_fs()
6028 le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); in ext4_trim_fs()
6029 ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es); in ext4_trim_fs()
6032 start = range->start >> sb->s_blocksize_bits; in ext4_trim_fs()
6033 end = start + (range->len >> sb->s_blocksize_bits) - 1; in ext4_trim_fs()
6035 range->minlen >> sb->s_blocksize_bits); in ext4_trim_fs()
6039 range->len < sb->s_blocksize) in ext4_trim_fs()
6040 return -EINVAL; in ext4_trim_fs()
6042 if (range->minlen < q->limits.discard_granularity) { in ext4_trim_fs()
6044 q->limits.discard_granularity >> sb->s_blocksize_bits); in ext4_trim_fs()
6049 end = max_blks - 1; in ext4_trim_fs()
6062 end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; in ext4_trim_fs()
6077 * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to in ext4_trim_fs()
6084 if (grp->bb_free >= minlen) { in ext4_trim_fs()
6102 atomic_set(&EXT4_SB(sb)->s_last_trim_minblks, minlen); in ext4_trim_fs()
6105 range->len = EXT4_C2B(EXT4_SB(sb), trimmed) << sb->s_blocksize_bits; in ext4_trim_fs()
6131 start = (e4b.bd_info->bb_first_free > start) ? in ext4_mballoc_query_range()
6132 e4b.bd_info->bb_first_free : start; in ext4_mballoc_query_range()
6134 end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; in ext4_mballoc_query_range()
6143 error = formatter(sb, group, start, next - start, priv); in ext4_mballoc_query_range()