Lines Matching refs:ctl
38 static int link_free_space(struct btrfs_free_space_ctl *ctl,
40 static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
42 static int search_bitmap(struct btrfs_free_space_ctl *ctl,
45 static void free_bitmap(struct btrfs_free_space_ctl *ctl,
47 static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl,
51 static void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl) in __btrfs_remove_free_space_cache() argument
56 while ((node = rb_last(&ctl->free_space_offset)) != NULL) { in __btrfs_remove_free_space_cache()
59 unlink_free_space(ctl, info, true); in __btrfs_remove_free_space_cache()
62 free_bitmap(ctl, info); in __btrfs_remove_free_space_cache()
65 cond_resched_lock(&ctl->tree_lock); in __btrfs_remove_free_space_cache()
699 static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) in recalculate_thresholds() argument
701 struct btrfs_block_group *block_group = ctl->block_group; in recalculate_thresholds()
706 u64 bytes_per_bg = BITS_PER_BITMAP * ctl->unit; in recalculate_thresholds()
711 if (ctl->total_bitmaps > max_bitmaps) in recalculate_thresholds()
715 ctl->total_bitmaps, ctl->unit, max_bitmaps, in recalculate_thresholds()
717 ASSERT(ctl->total_bitmaps <= max_bitmaps); in recalculate_thresholds()
730 bitmap_bytes = ctl->total_bitmaps * ctl->unit; in recalculate_thresholds()
739 ctl->extents_thresh = in recalculate_thresholds()
744 struct btrfs_free_space_ctl *ctl, in __load_free_space_cache() argument
842 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
843 ret = link_free_space(ctl, e); in __load_free_space_cache()
844 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
862 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
863 ret = link_free_space(ctl, e); in __load_free_space_cache()
865 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
871 ctl->total_bitmaps++; in __load_free_space_cache()
872 recalculate_thresholds(ctl); in __load_free_space_cache()
873 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
901 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
902 __btrfs_remove_free_space_cache(ctl); in __load_free_space_cache()
903 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
908 struct btrfs_free_space_ctl *ctl) in copy_free_space_cache() argument
914 while (!ret && (n = rb_first(&ctl->free_space_offset)) != NULL) { in copy_free_space_cache()
917 unlink_free_space(ctl, info, true); in copy_free_space_cache()
923 u64 bytes = ctl->unit; in copy_free_space_cache()
925 while (search_bitmap(ctl, info, &offset, &bytes, in copy_free_space_cache()
931 bitmap_clear_bits(ctl, info, offset, bytes, true); in copy_free_space_cache()
933 bytes = ctl->unit; in copy_free_space_cache()
935 free_bitmap(ctl, info); in copy_free_space_cache()
947 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in load_free_space_cache() local
1064 spin_lock(&ctl->tree_lock); in load_free_space_cache()
1066 spin_unlock(&ctl->tree_lock); in load_free_space_cache()
1073 struct btrfs_free_space_ctl *ctl, in write_cache_extent_entries() argument
1081 struct rb_node *node = rb_first(&ctl->free_space_offset); in write_cache_extent_entries()
1133 list_for_each_entry(trim_entry, &ctl->trimming_ranges, list) { in write_cache_extent_entries()
1382 struct btrfs_free_space_ctl *ctl, in __btrfs_write_out_cache() argument
1427 mutex_lock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1429 spin_lock(&ctl->tree_lock); in __btrfs_write_out_cache()
1430 ret = write_cache_extent_entries(io_ctl, ctl, in __btrfs_write_out_cache()
1454 spin_unlock(&ctl->tree_lock); in __btrfs_write_out_cache()
1455 mutex_unlock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1497 spin_unlock(&ctl->tree_lock); in __btrfs_write_out_cache()
1498 mutex_unlock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1525 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_write_out_cache() local
1540 ret = __btrfs_write_out_cache(fs_info->tree_root, inode, ctl, in btrfs_write_out_cache()
1575 static inline u64 offset_to_bitmap(struct btrfs_free_space_ctl *ctl, in offset_to_bitmap() argument
1581 bytes_per_bitmap = BITS_PER_BITMAP * ctl->unit; in offset_to_bitmap()
1582 bitmap_start = offset - ctl->start; in offset_to_bitmap()
1585 bitmap_start += ctl->start; in offset_to_bitmap()
1693 tree_search_offset(struct btrfs_free_space_ctl *ctl, in tree_search_offset() argument
1696 struct rb_node *n = ctl->free_space_offset.rb_node; in tree_search_offset()
1779 if (entry->offset + BITS_PER_BITMAP * ctl->unit > offset) in tree_search_offset()
1794 ctl->unit > offset) in tree_search_offset()
1804 static inline void unlink_free_space(struct btrfs_free_space_ctl *ctl, in unlink_free_space() argument
1808 rb_erase(&info->offset_index, &ctl->free_space_offset); in unlink_free_space()
1809 rb_erase_cached(&info->bytes_index, &ctl->free_space_bytes); in unlink_free_space()
1810 ctl->free_extents--; in unlink_free_space()
1813 ctl->discardable_extents[BTRFS_STAT_CURR]--; in unlink_free_space()
1814 ctl->discardable_bytes[BTRFS_STAT_CURR] -= info->bytes; in unlink_free_space()
1818 ctl->free_space -= info->bytes; in unlink_free_space()
1821 static int link_free_space(struct btrfs_free_space_ctl *ctl, in link_free_space() argument
1827 ret = tree_insert_offset(&ctl->free_space_offset, info->offset, in link_free_space()
1832 rb_add_cached(&info->bytes_index, &ctl->free_space_bytes, entry_less); in link_free_space()
1835 ctl->discardable_extents[BTRFS_STAT_CURR]++; in link_free_space()
1836 ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes; in link_free_space()
1839 ctl->free_space += info->bytes; in link_free_space()
1840 ctl->free_extents++; in link_free_space()
1844 static void relink_bitmap_entry(struct btrfs_free_space_ctl *ctl, in relink_bitmap_entry() argument
1856 rb_erase_cached(&info->bytes_index, &ctl->free_space_bytes); in relink_bitmap_entry()
1857 rb_add_cached(&info->bytes_index, &ctl->free_space_bytes, entry_less); in relink_bitmap_entry()
1860 static inline void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, in bitmap_clear_bits() argument
1867 start = offset_to_bit(info->offset, ctl->unit, offset); in bitmap_clear_bits()
1868 count = bytes_to_bits(bytes, ctl->unit); in bitmap_clear_bits()
1875 if (info->max_extent_size > ctl->unit) in bitmap_clear_bits()
1878 relink_bitmap_entry(ctl, info); in bitmap_clear_bits()
1888 ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta; in bitmap_clear_bits()
1889 ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes; in bitmap_clear_bits()
1893 ctl->free_space -= bytes; in bitmap_clear_bits()
1896 static void bitmap_set_bits(struct btrfs_free_space_ctl *ctl, in bitmap_set_bits() argument
1903 start = offset_to_bit(info->offset, ctl->unit, offset); in bitmap_set_bits()
1904 count = bytes_to_bits(bytes, ctl->unit); in bitmap_set_bits()
1916 ctl->free_space += bytes; in bitmap_set_bits()
1918 relink_bitmap_entry(ctl, info); in bitmap_set_bits()
1928 ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta; in bitmap_set_bits()
1929 ctl->discardable_bytes[BTRFS_STAT_CURR] += bytes; in bitmap_set_bits()
1937 static int search_bitmap(struct btrfs_free_space_ctl *ctl, in search_bitmap() argument
1958 i = offset_to_bit(bitmap_info->offset, ctl->unit, in search_bitmap()
1960 bits = bytes_to_bits(*bytes, ctl->unit); in search_bitmap()
1980 *offset = (u64)(i * ctl->unit) + bitmap_info->offset; in search_bitmap()
1981 *bytes = (u64)(found_bits) * ctl->unit; in search_bitmap()
1985 *bytes = (u64)(max_bits) * ctl->unit; in search_bitmap()
1987 relink_bitmap_entry(ctl, bitmap_info); in search_bitmap()
1993 find_free_space(struct btrfs_free_space_ctl *ctl, u64 *offset, u64 *bytes, in find_free_space() argument
2002 if (!ctl->free_space_offset.rb_node) in find_free_space()
2006 node = rb_first_cached(&ctl->free_space_bytes); in find_free_space()
2008 entry = tree_search_offset(ctl, offset_to_bitmap(ctl, *offset), in find_free_space()
2043 tmp = entry->offset - ctl->start + align - 1; in find_free_space()
2045 tmp = tmp * align + ctl->start; in find_free_space()
2069 ret = search_bitmap(ctl, entry, &tmp, &size, true); in find_free_space()
2099 static void add_new_bitmap(struct btrfs_free_space_ctl *ctl, in add_new_bitmap() argument
2102 info->offset = offset_to_bitmap(ctl, offset); in add_new_bitmap()
2106 link_free_space(ctl, info); in add_new_bitmap()
2107 ctl->total_bitmaps++; in add_new_bitmap()
2108 recalculate_thresholds(ctl); in add_new_bitmap()
2111 static void free_bitmap(struct btrfs_free_space_ctl *ctl, in free_bitmap() argument
2121 ctl->discardable_extents[BTRFS_STAT_CURR] -= in free_bitmap()
2123 ctl->discardable_bytes[BTRFS_STAT_CURR] -= bitmap_info->bytes; in free_bitmap()
2126 unlink_free_space(ctl, bitmap_info, true); in free_bitmap()
2129 ctl->total_bitmaps--; in free_bitmap()
2130 recalculate_thresholds(ctl); in free_bitmap()
2133 static noinline int remove_from_bitmap(struct btrfs_free_space_ctl *ctl, in remove_from_bitmap() argument
2142 end = bitmap_info->offset + (u64)(BITS_PER_BITMAP * ctl->unit) - 1; in remove_from_bitmap()
2151 search_bytes = ctl->unit; in remove_from_bitmap()
2153 ret = search_bitmap(ctl, bitmap_info, &search_start, &search_bytes, in remove_from_bitmap()
2164 bitmap_clear_bits(ctl, bitmap_info, search_start, search_bytes, true); in remove_from_bitmap()
2171 free_bitmap(ctl, bitmap_info); in remove_from_bitmap()
2197 search_bytes = ctl->unit; in remove_from_bitmap()
2198 ret = search_bitmap(ctl, bitmap_info, &search_start, in remove_from_bitmap()
2205 free_bitmap(ctl, bitmap_info); in remove_from_bitmap()
2210 static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, in add_bytes_to_bitmap() argument
2223 ctl->discardable_extents[BTRFS_STAT_CURR] += in add_bytes_to_bitmap()
2225 ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes; in add_bytes_to_bitmap()
2230 end = info->offset + (u64)(BITS_PER_BITMAP * ctl->unit); in add_bytes_to_bitmap()
2234 bitmap_set_bits(ctl, info, offset, bytes_to_set); in add_bytes_to_bitmap()
2240 static bool use_bitmap(struct btrfs_free_space_ctl *ctl, in use_bitmap() argument
2243 struct btrfs_block_group *block_group = ctl->block_group; in use_bitmap()
2260 if (!forced && ctl->free_extents < ctl->extents_thresh) { in use_bitmap()
2269 if (ctl->free_extents * 3 <= ctl->extents_thresh) in use_bitmap()
2284 if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->length) in use_bitmap()
2294 static int insert_into_bitmap(struct btrfs_free_space_ctl *ctl, in insert_into_bitmap() argument
2308 if (!ctl->op->use_bitmap(ctl, info)) in insert_into_bitmap()
2311 if (ctl->op == &free_space_op) in insert_into_bitmap()
2312 block_group = ctl->block_group; in insert_into_bitmap()
2340 if (entry->offset == offset_to_bitmap(ctl, offset)) { in insert_into_bitmap()
2341 bytes_added = add_bytes_to_bitmap(ctl, entry, offset, in insert_into_bitmap()
2354 bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in insert_into_bitmap()
2361 bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes, in insert_into_bitmap()
2375 add_new_bitmap(ctl, info, offset); in insert_into_bitmap()
2380 spin_unlock(&ctl->tree_lock); in insert_into_bitmap()
2387 spin_lock(&ctl->tree_lock); in insert_into_bitmap()
2397 spin_lock(&ctl->tree_lock); in insert_into_bitmap()
2432 static bool try_merge_free_space(struct btrfs_free_space_ctl *ctl, in try_merge_free_space() argument
2447 right_info = tree_search_offset(ctl, offset + bytes, 0, 0); in try_merge_free_space()
2452 left_info = tree_search_offset(ctl, offset - 1, 0, 0); in try_merge_free_space()
2457 unlink_free_space(ctl, right_info, update_stat); in try_merge_free_space()
2467 unlink_free_space(ctl, left_info, update_stat); in try_merge_free_space()
2477 static bool steal_from_bitmap_to_end(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap_to_end() argument
2485 const u64 bitmap_offset = offset_to_bitmap(ctl, end); in steal_from_bitmap_to_end()
2488 bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); in steal_from_bitmap_to_end()
2492 i = offset_to_bit(bitmap->offset, ctl->unit, end); in steal_from_bitmap_to_end()
2496 bytes = (j - i) * ctl->unit; in steal_from_bitmap_to_end()
2503 bitmap_clear_bits(ctl, bitmap, end, bytes, update_stat); in steal_from_bitmap_to_end()
2506 free_bitmap(ctl, bitmap); in steal_from_bitmap_to_end()
2511 static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap_to_front() argument
2522 bitmap_offset = offset_to_bitmap(ctl, info->offset); in steal_from_bitmap_to_front()
2527 bitmap_offset = offset_to_bitmap(ctl, info->offset - 1); in steal_from_bitmap_to_front()
2530 bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); in steal_from_bitmap_to_front()
2534 i = offset_to_bit(bitmap->offset, ctl->unit, info->offset) - 1; in steal_from_bitmap_to_front()
2546 bytes = (i + 1) * ctl->unit; in steal_from_bitmap_to_front()
2548 bytes = (i - prev_j) * ctl->unit; in steal_from_bitmap_to_front()
2557 bitmap_clear_bits(ctl, bitmap, info->offset, bytes, update_stat); in steal_from_bitmap_to_front()
2560 free_bitmap(ctl, bitmap); in steal_from_bitmap_to_front()
2576 static void steal_from_bitmap(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap() argument
2587 if (ctl->total_bitmaps > 0) { in steal_from_bitmap()
2591 stole_end = steal_from_bitmap_to_end(ctl, info, update_stat); in steal_from_bitmap()
2592 if (ctl->total_bitmaps > 0) in steal_from_bitmap()
2593 stole_front = steal_from_bitmap_to_front(ctl, info, in steal_from_bitmap()
2597 try_merge_free_space(ctl, info, update_stat); in steal_from_bitmap()
2606 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in __btrfs_add_free_space() local
2623 spin_lock(&ctl->tree_lock); in __btrfs_add_free_space()
2625 if (try_merge_free_space(ctl, info, true)) in __btrfs_add_free_space()
2633 ret = insert_into_bitmap(ctl, info); in __btrfs_add_free_space()
2647 steal_from_bitmap(ctl, info, true); in __btrfs_add_free_space()
2651 ret = link_free_space(ctl, info); in __btrfs_add_free_space()
2656 spin_unlock(&ctl->tree_lock); in __btrfs_add_free_space()
2675 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in __btrfs_add_free_space_zoned() local
2687 spin_lock(&ctl->tree_lock); in __btrfs_add_free_space_zoned()
2700 ctl->free_space += to_free; in __btrfs_add_free_space_zoned()
2707 spin_unlock(&ctl->tree_lock); in __btrfs_add_free_space_zoned()
2778 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_remove_free_space() local
2803 spin_lock(&ctl->tree_lock); in btrfs_remove_free_space()
2810 info = tree_search_offset(ctl, offset, 0, 0); in btrfs_remove_free_space()
2816 info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in btrfs_remove_free_space()
2831 unlink_free_space(ctl, info, true); in btrfs_remove_free_space()
2838 ret = link_free_space(ctl, info); in btrfs_remove_free_space()
2851 ret = link_free_space(ctl, info); in btrfs_remove_free_space()
2865 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space()
2876 ret = remove_from_bitmap(ctl, info, &offset, &bytes); in btrfs_remove_free_space()
2883 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space()
2892 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_dump_free_space() local
2909 spin_lock(&ctl->tree_lock); in btrfs_dump_free_space()
2910 for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) { in btrfs_dump_free_space()
2918 spin_unlock(&ctl->tree_lock); in btrfs_dump_free_space()
2926 struct btrfs_free_space_ctl *ctl) in btrfs_init_free_space_ctl() argument
2930 spin_lock_init(&ctl->tree_lock); in btrfs_init_free_space_ctl()
2931 ctl->unit = fs_info->sectorsize; in btrfs_init_free_space_ctl()
2932 ctl->start = block_group->start; in btrfs_init_free_space_ctl()
2933 ctl->block_group = block_group; in btrfs_init_free_space_ctl()
2934 ctl->op = &free_space_op; in btrfs_init_free_space_ctl()
2935 ctl->free_space_bytes = RB_ROOT_CACHED; in btrfs_init_free_space_ctl()
2936 INIT_LIST_HEAD(&ctl->trimming_ranges); in btrfs_init_free_space_ctl()
2937 mutex_init(&ctl->cache_writeout_mutex); in btrfs_init_free_space_ctl()
2944 ctl->extents_thresh = (SZ_32K / 2) / sizeof(struct btrfs_free_space); in btrfs_init_free_space_ctl()
2957 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in __btrfs_return_cluster_to_free_space() local
2984 ctl->discardable_extents[BTRFS_STAT_CURR]--; in __btrfs_return_cluster_to_free_space()
2985 ctl->discardable_bytes[BTRFS_STAT_CURR] -= in __btrfs_return_cluster_to_free_space()
2989 try_merge_free_space(ctl, entry, false); in __btrfs_return_cluster_to_free_space()
2990 steal_from_bitmap(ctl, entry, false); in __btrfs_return_cluster_to_free_space()
2994 ctl->discardable_extents[BTRFS_STAT_CURR]++; in __btrfs_return_cluster_to_free_space()
2995 ctl->discardable_bytes[BTRFS_STAT_CURR] += in __btrfs_return_cluster_to_free_space()
2999 tree_insert_offset(&ctl->free_space_offset, in __btrfs_return_cluster_to_free_space()
3001 rb_add_cached(&entry->bytes_index, &ctl->free_space_bytes, in __btrfs_return_cluster_to_free_space()
3011 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_remove_free_space_cache() local
3015 spin_lock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
3024 cond_resched_lock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
3026 __btrfs_remove_free_space_cache(ctl); in btrfs_remove_free_space_cache()
3028 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
3040 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_is_free_space_trimmed() local
3045 spin_lock(&ctl->tree_lock); in btrfs_is_free_space_trimmed()
3046 node = rb_first(&ctl->free_space_offset); in btrfs_is_free_space_trimmed()
3059 spin_unlock(&ctl->tree_lock); in btrfs_is_free_space_trimmed()
3067 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_find_space_for_alloc() local
3080 spin_lock(&ctl->tree_lock); in btrfs_find_space_for_alloc()
3081 entry = find_free_space(ctl, &offset, &bytes_search, in btrfs_find_space_for_alloc()
3089 bitmap_clear_bits(ctl, entry, offset, bytes, true); in btrfs_find_space_for_alloc()
3095 free_bitmap(ctl, entry); in btrfs_find_space_for_alloc()
3097 unlink_free_space(ctl, entry, true); in btrfs_find_space_for_alloc()
3112 link_free_space(ctl, entry); in btrfs_find_space_for_alloc()
3116 spin_unlock(&ctl->tree_lock); in btrfs_find_space_for_alloc()
3136 struct btrfs_free_space_ctl *ctl; in btrfs_return_cluster_to_free_space() local
3154 ctl = block_group->free_space_ctl; in btrfs_return_cluster_to_free_space()
3157 spin_lock(&ctl->tree_lock); in btrfs_return_cluster_to_free_space()
3159 spin_unlock(&ctl->tree_lock); in btrfs_return_cluster_to_free_space()
3173 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_alloc_from_bitmap() local
3182 err = search_bitmap(ctl, entry, &search_start, &search_bytes, true); in btrfs_alloc_from_bitmap()
3190 bitmap_clear_bits(ctl, entry, ret, bytes, false); in btrfs_alloc_from_bitmap()
3204 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_alloc_from_cluster() local
3269 spin_lock(&ctl->tree_lock); in btrfs_alloc_from_cluster()
3274 ctl->free_space -= bytes; in btrfs_alloc_from_cluster()
3276 ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes; in btrfs_alloc_from_cluster()
3281 ctl->free_extents--; in btrfs_alloc_from_cluster()
3285 ctl->total_bitmaps--; in btrfs_alloc_from_cluster()
3286 recalculate_thresholds(ctl); in btrfs_alloc_from_cluster()
3288 ctl->discardable_extents[BTRFS_STAT_CURR]--; in btrfs_alloc_from_cluster()
3294 spin_unlock(&ctl->tree_lock); in btrfs_alloc_from_cluster()
3305 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_bitmap_cluster() local
3316 i = offset_to_bit(entry->offset, ctl->unit, in btrfs_bitmap_cluster()
3318 want_bits = bytes_to_bits(bytes, ctl->unit); in btrfs_bitmap_cluster()
3319 min_bits = bytes_to_bits(min_bytes, ctl->unit); in btrfs_bitmap_cluster()
3345 entry->max_extent_size = (u64)max_bits * ctl->unit; in btrfs_bitmap_cluster()
3356 if (cluster->max_size < found_bits * ctl->unit) in btrfs_bitmap_cluster()
3357 cluster->max_size = found_bits * ctl->unit; in btrfs_bitmap_cluster()
3364 cluster->window_start = start * ctl->unit + entry->offset; in btrfs_bitmap_cluster()
3365 rb_erase(&entry->offset_index, &ctl->free_space_offset); in btrfs_bitmap_cluster()
3366 rb_erase_cached(&entry->bytes_index, &ctl->free_space_bytes); in btrfs_bitmap_cluster()
3382 total_found * ctl->unit, 1); in btrfs_bitmap_cluster()
3397 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in setup_cluster_no_bitmap() local
3406 entry = tree_search_offset(ctl, offset, 0, 1); in setup_cluster_no_bitmap()
3466 rb_erase(&entry->offset_index, &ctl->free_space_offset); in setup_cluster_no_bitmap()
3467 rb_erase_cached(&entry->bytes_index, &ctl->free_space_bytes); in setup_cluster_no_bitmap()
3489 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in setup_cluster_bitmap() local
3492 u64 bitmap_offset = offset_to_bitmap(ctl, offset); in setup_cluster_bitmap()
3494 if (ctl->total_bitmaps == 0) in setup_cluster_bitmap()
3505 entry = tree_search_offset(ctl, bitmap_offset, 1, 0); in setup_cluster_bitmap()
3539 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_find_space_cluster() local
3563 spin_lock(&ctl->tree_lock); in btrfs_find_space_cluster()
3569 if (ctl->free_space < bytes) { in btrfs_find_space_cluster()
3570 spin_unlock(&ctl->tree_lock); in btrfs_find_space_cluster()
3607 spin_unlock(&ctl->tree_lock); in btrfs_find_space_cluster()
3634 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in do_trimming() local
3658 mutex_lock(&ctl->cache_writeout_mutex); in do_trimming()
3668 mutex_unlock(&ctl->cache_writeout_mutex); in do_trimming()
3693 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in trim_no_bitmap() local
3706 mutex_lock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3707 spin_lock(&ctl->tree_lock); in trim_no_bitmap()
3709 if (ctl->free_space < minlen) in trim_no_bitmap()
3712 entry = tree_search_offset(ctl, start, 0, 1); in trim_no_bitmap()
3736 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3737 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3740 unlink_free_space(ctl, entry, true); in trim_no_bitmap()
3753 link_free_space(ctl, entry); in trim_no_bitmap()
3761 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3762 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3766 unlink_free_space(ctl, entry, true); in trim_no_bitmap()
3770 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3773 list_add_tail(&trim_entry.list, &ctl->trimming_ranges); in trim_no_bitmap()
3774 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3801 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3802 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3821 static void reset_trimming_bitmap(struct btrfs_free_space_ctl *ctl, u64 offset) in reset_trimming_bitmap() argument
3825 spin_lock(&ctl->tree_lock); in reset_trimming_bitmap()
3826 entry = tree_search_offset(ctl, offset, 1, 0); in reset_trimming_bitmap()
3829 ctl->discardable_extents[BTRFS_STAT_CURR] += in reset_trimming_bitmap()
3831 ctl->discardable_bytes[BTRFS_STAT_CURR] += entry->bytes; in reset_trimming_bitmap()
3836 spin_unlock(&ctl->tree_lock); in reset_trimming_bitmap()
3839 static void end_trimming_bitmap(struct btrfs_free_space_ctl *ctl, in end_trimming_bitmap() argument
3844 ctl->discardable_extents[BTRFS_STAT_CURR] -= in end_trimming_bitmap()
3846 ctl->discardable_bytes[BTRFS_STAT_CURR] -= entry->bytes; in end_trimming_bitmap()
3859 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in trim_bitmaps() local
3864 u64 offset = offset_to_bitmap(ctl, start); in trim_bitmaps()
3871 mutex_lock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3872 spin_lock(&ctl->tree_lock); in trim_bitmaps()
3874 if (ctl->free_space < minlen) { in trim_bitmaps()
3877 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3878 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3882 entry = tree_search_offset(ctl, offset, 1, 0); in trim_bitmaps()
3893 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3894 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3909 ret2 = search_bitmap(ctl, entry, &start, &bytes, false); in trim_bitmaps()
3916 end_trimming_bitmap(ctl, entry); in trim_bitmaps()
3919 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3920 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3930 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3931 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3937 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3938 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3953 bitmap_clear_bits(ctl, entry, start, bytes, true); in trim_bitmaps()
3955 free_bitmap(ctl, entry); in trim_bitmaps()
3957 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3960 list_add_tail(&trim_entry.list, &ctl->trimming_ranges); in trim_bitmaps()
3961 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3966 reset_trimming_bitmap(ctl, offset); in trim_bitmaps()
3973 offset += BITS_PER_BITMAP * ctl->unit; in trim_bitmaps()
3982 reset_trimming_bitmap(ctl, offset); in trim_bitmaps()
4000 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_trim_block_group() local
4021 div64_u64_rem(end, BITS_PER_BITMAP * ctl->unit, &rem); in btrfs_trim_block_group()
4024 reset_trimming_bitmap(ctl, offset_to_bitmap(ctl, end)); in btrfs_trim_block_group()
4146 struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; in test_add_free_space_entry() local
4161 spin_lock(&ctl->tree_lock); in test_add_free_space_entry()
4165 ret = link_free_space(ctl, info); in test_add_free_space_entry()
4166 spin_unlock(&ctl->tree_lock); in test_add_free_space_entry()
4180 spin_lock(&ctl->tree_lock); in test_add_free_space_entry()
4181 bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in test_add_free_space_entry()
4186 add_new_bitmap(ctl, info, offset); in test_add_free_space_entry()
4191 bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes, in test_add_free_space_entry()
4196 spin_unlock(&ctl->tree_lock); in test_add_free_space_entry()
4216 struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; in test_check_exists() local
4220 spin_lock(&ctl->tree_lock); in test_check_exists()
4221 info = tree_search_offset(ctl, offset, 0, 0); in test_check_exists()
4223 info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in test_check_exists()
4236 bit_bytes = ctl->unit; in test_check_exists()
4237 ret = search_bitmap(ctl, info, &bit_off, &bit_bytes, false); in test_check_exists()
4289 spin_unlock(&ctl->tree_lock); in test_check_exists()