Lines Matching full:ctl
31 static int link_free_space(struct btrfs_free_space_ctl *ctl,
33 static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
636 static void merge_space_tree(struct btrfs_free_space_ctl *ctl) in merge_space_tree() argument
642 spin_lock(&ctl->tree_lock); in merge_space_tree()
643 for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) { in merge_space_tree()
650 unlink_free_space(ctl, prev); in merge_space_tree()
651 unlink_free_space(ctl, e); in merge_space_tree()
654 link_free_space(ctl, prev); in merge_space_tree()
656 spin_unlock(&ctl->tree_lock); in merge_space_tree()
662 spin_unlock(&ctl->tree_lock); in merge_space_tree()
666 struct btrfs_free_space_ctl *ctl, in __load_free_space_cache() argument
761 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
762 ret = link_free_space(ctl, e); in __load_free_space_cache()
763 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
780 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
781 ret = link_free_space(ctl, e); in __load_free_space_cache()
782 ctl->total_bitmaps++; in __load_free_space_cache()
783 ctl->op->recalc_thresholds(ctl); in __load_free_space_cache()
784 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
811 merge_space_tree(ctl); in __load_free_space_cache()
818 __btrfs_remove_free_space_cache(ctl); in __load_free_space_cache()
825 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in load_free_space_cache() local
883 ret = __load_free_space_cache(fs_info->tree_root, inode, ctl, in load_free_space_cache()
889 spin_lock(&ctl->tree_lock); in load_free_space_cache()
890 matched = (ctl->free_space == (block_group->key.offset - used - in load_free_space_cache()
892 spin_unlock(&ctl->tree_lock); in load_free_space_cache()
895 __btrfs_remove_free_space_cache(ctl); in load_free_space_cache()
920 struct btrfs_free_space_ctl *ctl, in write_cache_extent_entries() argument
928 struct rb_node *node = rb_first(&ctl->free_space_offset); in write_cache_extent_entries()
980 list_for_each_entry(trim_entry, &ctl->trimming_ranges, list) { in write_cache_extent_entries()
1230 * @ctl - the free space cache we are going to write out
1239 struct btrfs_free_space_ctl *ctl, in __btrfs_write_out_cache() argument
1285 mutex_lock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1287 spin_lock(&ctl->tree_lock); in __btrfs_write_out_cache()
1288 ret = write_cache_extent_entries(io_ctl, ctl, in __btrfs_write_out_cache()
1313 spin_unlock(&ctl->tree_lock); in __btrfs_write_out_cache()
1314 mutex_unlock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1367 spin_unlock(&ctl->tree_lock); in __btrfs_write_out_cache()
1368 mutex_unlock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1385 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_write_out_cache() local
1400 ret = __btrfs_write_out_cache(fs_info->tree_root, inode, ctl, in btrfs_write_out_cache()
1437 static inline u64 offset_to_bitmap(struct btrfs_free_space_ctl *ctl, in offset_to_bitmap() argument
1443 bytes_per_bitmap = BITS_PER_BITMAP * ctl->unit; in offset_to_bitmap()
1444 bitmap_start = offset - ctl->start; in offset_to_bitmap()
1447 bitmap_start += ctl->start; in offset_to_bitmap()
1511 tree_search_offset(struct btrfs_free_space_ctl *ctl, in tree_search_offset() argument
1514 struct rb_node *n = ctl->free_space_offset.rb_node; in tree_search_offset()
1600 if (entry->offset + BITS_PER_BITMAP * ctl->unit > offset) in tree_search_offset()
1611 ctl->unit > offset) in tree_search_offset()
1627 __unlink_free_space(struct btrfs_free_space_ctl *ctl, in __unlink_free_space() argument
1630 rb_erase(&info->offset_index, &ctl->free_space_offset); in __unlink_free_space()
1631 ctl->free_extents--; in __unlink_free_space()
1634 static void unlink_free_space(struct btrfs_free_space_ctl *ctl, in unlink_free_space() argument
1637 __unlink_free_space(ctl, info); in unlink_free_space()
1638 ctl->free_space -= info->bytes; in unlink_free_space()
1641 static int link_free_space(struct btrfs_free_space_ctl *ctl, in link_free_space() argument
1647 ret = tree_insert_offset(&ctl->free_space_offset, info->offset, in link_free_space()
1652 ctl->free_space += info->bytes; in link_free_space()
1653 ctl->free_extents++; in link_free_space()
1657 static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) in recalculate_thresholds() argument
1659 struct btrfs_block_group_cache *block_group = ctl->private; in recalculate_thresholds()
1664 u64 bytes_per_bg = BITS_PER_BITMAP * ctl->unit; in recalculate_thresholds()
1669 ASSERT(ctl->total_bitmaps <= max_bitmaps); in recalculate_thresholds()
1686 bitmap_bytes = (ctl->total_bitmaps + 1) * ctl->unit; in recalculate_thresholds()
1689 ctl->extents_thresh = 0; in recalculate_thresholds()
1700 ctl->extents_thresh = in recalculate_thresholds()
1704 static inline void __bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, in __bitmap_clear_bits() argument
1710 start = offset_to_bit(info->offset, ctl->unit, offset); in __bitmap_clear_bits()
1711 count = bytes_to_bits(bytes, ctl->unit); in __bitmap_clear_bits()
1717 if (info->max_extent_size > ctl->unit) in __bitmap_clear_bits()
1721 static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, in bitmap_clear_bits() argument
1725 __bitmap_clear_bits(ctl, info, offset, bytes); in bitmap_clear_bits()
1726 ctl->free_space -= bytes; in bitmap_clear_bits()
1729 static void bitmap_set_bits(struct btrfs_free_space_ctl *ctl, in bitmap_set_bits() argument
1735 start = offset_to_bit(info->offset, ctl->unit, offset); in bitmap_set_bits()
1736 count = bytes_to_bits(bytes, ctl->unit); in bitmap_set_bits()
1742 ctl->free_space += bytes; in bitmap_set_bits()
1749 static int search_bitmap(struct btrfs_free_space_ctl *ctl, in search_bitmap() argument
1770 i = offset_to_bit(bitmap_info->offset, ctl->unit, in search_bitmap()
1772 bits = bytes_to_bits(*bytes, ctl->unit); in search_bitmap()
1792 *offset = (u64)(i * ctl->unit) + bitmap_info->offset; in search_bitmap()
1793 *bytes = (u64)(found_bits) * ctl->unit; in search_bitmap()
1797 *bytes = (u64)(max_bits) * ctl->unit; in search_bitmap()
1811 find_free_space(struct btrfs_free_space_ctl *ctl, u64 *offset, u64 *bytes, in find_free_space() argument
1820 if (!ctl->free_space_offset.rb_node) in find_free_space()
1823 entry = tree_search_offset(ctl, offset_to_bitmap(ctl, *offset), 0, 1); in find_free_space()
1839 tmp = entry->offset - ctl->start + align - 1; in find_free_space()
1841 tmp = tmp * align + ctl->start; in find_free_space()
1857 ret = search_bitmap(ctl, entry, &tmp, &size, true); in find_free_space()
1878 static void add_new_bitmap(struct btrfs_free_space_ctl *ctl, in add_new_bitmap() argument
1881 info->offset = offset_to_bitmap(ctl, offset); in add_new_bitmap()
1884 link_free_space(ctl, info); in add_new_bitmap()
1885 ctl->total_bitmaps++; in add_new_bitmap()
1887 ctl->op->recalc_thresholds(ctl); in add_new_bitmap()
1890 static void free_bitmap(struct btrfs_free_space_ctl *ctl, in free_bitmap() argument
1893 unlink_free_space(ctl, bitmap_info); in free_bitmap()
1896 ctl->total_bitmaps--; in free_bitmap()
1897 ctl->op->recalc_thresholds(ctl); in free_bitmap()
1900 static noinline int remove_from_bitmap(struct btrfs_free_space_ctl *ctl, in remove_from_bitmap() argument
1909 end = bitmap_info->offset + (u64)(BITS_PER_BITMAP * ctl->unit) - 1; in remove_from_bitmap()
1918 search_bytes = ctl->unit; in remove_from_bitmap()
1920 ret = search_bitmap(ctl, bitmap_info, &search_start, &search_bytes, in remove_from_bitmap()
1931 bitmap_clear_bits(ctl, bitmap_info, search_start, search_bytes); in remove_from_bitmap()
1938 free_bitmap(ctl, bitmap_info); in remove_from_bitmap()
1964 search_bytes = ctl->unit; in remove_from_bitmap()
1965 ret = search_bitmap(ctl, bitmap_info, &search_start, in remove_from_bitmap()
1972 free_bitmap(ctl, bitmap_info); in remove_from_bitmap()
1977 static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, in add_bytes_to_bitmap() argument
1984 end = info->offset + (u64)(BITS_PER_BITMAP * ctl->unit); in add_bytes_to_bitmap()
1988 bitmap_set_bits(ctl, info, offset, bytes_to_set); in add_bytes_to_bitmap()
2000 static bool use_bitmap(struct btrfs_free_space_ctl *ctl, in use_bitmap() argument
2003 struct btrfs_block_group_cache *block_group = ctl->private; in use_bitmap()
2016 if (!forced && ctl->free_extents < ctl->extents_thresh) { in use_bitmap()
2025 if (ctl->free_extents * 2 <= ctl->extents_thresh) in use_bitmap()
2040 if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->key.offset) in use_bitmap()
2051 static int insert_into_bitmap(struct btrfs_free_space_ctl *ctl, in insert_into_bitmap() argument
2063 if (!ctl->op->use_bitmap(ctl, info)) in insert_into_bitmap()
2066 if (ctl->op == &free_space_op) in insert_into_bitmap()
2067 block_group = ctl->private; in insert_into_bitmap()
2095 if (entry->offset == offset_to_bitmap(ctl, offset)) { in insert_into_bitmap()
2096 bytes_added = add_bytes_to_bitmap(ctl, entry, in insert_into_bitmap()
2109 bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in insert_into_bitmap()
2116 bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes); in insert_into_bitmap()
2129 add_new_bitmap(ctl, info, offset); in insert_into_bitmap()
2134 spin_unlock(&ctl->tree_lock); in insert_into_bitmap()
2141 spin_lock(&ctl->tree_lock); in insert_into_bitmap()
2150 spin_lock(&ctl->tree_lock); in insert_into_bitmap()
2169 static bool try_merge_free_space(struct btrfs_free_space_ctl *ctl, in try_merge_free_space() argument
2183 right_info = tree_search_offset(ctl, offset + bytes, 0, 0); in try_merge_free_space()
2188 left_info = tree_search_offset(ctl, offset - 1, 0, 0); in try_merge_free_space()
2192 unlink_free_space(ctl, right_info); in try_merge_free_space()
2194 __unlink_free_space(ctl, right_info); in try_merge_free_space()
2203 unlink_free_space(ctl, left_info); in try_merge_free_space()
2205 __unlink_free_space(ctl, left_info); in try_merge_free_space()
2215 static bool steal_from_bitmap_to_end(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap_to_end() argument
2223 const u64 bitmap_offset = offset_to_bitmap(ctl, end); in steal_from_bitmap_to_end()
2226 bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); in steal_from_bitmap_to_end()
2230 i = offset_to_bit(bitmap->offset, ctl->unit, end); in steal_from_bitmap_to_end()
2234 bytes = (j - i) * ctl->unit; in steal_from_bitmap_to_end()
2238 bitmap_clear_bits(ctl, bitmap, end, bytes); in steal_from_bitmap_to_end()
2240 __bitmap_clear_bits(ctl, bitmap, end, bytes); in steal_from_bitmap_to_end()
2243 free_bitmap(ctl, bitmap); in steal_from_bitmap_to_end()
2248 static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap_to_front() argument
2259 bitmap_offset = offset_to_bitmap(ctl, info->offset); in steal_from_bitmap_to_front()
2264 bitmap_offset = offset_to_bitmap(ctl, info->offset - 1); in steal_from_bitmap_to_front()
2267 bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); in steal_from_bitmap_to_front()
2271 i = offset_to_bit(bitmap->offset, ctl->unit, info->offset) - 1; in steal_from_bitmap_to_front()
2283 bytes = (i + 1) * ctl->unit; in steal_from_bitmap_to_front()
2285 bytes = (i - prev_j) * ctl->unit; in steal_from_bitmap_to_front()
2291 bitmap_clear_bits(ctl, bitmap, info->offset, bytes); in steal_from_bitmap_to_front()
2293 __bitmap_clear_bits(ctl, bitmap, info->offset, bytes); in steal_from_bitmap_to_front()
2296 free_bitmap(ctl, bitmap); in steal_from_bitmap_to_front()
2312 static void steal_from_bitmap(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap() argument
2323 if (ctl->total_bitmaps > 0) { in steal_from_bitmap()
2327 stole_end = steal_from_bitmap_to_end(ctl, info, update_stat); in steal_from_bitmap()
2328 if (ctl->total_bitmaps > 0) in steal_from_bitmap()
2329 stole_front = steal_from_bitmap_to_front(ctl, info, in steal_from_bitmap()
2333 try_merge_free_space(ctl, info, update_stat); in steal_from_bitmap()
2338 struct btrfs_free_space_ctl *ctl, in __btrfs_add_free_space() argument
2352 spin_lock(&ctl->tree_lock); in __btrfs_add_free_space()
2354 if (try_merge_free_space(ctl, info, true)) in __btrfs_add_free_space()
2362 ret = insert_into_bitmap(ctl, info); in __btrfs_add_free_space()
2376 steal_from_bitmap(ctl, info, true); in __btrfs_add_free_space()
2378 ret = link_free_space(ctl, info); in __btrfs_add_free_space()
2382 spin_unlock(&ctl->tree_lock); in __btrfs_add_free_space()
2395 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_remove_free_space() local
2400 spin_lock(&ctl->tree_lock); in btrfs_remove_free_space()
2407 info = tree_search_offset(ctl, offset, 0, 0); in btrfs_remove_free_space()
2413 info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in btrfs_remove_free_space()
2428 unlink_free_space(ctl, info); in btrfs_remove_free_space()
2435 ret = link_free_space(ctl, info); in btrfs_remove_free_space()
2448 ret = link_free_space(ctl, info); in btrfs_remove_free_space()
2462 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space()
2471 ret = remove_from_bitmap(ctl, info, &offset, &bytes); in btrfs_remove_free_space()
2477 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space()
2486 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_dump_free_space() local
2491 spin_lock(&ctl->tree_lock); in btrfs_dump_free_space()
2492 for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) { in btrfs_dump_free_space()
2500 spin_unlock(&ctl->tree_lock); in btrfs_dump_free_space()
2510 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_init_free_space_ctl() local
2512 spin_lock_init(&ctl->tree_lock); in btrfs_init_free_space_ctl()
2513 ctl->unit = fs_info->sectorsize; in btrfs_init_free_space_ctl()
2514 ctl->start = block_group->key.objectid; in btrfs_init_free_space_ctl()
2515 ctl->private = block_group; in btrfs_init_free_space_ctl()
2516 ctl->op = &free_space_op; in btrfs_init_free_space_ctl()
2517 INIT_LIST_HEAD(&ctl->trimming_ranges); in btrfs_init_free_space_ctl()
2518 mutex_init(&ctl->cache_writeout_mutex); in btrfs_init_free_space_ctl()
2525 ctl->extents_thresh = (SZ_32K / 2) / sizeof(struct btrfs_free_space); in btrfs_init_free_space_ctl()
2539 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in __btrfs_return_cluster_to_free_space() local
2562 try_merge_free_space(ctl, entry, false); in __btrfs_return_cluster_to_free_space()
2563 steal_from_bitmap(ctl, entry, false); in __btrfs_return_cluster_to_free_space()
2565 tree_insert_offset(&ctl->free_space_offset, in __btrfs_return_cluster_to_free_space()
2577 struct btrfs_free_space_ctl *ctl) in __btrfs_remove_free_space_cache_locked() argument
2582 while ((node = rb_last(&ctl->free_space_offset)) != NULL) { in __btrfs_remove_free_space_cache_locked()
2585 unlink_free_space(ctl, info); in __btrfs_remove_free_space_cache_locked()
2588 free_bitmap(ctl, info); in __btrfs_remove_free_space_cache_locked()
2591 cond_resched_lock(&ctl->tree_lock); in __btrfs_remove_free_space_cache_locked()
2595 void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl) in __btrfs_remove_free_space_cache() argument
2597 spin_lock(&ctl->tree_lock); in __btrfs_remove_free_space_cache()
2598 __btrfs_remove_free_space_cache_locked(ctl); in __btrfs_remove_free_space_cache()
2599 spin_unlock(&ctl->tree_lock); in __btrfs_remove_free_space_cache()
2604 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_remove_free_space_cache() local
2608 spin_lock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
2617 cond_resched_lock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
2619 __btrfs_remove_free_space_cache_locked(ctl); in btrfs_remove_free_space_cache()
2620 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
2628 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_find_space_for_alloc() local
2635 spin_lock(&ctl->tree_lock); in btrfs_find_space_for_alloc()
2636 entry = find_free_space(ctl, &offset, &bytes_search, in btrfs_find_space_for_alloc()
2643 bitmap_clear_bits(ctl, entry, offset, bytes); in btrfs_find_space_for_alloc()
2645 free_bitmap(ctl, entry); in btrfs_find_space_for_alloc()
2647 unlink_free_space(ctl, entry); in btrfs_find_space_for_alloc()
2658 link_free_space(ctl, entry); in btrfs_find_space_for_alloc()
2661 spin_unlock(&ctl->tree_lock); in btrfs_find_space_for_alloc()
2664 __btrfs_add_free_space(block_group->fs_info, ctl, in btrfs_find_space_for_alloc()
2681 struct btrfs_free_space_ctl *ctl; in btrfs_return_cluster_to_free_space() local
2700 ctl = block_group->free_space_ctl; in btrfs_return_cluster_to_free_space()
2703 spin_lock(&ctl->tree_lock); in btrfs_return_cluster_to_free_space()
2705 spin_unlock(&ctl->tree_lock); in btrfs_return_cluster_to_free_space()
2718 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_alloc_from_bitmap() local
2727 err = search_bitmap(ctl, entry, &search_start, &search_bytes, true); in btrfs_alloc_from_bitmap()
2735 __bitmap_clear_bits(ctl, entry, ret, bytes); in btrfs_alloc_from_bitmap()
2749 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_alloc_from_cluster() local
2812 spin_lock(&ctl->tree_lock); in btrfs_alloc_from_cluster()
2814 ctl->free_space -= bytes; in btrfs_alloc_from_cluster()
2816 ctl->free_extents--; in btrfs_alloc_from_cluster()
2820 ctl->total_bitmaps--; in btrfs_alloc_from_cluster()
2821 ctl->op->recalc_thresholds(ctl); in btrfs_alloc_from_cluster()
2826 spin_unlock(&ctl->tree_lock); in btrfs_alloc_from_cluster()
2837 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_bitmap_cluster() local
2848 i = offset_to_bit(entry->offset, ctl->unit, in btrfs_bitmap_cluster()
2850 want_bits = bytes_to_bits(bytes, ctl->unit); in btrfs_bitmap_cluster()
2851 min_bits = bytes_to_bits(min_bytes, ctl->unit); in btrfs_bitmap_cluster()
2877 entry->max_extent_size = (u64)max_bits * ctl->unit; in btrfs_bitmap_cluster()
2888 if (cluster->max_size < found_bits * ctl->unit) in btrfs_bitmap_cluster()
2889 cluster->max_size = found_bits * ctl->unit; in btrfs_bitmap_cluster()
2896 cluster->window_start = start * ctl->unit + entry->offset; in btrfs_bitmap_cluster()
2897 rb_erase(&entry->offset_index, &ctl->free_space_offset); in btrfs_bitmap_cluster()
2903 total_found * ctl->unit, 1); in btrfs_bitmap_cluster()
2918 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in setup_cluster_no_bitmap() local
2927 entry = tree_search_offset(ctl, offset, 0, 1); in setup_cluster_no_bitmap()
2987 rb_erase(&entry->offset_index, &ctl->free_space_offset); in setup_cluster_no_bitmap()
3009 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in setup_cluster_bitmap() local
3012 u64 bitmap_offset = offset_to_bitmap(ctl, offset); in setup_cluster_bitmap()
3014 if (ctl->total_bitmaps == 0) in setup_cluster_bitmap()
3025 entry = tree_search_offset(ctl, bitmap_offset, 1, 0); in setup_cluster_bitmap()
3059 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_find_space_cluster() local
3082 spin_lock(&ctl->tree_lock); in btrfs_find_space_cluster()
3088 if (ctl->free_space < bytes) { in btrfs_find_space_cluster()
3089 spin_unlock(&ctl->tree_lock); in btrfs_find_space_cluster()
3126 spin_unlock(&ctl->tree_lock); in btrfs_find_space_cluster()
3152 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in do_trimming() local
3171 mutex_lock(&ctl->cache_writeout_mutex); in do_trimming()
3174 mutex_unlock(&ctl->cache_writeout_mutex); in do_trimming()
3193 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in trim_no_bitmap() local
3204 mutex_lock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3205 spin_lock(&ctl->tree_lock); in trim_no_bitmap()
3207 if (ctl->free_space < minlen) { in trim_no_bitmap()
3208 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3209 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3213 entry = tree_search_offset(ctl, start, 0, 1); in trim_no_bitmap()
3215 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3216 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3224 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3225 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3233 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3234 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3243 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3244 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3248 unlink_free_space(ctl, entry); in trim_no_bitmap()
3251 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3254 list_add_tail(&trim_entry.list, &ctl->trimming_ranges); in trim_no_bitmap()
3255 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3278 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in trim_bitmaps() local
3283 u64 offset = offset_to_bitmap(ctl, start); in trim_bitmaps()
3289 mutex_lock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3290 spin_lock(&ctl->tree_lock); in trim_bitmaps()
3292 if (ctl->free_space < minlen) { in trim_bitmaps()
3293 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3294 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3298 entry = tree_search_offset(ctl, offset, 1, 0); in trim_bitmaps()
3300 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3301 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3307 ret2 = search_bitmap(ctl, entry, &start, &bytes, false); in trim_bitmaps()
3309 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3310 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3317 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3318 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3322 bitmap_clear_bits(ctl, entry, start, bytes); in trim_bitmaps()
3324 free_bitmap(ctl, entry); in trim_bitmaps()
3326 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3329 list_add_tail(&trim_entry.list, &ctl->trimming_ranges); in trim_bitmaps()
3330 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3338 offset += BITS_PER_BITMAP * ctl->unit; in trim_bitmaps()
3341 if (start >= offset + BITS_PER_BITMAP * ctl->unit) in trim_bitmaps()
3342 offset += BITS_PER_BITMAP * ctl->unit; in trim_bitmaps()
3434 struct btrfs_free_space_ctl *ctl = fs_root->free_ino_ctl; in btrfs_find_ino_for_alloc() local
3438 spin_lock(&ctl->tree_lock); in btrfs_find_ino_for_alloc()
3440 if (RB_EMPTY_ROOT(&ctl->free_space_offset)) in btrfs_find_ino_for_alloc()
3443 entry = rb_entry(rb_first(&ctl->free_space_offset), in btrfs_find_ino_for_alloc()
3449 unlink_free_space(ctl, entry); in btrfs_find_ino_for_alloc()
3455 link_free_space(ctl, entry); in btrfs_find_ino_for_alloc()
3461 ret = search_bitmap(ctl, entry, &offset, &count, true); in btrfs_find_ino_for_alloc()
3466 bitmap_clear_bits(ctl, entry, offset, 1); in btrfs_find_ino_for_alloc()
3468 free_bitmap(ctl, entry); in btrfs_find_ino_for_alloc()
3471 spin_unlock(&ctl->tree_lock); in btrfs_find_ino_for_alloc()
3510 struct btrfs_free_space_ctl *ctl = root->free_ino_ctl; in load_free_ino_cache() local
3537 ret = __load_free_space_cache(root, inode, ctl, path, 0); in load_free_ino_cache()
3556 struct btrfs_free_space_ctl *ctl = root->free_ino_ctl; in btrfs_write_out_ino_cache() local
3565 ret = __btrfs_write_out_cache(root, inode, ctl, NULL, &io_ctl, trans); in btrfs_write_out_ino_cache()
3601 struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; in test_add_free_space_entry() local
3615 spin_lock(&ctl->tree_lock); in test_add_free_space_entry()
3619 ret = link_free_space(ctl, info); in test_add_free_space_entry()
3620 spin_unlock(&ctl->tree_lock); in test_add_free_space_entry()
3634 spin_lock(&ctl->tree_lock); in test_add_free_space_entry()
3635 bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in test_add_free_space_entry()
3640 add_new_bitmap(ctl, info, offset); in test_add_free_space_entry()
3645 bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes); in test_add_free_space_entry()
3649 spin_unlock(&ctl->tree_lock); in test_add_free_space_entry()
3669 struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; in test_check_exists() local
3673 spin_lock(&ctl->tree_lock); in test_check_exists()
3674 info = tree_search_offset(ctl, offset, 0, 0); in test_check_exists()
3676 info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in test_check_exists()
3689 bit_bytes = ctl->unit; in test_check_exists()
3690 ret = search_bitmap(ctl, info, &bit_off, &bit_bytes, false); in test_check_exists()
3742 spin_unlock(&ctl->tree_lock); in test_check_exists()