Lines Matching refs:sbi
172 bool need_SSR(struct f2fs_sb_info *sbi) in need_SSR() argument
174 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); in need_SSR()
175 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); in need_SSR()
176 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); in need_SSR()
178 if (test_opt(sbi, LFS)) in need_SSR()
180 if (sbi->gc_thread && sbi->gc_thread->gc_urgent) in need_SSR()
183 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + in need_SSR()
184 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi)); in need_SSR()
189 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in register_inmem_page() local
208 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); in register_inmem_page()
210 list_add_tail(&fi->inmem_ilist, &sbi->inode_list[ATOMIC_FILE]); in register_inmem_page()
211 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); in register_inmem_page()
221 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in __revoke_inmem_pages() local
250 get_node_info(sbi, dn.nid, &ni); in __revoke_inmem_pages()
252 invalidate_blocks(sbi, dn.data_blkaddr); in __revoke_inmem_pages()
255 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, in __revoke_inmem_pages()
274 void drop_inmem_pages_all(struct f2fs_sb_info *sbi) in drop_inmem_pages_all() argument
276 struct list_head *head = &sbi->inode_list[ATOMIC_FILE]; in drop_inmem_pages_all()
280 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); in drop_inmem_pages_all()
282 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); in drop_inmem_pages_all()
287 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); in drop_inmem_pages_all()
300 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in drop_inmem_pages() local
305 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); in drop_inmem_pages()
308 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); in drop_inmem_pages()
319 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in drop_inmem_page() local
323 f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page)); in drop_inmem_page()
331 f2fs_bug_on(sbi, !cur || cur->page != page); in drop_inmem_page()
335 dec_page_count(sbi, F2FS_INMEM_PAGES); in drop_inmem_page()
349 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in __commit_inmem_pages() local
353 .sbi = sbi, in __commit_inmem_pages()
400 f2fs_submit_merged_write_cond(sbi, inode, 0, last_idx, DATA); in __commit_inmem_pages()
410 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in commit_inmem_pages() local
416 f2fs_balance_fs(sbi, true); in commit_inmem_pages()
417 f2fs_lock_op(sbi); in commit_inmem_pages()
440 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); in commit_inmem_pages()
443 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); in commit_inmem_pages()
448 f2fs_unlock_op(sbi); in commit_inmem_pages()
456 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) in f2fs_balance_fs() argument
459 if (time_to_inject(sbi, FAULT_CHECKPOINT)) { in f2fs_balance_fs()
461 f2fs_stop_checkpoint(sbi, false); in f2fs_balance_fs()
466 if (need && excess_cached_nats(sbi)) in f2fs_balance_fs()
467 f2fs_balance_fs_bg(sbi); in f2fs_balance_fs()
473 if (has_not_enough_free_secs(sbi, 0, 0)) { in f2fs_balance_fs()
474 mutex_lock(&sbi->gc_mutex); in f2fs_balance_fs()
475 f2fs_gc(sbi, false, false, NULL_SEGNO); in f2fs_balance_fs()
479 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi) in f2fs_balance_fs_bg() argument
482 if (!available_free_memory(sbi, EXTENT_CACHE)) in f2fs_balance_fs_bg()
483 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER); in f2fs_balance_fs_bg()
486 if (!available_free_memory(sbi, NAT_ENTRIES)) in f2fs_balance_fs_bg()
487 try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK); in f2fs_balance_fs_bg()
489 if (!available_free_memory(sbi, FREE_NIDS)) in f2fs_balance_fs_bg()
490 try_to_free_nids(sbi, MAX_FREE_NIDS); in f2fs_balance_fs_bg()
492 build_free_nids(sbi, false, false); in f2fs_balance_fs_bg()
494 if (!is_idle(sbi) && !excess_dirty_nats(sbi)) in f2fs_balance_fs_bg()
498 if (!available_free_memory(sbi, NAT_ENTRIES) || in f2fs_balance_fs_bg()
499 !available_free_memory(sbi, INO_ENTRIES) || in f2fs_balance_fs_bg()
500 excess_prefree_segs(sbi) || in f2fs_balance_fs_bg()
501 excess_dirty_nats(sbi) || in f2fs_balance_fs_bg()
502 f2fs_time_over(sbi, CP_TIME)) { in f2fs_balance_fs_bg()
503 if (test_opt(sbi, DATA_FLUSH)) { in f2fs_balance_fs_bg()
507 sync_dirty_inodes(sbi, FILE_INODE); in f2fs_balance_fs_bg()
510 f2fs_sync_fs(sbi->sb, true); in f2fs_balance_fs_bg()
511 stat_inc_bg_cp_count(sbi->stat_info); in f2fs_balance_fs_bg()
515 static int __submit_flush_wait(struct f2fs_sb_info *sbi, in __submit_flush_wait() argument
518 struct bio *bio = f2fs_bio_alloc(sbi, 0, true); in __submit_flush_wait()
526 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER), in __submit_flush_wait()
527 test_opt(sbi, FLUSH_MERGE), ret); in __submit_flush_wait()
531 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino) in submit_flush_wait() argument
536 if (!sbi->s_ndevs) in submit_flush_wait()
537 return __submit_flush_wait(sbi, sbi->sb->s_bdev); in submit_flush_wait()
539 for (i = 0; i < sbi->s_ndevs; i++) { in submit_flush_wait()
540 if (!is_dirty_device(sbi, ino, i, FLUSH_INO)) in submit_flush_wait()
542 ret = __submit_flush_wait(sbi, FDEV(i).bdev); in submit_flush_wait()
551 struct f2fs_sb_info *sbi = data; in issue_flush_thread() local
552 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; in issue_flush_thread()
558 sb_start_intwrite(sbi->sb); in issue_flush_thread()
569 ret = submit_flush_wait(sbi, cmd->ino); in issue_flush_thread()
580 sb_end_intwrite(sbi->sb); in issue_flush_thread()
587 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino) in f2fs_issue_flush() argument
589 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; in f2fs_issue_flush()
593 if (test_opt(sbi, NOBARRIER)) in f2fs_issue_flush()
596 if (!test_opt(sbi, FLUSH_MERGE)) { in f2fs_issue_flush()
597 ret = submit_flush_wait(sbi, ino); in f2fs_issue_flush()
602 if (atomic_inc_return(&fcc->issing_flush) == 1 || sbi->s_ndevs > 1) { in f2fs_issue_flush()
603 ret = submit_flush_wait(sbi, ino); in f2fs_issue_flush()
634 ret = submit_flush_wait(sbi, ino); in f2fs_issue_flush()
651 int create_flush_cmd_control(struct f2fs_sb_info *sbi) in create_flush_cmd_control() argument
653 dev_t dev = sbi->sb->s_bdev->bd_dev; in create_flush_cmd_control()
657 if (SM_I(sbi)->fcc_info) { in create_flush_cmd_control()
658 fcc = SM_I(sbi)->fcc_info; in create_flush_cmd_control()
664 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL); in create_flush_cmd_control()
671 SM_I(sbi)->fcc_info = fcc; in create_flush_cmd_control()
672 if (!test_opt(sbi, FLUSH_MERGE)) in create_flush_cmd_control()
676 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, in create_flush_cmd_control()
681 SM_I(sbi)->fcc_info = NULL; in create_flush_cmd_control()
688 void destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free) in destroy_flush_cmd_control() argument
690 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; in destroy_flush_cmd_control()
700 SM_I(sbi)->fcc_info = NULL; in destroy_flush_cmd_control()
704 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi) in f2fs_flush_device_cache() argument
708 if (!sbi->s_ndevs) in f2fs_flush_device_cache()
711 for (i = 1; i < sbi->s_ndevs; i++) { in f2fs_flush_device_cache()
712 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device)) in f2fs_flush_device_cache()
714 ret = __submit_flush_wait(sbi, FDEV(i).bdev); in f2fs_flush_device_cache()
718 spin_lock(&sbi->dev_lock); in f2fs_flush_device_cache()
719 f2fs_clear_bit(i, (char *)&sbi->dirty_device); in f2fs_flush_device_cache()
720 spin_unlock(&sbi->dev_lock); in f2fs_flush_device_cache()
726 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, in __locate_dirty_segment() argument
729 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in __locate_dirty_segment()
732 if (IS_CURSEG(sbi, segno)) in __locate_dirty_segment()
739 struct seg_entry *sentry = get_seg_entry(sbi, segno); in __locate_dirty_segment()
743 f2fs_bug_on(sbi, 1); in __locate_dirty_segment()
751 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, in __remove_dirty_segment() argument
754 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in __remove_dirty_segment()
760 struct seg_entry *sentry = get_seg_entry(sbi, segno); in __remove_dirty_segment()
766 if (get_valid_blocks(sbi, segno, true) == 0) in __remove_dirty_segment()
767 clear_bit(GET_SEC_FROM_SEG(sbi, segno), in __remove_dirty_segment()
777 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) in locate_dirty_segment() argument
779 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in locate_dirty_segment()
782 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno)) in locate_dirty_segment()
787 valid_blocks = get_valid_blocks(sbi, segno, false); in locate_dirty_segment()
790 __locate_dirty_segment(sbi, segno, PRE); in locate_dirty_segment()
791 __remove_dirty_segment(sbi, segno, DIRTY); in locate_dirty_segment()
792 } else if (valid_blocks < sbi->blocks_per_seg) { in locate_dirty_segment()
793 __locate_dirty_segment(sbi, segno, DIRTY); in locate_dirty_segment()
796 __remove_dirty_segment(sbi, segno, DIRTY); in locate_dirty_segment()
802 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi, in __create_discard_cmd() argument
806 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __create_discard_cmd()
810 f2fs_bug_on(sbi, !len); in __create_discard_cmd()
831 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi, in __attach_discard_cmd() argument
836 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __attach_discard_cmd()
839 dc = __create_discard_cmd(sbi, bdev, lstart, start, len); in __attach_discard_cmd()
862 static void __remove_discard_cmd(struct f2fs_sb_info *sbi, in __remove_discard_cmd() argument
865 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __remove_discard_cmd()
869 f2fs_bug_on(sbi, dc->ref); in __remove_discard_cmd()
875 f2fs_msg(sbi->sb, KERN_INFO, in __remove_discard_cmd()
891 static void __check_sit_bitmap(struct f2fs_sb_info *sbi, in __check_sit_bitmap() argument
898 unsigned long offset, size, max_blocks = sbi->blocks_per_seg; in __check_sit_bitmap()
902 segno = GET_SEGNO(sbi, blk); in __check_sit_bitmap()
903 sentry = get_seg_entry(sbi, segno); in __check_sit_bitmap()
904 offset = GET_BLKOFF_FROM_SEG0(sbi, blk); in __check_sit_bitmap()
906 if (end < START_BLOCK(sbi, segno + 1)) in __check_sit_bitmap()
907 size = GET_BLKOFF_FROM_SEG0(sbi, end); in __check_sit_bitmap()
912 f2fs_bug_on(sbi, offset != size); in __check_sit_bitmap()
913 blk = START_BLOCK(sbi, segno + 1); in __check_sit_bitmap()
919 static void __submit_discard_cmd(struct f2fs_sb_info *sbi, in __submit_discard_cmd() argument
923 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __submit_discard_cmd()
949 __check_sit_bitmap(sbi, dc->start, dc->start + dc->len); in __submit_discard_cmd()
951 f2fs_update_iostat(sbi, FS_DISCARD, 1); in __submit_discard_cmd()
954 __remove_discard_cmd(sbi, dc); in __submit_discard_cmd()
958 static struct discard_cmd *__insert_discard_tree(struct f2fs_sb_info *sbi, in __insert_discard_tree() argument
964 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __insert_discard_tree()
975 p = __lookup_rb_tree_for_insert(sbi, &dcc->root, &parent, lstart); in __insert_discard_tree()
977 dc = __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, p); in __insert_discard_tree()
990 static void __punch_discard_cmd(struct f2fs_sb_info *sbi, in __punch_discard_cmd() argument
993 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __punch_discard_cmd()
998 __remove_discard_cmd(sbi, dc); in __punch_discard_cmd()
1013 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1, in __punch_discard_cmd()
1027 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, in __update_discard_tree_range() argument
1031 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __update_discard_tree_range()
1097 __remove_discard_cmd(sbi, tdc); in __update_discard_tree_range()
1102 __insert_discard_tree(sbi, bdev, di.lstart, di.start, in __update_discard_tree_range()
1117 static int __queue_discard_cmd(struct f2fs_sb_info *sbi, in __queue_discard_cmd() argument
1124 if (sbi->s_ndevs) { in __queue_discard_cmd()
1125 int devi = f2fs_target_device_index(sbi, blkstart); in __queue_discard_cmd()
1129 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen); in __queue_discard_cmd()
1133 static void __issue_discard_cmd_range(struct f2fs_sb_info *sbi, in __issue_discard_cmd_range() argument
1137 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __issue_discard_cmd_range()
1148 f2fs_bug_on(sbi, !__check_rb_tree_consistence(sbi, &dcc->root)); in __issue_discard_cmd_range()
1171 __submit_discard_cmd(sbi, dpolicy, dc); in __issue_discard_cmd_range()
1195 static int __issue_discard_cmd(struct f2fs_sb_info *sbi, in __issue_discard_cmd() argument
1198 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __issue_discard_cmd()
1213 f2fs_bug_on(sbi, !__check_rb_tree_consistence(sbi, &dcc->root)); in __issue_discard_cmd()
1216 f2fs_bug_on(sbi, dc->state != D_PREP); in __issue_discard_cmd()
1219 !is_idle(sbi)) { in __issue_discard_cmd()
1224 __submit_discard_cmd(sbi, dpolicy, dc); in __issue_discard_cmd()
1244 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi) in __drop_discard_cmd() argument
1246 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __drop_discard_cmd()
1256 f2fs_bug_on(sbi, dc->state != D_PREP); in __drop_discard_cmd()
1257 __remove_discard_cmd(sbi, dc); in __drop_discard_cmd()
1266 void drop_discard_cmd(struct f2fs_sb_info *sbi) in drop_discard_cmd() argument
1268 __drop_discard_cmd(sbi); in drop_discard_cmd()
1271 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi, in __wait_one_discard_bio() argument
1274 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __wait_one_discard_bio()
1279 f2fs_bug_on(sbi, dc->state != D_DONE); in __wait_one_discard_bio()
1284 __remove_discard_cmd(sbi, dc); in __wait_one_discard_bio()
1291 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi, in __wait_discard_cmd_range() argument
1295 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __wait_discard_cmd_range()
1315 __remove_discard_cmd(sbi, dc); in __wait_discard_cmd_range()
1325 trimmed += __wait_one_discard_bio(sbi, dc); in __wait_discard_cmd_range()
1332 static void __wait_all_discard_cmd(struct f2fs_sb_info *sbi, in __wait_all_discard_cmd() argument
1335 __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); in __wait_all_discard_cmd()
1339 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) in f2fs_wait_discard_bio() argument
1341 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in f2fs_wait_discard_bio()
1349 __punch_discard_cmd(sbi, dc, blkaddr); in f2fs_wait_discard_bio()
1358 __wait_one_discard_bio(sbi, dc); in f2fs_wait_discard_bio()
1361 void stop_discard_thread(struct f2fs_sb_info *sbi) in stop_discard_thread() argument
1363 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in stop_discard_thread()
1374 bool f2fs_wait_discard_bios(struct f2fs_sb_info *sbi) in f2fs_wait_discard_bios() argument
1376 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in f2fs_wait_discard_bios()
1381 __issue_discard_cmd(sbi, &dpolicy); in f2fs_wait_discard_bios()
1382 dropped = __drop_discard_cmd(sbi); in f2fs_wait_discard_bios()
1383 __wait_all_discard_cmd(sbi, &dpolicy); in f2fs_wait_discard_bios()
1390 struct f2fs_sb_info *sbi = data; in issue_discard_thread() local
1391 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in issue_discard_thread()
1409 if (f2fs_readonly(sbi->sb)) in issue_discard_thread()
1417 if (sbi->gc_thread && sbi->gc_thread->gc_urgent) in issue_discard_thread()
1420 sb_start_intwrite(sbi->sb); in issue_discard_thread()
1422 issued = __issue_discard_cmd(sbi, &dpolicy); in issue_discard_thread()
1424 __wait_all_discard_cmd(sbi, &dpolicy); in issue_discard_thread()
1430 sb_end_intwrite(sbi->sb); in issue_discard_thread()
1437 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, in __f2fs_issue_discard_zone() argument
1444 if (sbi->s_ndevs) { in __f2fs_issue_discard_zone()
1445 devi = f2fs_target_device_index(sbi, blkstart); in __f2fs_issue_discard_zone()
1454 switch (get_blkz_type(sbi, bdev, blkstart)) { in __f2fs_issue_discard_zone()
1459 return __queue_discard_cmd(sbi, bdev, lblkstart, blklen); in __f2fs_issue_discard_zone()
1467 f2fs_msg(sbi->sb, KERN_INFO, in __f2fs_issue_discard_zone()
1469 devi, sbi->s_ndevs ? FDEV(devi).path: "", in __f2fs_issue_discard_zone()
1483 static int __issue_discard_async(struct f2fs_sb_info *sbi, in __issue_discard_async() argument
1487 if (f2fs_sb_has_blkzoned(sbi->sb) && in __issue_discard_async()
1489 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); in __issue_discard_async()
1491 return __queue_discard_cmd(sbi, bdev, blkstart, blklen); in __issue_discard_async()
1494 static int f2fs_issue_discard(struct f2fs_sb_info *sbi, in f2fs_issue_discard() argument
1504 bdev = f2fs_target_device(sbi, blkstart, NULL); in f2fs_issue_discard()
1509 f2fs_target_device(sbi, i, NULL); in f2fs_issue_discard()
1512 err = __issue_discard_async(sbi, bdev, in f2fs_issue_discard()
1522 se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); in f2fs_issue_discard()
1523 offset = GET_BLKOFF_FROM_SEG0(sbi, i); in f2fs_issue_discard()
1526 sbi->discard_blks--; in f2fs_issue_discard()
1530 err = __issue_discard_async(sbi, bdev, start, len); in f2fs_issue_discard()
1534 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, in add_discard_addrs() argument
1538 int max_blocks = sbi->blocks_per_seg; in add_discard_addrs()
1539 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); in add_discard_addrs()
1543 unsigned long *dmap = SIT_I(sbi)->tmp_map; in add_discard_addrs()
1547 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list; in add_discard_addrs()
1550 if (se->valid_blocks == max_blocks || !f2fs_discard_en(sbi)) in add_discard_addrs()
1554 if (!test_opt(sbi, DISCARD) || !se->valid_blocks || in add_discard_addrs()
1555 SM_I(sbi)->dcc_info->nr_discards >= in add_discard_addrs()
1556 SM_I(sbi)->dcc_info->max_discards) in add_discard_addrs()
1565 while (force || SM_I(sbi)->dcc_info->nr_discards <= in add_discard_addrs()
1566 SM_I(sbi)->dcc_info->max_discards) { in add_discard_addrs()
1582 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start); in add_discard_addrs()
1589 SM_I(sbi)->dcc_info->nr_discards += end - start; in add_discard_addrs()
1594 void release_discard_addrs(struct f2fs_sb_info *sbi) in release_discard_addrs() argument
1596 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); in release_discard_addrs()
1609 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) in set_prefree_as_free_segments() argument
1611 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in set_prefree_as_free_segments()
1615 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi)) in set_prefree_as_free_segments()
1616 __set_test_and_free(sbi, segno); in set_prefree_as_free_segments()
1620 void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc) in clear_prefree_segments() argument
1622 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in clear_prefree_segments()
1625 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in clear_prefree_segments()
1635 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1); in clear_prefree_segments()
1636 if (start >= MAIN_SEGS(sbi)) in clear_prefree_segments()
1638 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi), in clear_prefree_segments()
1646 if (!test_opt(sbi, DISCARD)) in clear_prefree_segments()
1653 if (!test_opt(sbi, LFS) || sbi->segs_per_sec == 1) { in clear_prefree_segments()
1654 f2fs_issue_discard(sbi, START_BLOCK(sbi, start), in clear_prefree_segments()
1655 (end - start) << sbi->log_blocks_per_seg); in clear_prefree_segments()
1659 secno = GET_SEC_FROM_SEG(sbi, start); in clear_prefree_segments()
1660 start_segno = GET_SEG_FROM_SEC(sbi, secno); in clear_prefree_segments()
1661 if (!IS_CURSEC(sbi, secno) && in clear_prefree_segments()
1662 !get_valid_blocks(sbi, start, true)) in clear_prefree_segments()
1663 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno), in clear_prefree_segments()
1664 sbi->segs_per_sec << sbi->log_blocks_per_seg); in clear_prefree_segments()
1666 start = start_segno + sbi->segs_per_sec; in clear_prefree_segments()
1682 sbi->blocks_per_seg, cur_pos); in clear_prefree_segments()
1685 if (f2fs_sb_has_blkzoned(sbi->sb) || in clear_prefree_segments()
1689 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos, in clear_prefree_segments()
1694 sbi->blocks_per_seg, cur_pos); in clear_prefree_segments()
1700 if (cur_pos < sbi->blocks_per_seg) in clear_prefree_segments()
1708 wake_up_discard_thread(sbi, false); in clear_prefree_segments()
1737 static int create_discard_cmd_control(struct f2fs_sb_info *sbi) in create_discard_cmd_control() argument
1739 dev_t dev = sbi->sb->s_bdev->bd_dev; in create_discard_cmd_control()
1743 if (SM_I(sbi)->dcc_info) { in create_discard_cmd_control()
1744 dcc = SM_I(sbi)->dcc_info; in create_discard_cmd_control()
1748 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL); in create_discard_cmd_control()
1763 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg; in create_discard_cmd_control()
1768 SM_I(sbi)->dcc_info = dcc; in create_discard_cmd_control()
1770 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi, in create_discard_cmd_control()
1775 SM_I(sbi)->dcc_info = NULL; in create_discard_cmd_control()
1782 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) in destroy_discard_cmd_control() argument
1784 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in destroy_discard_cmd_control()
1789 stop_discard_thread(sbi); in destroy_discard_cmd_control()
1792 SM_I(sbi)->dcc_info = NULL; in destroy_discard_cmd_control()
1795 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) in __mark_sit_entry_dirty() argument
1797 struct sit_info *sit_i = SIT_I(sbi); in __mark_sit_entry_dirty()
1807 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, in __set_sit_entry_type() argument
1810 struct seg_entry *se = get_seg_entry(sbi, segno); in __set_sit_entry_type()
1813 __mark_sit_entry_dirty(sbi, segno); in __set_sit_entry_type()
1816 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) in update_sit_entry() argument
1826 segno = GET_SEGNO(sbi, blkaddr); in update_sit_entry()
1828 se = get_seg_entry(sbi, segno); in update_sit_entry()
1830 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); in update_sit_entry()
1832 f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) || in update_sit_entry()
1833 (new_vblocks > sbi->blocks_per_seg))); in update_sit_entry()
1836 se->mtime = get_mtime(sbi); in update_sit_entry()
1837 SIT_I(sbi)->max_mtime = se->mtime; in update_sit_entry()
1846 f2fs_msg(sbi->sb, KERN_ERR, "Inconsistent error " in update_sit_entry()
1849 f2fs_bug_on(sbi, 1); in update_sit_entry()
1853 f2fs_msg(sbi->sb, KERN_ERR, in update_sit_entry()
1855 f2fs_bug_on(sbi, 1); in update_sit_entry()
1860 if (f2fs_discard_en(sbi) && in update_sit_entry()
1862 sbi->discard_blks--; in update_sit_entry()
1875 f2fs_msg(sbi->sb, KERN_ERR, "Inconsistent error " in update_sit_entry()
1878 f2fs_bug_on(sbi, 1); in update_sit_entry()
1882 f2fs_msg(sbi->sb, KERN_ERR, in update_sit_entry()
1884 f2fs_bug_on(sbi, 1); in update_sit_entry()
1889 if (f2fs_discard_en(sbi) && in update_sit_entry()
1891 sbi->discard_blks++; in update_sit_entry()
1896 __mark_sit_entry_dirty(sbi, segno); in update_sit_entry()
1899 SIT_I(sbi)->written_valid_blocks += del; in update_sit_entry()
1901 if (sbi->segs_per_sec > 1) in update_sit_entry()
1902 get_sec_entry(sbi, segno)->valid_blocks += del; in update_sit_entry()
1905 void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr) in invalidate_blocks() argument
1907 unsigned int segno = GET_SEGNO(sbi, addr); in invalidate_blocks()
1908 struct sit_info *sit_i = SIT_I(sbi); in invalidate_blocks()
1910 f2fs_bug_on(sbi, addr == NULL_ADDR); in invalidate_blocks()
1917 update_sit_entry(sbi, addr, -1); in invalidate_blocks()
1920 locate_dirty_segment(sbi, segno); in invalidate_blocks()
1925 bool is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) in is_checkpointed_data() argument
1927 struct sit_info *sit_i = SIT_I(sbi); in is_checkpointed_data()
1937 segno = GET_SEGNO(sbi, blkaddr); in is_checkpointed_data()
1938 se = get_seg_entry(sbi, segno); in is_checkpointed_data()
1939 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); in is_checkpointed_data()
1952 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type, in __add_sum_entry() argument
1955 struct curseg_info *curseg = CURSEG_I(sbi, type); in __add_sum_entry()
1964 int npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) in npages_for_summary_flush() argument
1970 if (sbi->ckpt->alloc_type[i] == SSR) in npages_for_summary_flush()
1971 valid_sum_count += sbi->blocks_per_seg; in npages_for_summary_flush()
1975 F2FS_CKPT(sbi)->cur_data_blkoff[i]); in npages_for_summary_flush()
1977 valid_sum_count += curseg_blkoff(sbi, i); in npages_for_summary_flush()
1994 struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno) in get_sum_page() argument
1996 return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno)); in get_sum_page()
1999 void update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr) in update_meta_page() argument
2001 struct page *page = grab_meta_page(sbi, blk_addr); in update_meta_page()
2008 static void write_sum_page(struct f2fs_sb_info *sbi, in write_sum_page() argument
2011 update_meta_page(sbi, (void *)sum_blk, blk_addr); in write_sum_page()
2014 static void write_current_sum_page(struct f2fs_sb_info *sbi, in write_current_sum_page() argument
2017 struct curseg_info *curseg = CURSEG_I(sbi, type); in write_current_sum_page()
2018 struct page *page = grab_meta_page(sbi, blk_addr); in write_current_sum_page()
2039 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type) in is_next_segment_free() argument
2041 struct curseg_info *curseg = CURSEG_I(sbi, type); in is_next_segment_free()
2043 struct free_segmap_info *free_i = FREE_I(sbi); in is_next_segment_free()
2045 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec) in is_next_segment_free()
2054 static void get_new_segment(struct f2fs_sb_info *sbi, in get_new_segment() argument
2057 struct free_segmap_info *free_i = FREE_I(sbi); in get_new_segment()
2059 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone; in get_new_segment()
2060 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg); in get_new_segment()
2061 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg); in get_new_segment()
2069 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) { in get_new_segment()
2071 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1); in get_new_segment()
2072 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) in get_new_segment()
2076 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); in get_new_segment()
2077 if (secno >= MAIN_SECS(sbi)) { in get_new_segment()
2080 MAIN_SECS(sbi), 0); in get_new_segment()
2081 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi)); in get_new_segment()
2096 MAIN_SECS(sbi), 0); in get_new_segment()
2097 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi)); in get_new_segment()
2102 segno = GET_SEG_FROM_SEC(sbi, secno); in get_new_segment()
2103 zoneno = GET_ZONE_FROM_SEC(sbi, secno); in get_new_segment()
2108 if (sbi->secs_per_zone == 1) in get_new_segment()
2119 if (CURSEG_I(sbi, i)->zone == zoneno) in get_new_segment()
2125 hint = zoneno * sbi->secs_per_zone - 1; in get_new_segment()
2129 hint = (zoneno + 1) * sbi->secs_per_zone; in get_new_segment()
2135 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap)); in get_new_segment()
2136 __set_inuse(sbi, segno); in get_new_segment()
2141 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) in reset_curseg() argument
2143 struct curseg_info *curseg = CURSEG_I(sbi, type); in reset_curseg()
2147 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno); in reset_curseg()
2157 __set_sit_entry_type(sbi, type, curseg->segno, modified); in reset_curseg()
2160 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) in __get_next_segno() argument
2163 if (sbi->segs_per_sec != 1) in __get_next_segno()
2164 return CURSEG_I(sbi, type)->segno; in __get_next_segno()
2166 if (test_opt(sbi, NOHEAP) && in __get_next_segno()
2170 if (SIT_I(sbi)->last_victim[ALLOC_NEXT]) in __get_next_segno()
2171 return SIT_I(sbi)->last_victim[ALLOC_NEXT]; in __get_next_segno()
2174 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) in __get_next_segno()
2177 return CURSEG_I(sbi, type)->segno; in __get_next_segno()
2184 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) in new_curseg() argument
2186 struct curseg_info *curseg = CURSEG_I(sbi, type); in new_curseg()
2190 write_sum_page(sbi, curseg->sum_blk, in new_curseg()
2191 GET_SUM_BLOCK(sbi, segno)); in new_curseg()
2195 if (test_opt(sbi, NOHEAP)) in new_curseg()
2198 segno = __get_next_segno(sbi, type); in new_curseg()
2199 get_new_segment(sbi, &segno, new_sec, dir); in new_curseg()
2201 reset_curseg(sbi, type, 1); in new_curseg()
2205 static void __next_free_blkoff(struct f2fs_sb_info *sbi, in __next_free_blkoff() argument
2208 struct seg_entry *se = get_seg_entry(sbi, seg->segno); in __next_free_blkoff()
2210 unsigned long *target_map = SIT_I(sbi)->tmp_map; in __next_free_blkoff()
2218 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start); in __next_free_blkoff()
2228 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, in __refresh_next_blkoff() argument
2232 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1); in __refresh_next_blkoff()
2241 static void change_curseg(struct f2fs_sb_info *sbi, int type) in change_curseg() argument
2243 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in change_curseg()
2244 struct curseg_info *curseg = CURSEG_I(sbi, type); in change_curseg()
2249 write_sum_page(sbi, curseg->sum_blk, in change_curseg()
2250 GET_SUM_BLOCK(sbi, curseg->segno)); in change_curseg()
2251 __set_test_and_inuse(sbi, new_segno); in change_curseg()
2254 __remove_dirty_segment(sbi, new_segno, PRE); in change_curseg()
2255 __remove_dirty_segment(sbi, new_segno, DIRTY); in change_curseg()
2258 reset_curseg(sbi, type, 1); in change_curseg()
2260 __next_free_blkoff(sbi, curseg, 0); in change_curseg()
2262 sum_page = get_sum_page(sbi, new_segno); in change_curseg()
2268 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type) in get_ssr_segment() argument
2270 struct curseg_info *curseg = CURSEG_I(sbi, type); in get_ssr_segment()
2271 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops; in get_ssr_segment()
2277 if (v_ops->get_victim(sbi, &segno, BG_GC, type, SSR)) { in get_ssr_segment()
2304 if (v_ops->get_victim(sbi, &segno, BG_GC, i, SSR)) { in get_ssr_segment()
2316 static void allocate_segment_by_default(struct f2fs_sb_info *sbi, in allocate_segment_by_default() argument
2319 struct curseg_info *curseg = CURSEG_I(sbi, type); in allocate_segment_by_default()
2322 new_curseg(sbi, type, true); in allocate_segment_by_default()
2323 else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) && in allocate_segment_by_default()
2325 new_curseg(sbi, type, false); in allocate_segment_by_default()
2326 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type)) in allocate_segment_by_default()
2327 new_curseg(sbi, type, false); in allocate_segment_by_default()
2328 else if (need_SSR(sbi) && get_ssr_segment(sbi, type)) in allocate_segment_by_default()
2329 change_curseg(sbi, type); in allocate_segment_by_default()
2331 new_curseg(sbi, type, false); in allocate_segment_by_default()
2333 stat_inc_seg_type(sbi, curseg); in allocate_segment_by_default()
2336 void allocate_new_segments(struct f2fs_sb_info *sbi) in allocate_new_segments() argument
2342 down_write(&SIT_I(sbi)->sentry_lock); in allocate_new_segments()
2345 curseg = CURSEG_I(sbi, i); in allocate_new_segments()
2347 SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true); in allocate_new_segments()
2348 locate_dirty_segment(sbi, old_segno); in allocate_new_segments()
2351 up_write(&SIT_I(sbi)->sentry_lock); in allocate_new_segments()
2358 bool exist_trim_candidates(struct f2fs_sb_info *sbi, struct cp_control *cpc) in exist_trim_candidates() argument
2363 down_write(&SIT_I(sbi)->sentry_lock); in exist_trim_candidates()
2365 if (add_discard_addrs(sbi, cpc, true)) { in exist_trim_candidates()
2370 up_write(&SIT_I(sbi)->sentry_lock); in exist_trim_candidates()
2376 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) in f2fs_trim_fs() argument
2387 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) in f2fs_trim_fs()
2390 if (end <= MAIN_BLKADDR(sbi)) in f2fs_trim_fs()
2393 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { in f2fs_trim_fs()
2394 f2fs_msg(sbi->sb, KERN_WARNING, in f2fs_trim_fs()
2400 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); in f2fs_trim_fs()
2401 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : in f2fs_trim_fs()
2402 GET_SEGNO(sbi, end); in f2fs_trim_fs()
2412 if (sbi->discard_blks == 0) in f2fs_trim_fs()
2414 else if (sbi->discard_blks < BATCHED_TRIM_BLOCKS(sbi)) in f2fs_trim_fs()
2419 BATCHED_TRIM_SEGMENTS(sbi), in f2fs_trim_fs()
2420 sbi->segs_per_sec) - 1, end_segno); in f2fs_trim_fs()
2422 mutex_lock(&sbi->gc_mutex); in f2fs_trim_fs()
2423 err = write_checkpoint(sbi, &cpc); in f2fs_trim_fs()
2424 mutex_unlock(&sbi->gc_mutex); in f2fs_trim_fs()
2431 start_block = START_BLOCK(sbi, start_segno); in f2fs_trim_fs()
2432 end_block = START_BLOCK(sbi, min(cur_segno, end_segno) + 1); in f2fs_trim_fs()
2435 __issue_discard_cmd_range(sbi, &dpolicy, start_block, end_block); in f2fs_trim_fs()
2436 trimmed = __wait_discard_cmd_range(sbi, &dpolicy, in f2fs_trim_fs()
2443 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type) in __has_curseg_space() argument
2445 struct curseg_info *curseg = CURSEG_I(sbi, type); in __has_curseg_space()
2446 if (curseg->next_blkoff < sbi->blocks_per_seg) in __has_curseg_space()
2524 enum rw_hint io_type_to_rw_hint(struct f2fs_sb_info *sbi, in io_type_to_rw_hint() argument
2527 if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) { in io_type_to_rw_hint()
2538 } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) { in io_type_to_rw_hint()
2607 switch (F2FS_OPTION(fio->sbi).active_logs) { in __get_segment_type()
2618 f2fs_bug_on(fio->sbi, true); in __get_segment_type()
2630 void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, in allocate_data_block() argument
2635 struct sit_info *sit_i = SIT_I(sbi); in allocate_data_block()
2636 struct curseg_info *curseg = CURSEG_I(sbi, type); in allocate_data_block()
2638 down_read(&SM_I(sbi)->curseg_lock); in allocate_data_block()
2643 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); in allocate_data_block()
2645 f2fs_wait_discard_bio(sbi, *new_blkaddr); in allocate_data_block()
2652 __add_sum_entry(sbi, type, sum); in allocate_data_block()
2654 __refresh_next_blkoff(sbi, curseg); in allocate_data_block()
2656 stat_inc_block_count(sbi, curseg); in allocate_data_block()
2662 update_sit_entry(sbi, *new_blkaddr, 1); in allocate_data_block()
2663 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) in allocate_data_block()
2664 update_sit_entry(sbi, old_blkaddr, -1); in allocate_data_block()
2666 if (!__has_curseg_space(sbi, type)) in allocate_data_block()
2667 sit_i->s_ops->allocate_segment(sbi, type, false); in allocate_data_block()
2674 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); in allocate_data_block()
2675 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr)); in allocate_data_block()
2680 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); in allocate_data_block()
2682 f2fs_inode_chksum_set(sbi, page); in allocate_data_block()
2690 io = sbi->write_io[fio->type] + fio->temp; in allocate_data_block()
2698 up_read(&SM_I(sbi)->curseg_lock); in allocate_data_block()
2703 struct f2fs_sb_info *sbi = fio->sbi; in update_device_state() local
2706 if (!sbi->s_ndevs) in update_device_state()
2709 devidx = f2fs_target_device_index(sbi, fio->new_blkaddr); in update_device_state()
2712 set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO); in update_device_state()
2715 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) { in update_device_state()
2716 spin_lock(&sbi->dev_lock); in update_device_state()
2717 f2fs_set_bit(devidx, (char *)&sbi->dirty_device); in update_device_state()
2718 spin_unlock(&sbi->dev_lock); in update_device_state()
2728 allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr, in do_write_page()
2741 void write_meta_page(struct f2fs_sb_info *sbi, struct page *page, in write_meta_page() argument
2745 .sbi = sbi, in write_meta_page()
2757 if (unlikely(page->index >= MAIN_BLKADDR(sbi))) in write_meta_page()
2763 f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE); in write_meta_page()
2773 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); in write_node_page()
2778 struct f2fs_sb_info *sbi = fio->sbi; in write_data_page() local
2782 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); in write_data_page()
2783 get_node_info(sbi, dn->nid, &ni); in write_data_page()
2788 f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE); in write_data_page()
2794 struct f2fs_sb_info *sbi = fio->sbi; in rewrite_data_page() local
2800 f2fs_bug_on(sbi, !IS_DATASEG(get_seg_entry(sbi, in rewrite_data_page()
2801 GET_SEGNO(sbi, fio->new_blkaddr))->type)); in rewrite_data_page()
2803 stat_inc_inplace_blocks(fio->sbi); in rewrite_data_page()
2809 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); in rewrite_data_page()
2814 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi, in __f2fs_get_curseg() argument
2820 if (CURSEG_I(sbi, i)->segno == segno) in __f2fs_get_curseg()
2826 void __f2fs_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, in __f2fs_replace_block() argument
2830 struct sit_info *sit_i = SIT_I(sbi); in __f2fs_replace_block()
2837 segno = GET_SEGNO(sbi, new_blkaddr); in __f2fs_replace_block()
2838 se = get_seg_entry(sbi, segno); in __f2fs_replace_block()
2841 down_write(&SM_I(sbi)->curseg_lock); in __f2fs_replace_block()
2845 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) { in __f2fs_replace_block()
2852 if (IS_CURSEG(sbi, segno)) { in __f2fs_replace_block()
2854 type = __f2fs_get_curseg(sbi, segno); in __f2fs_replace_block()
2855 f2fs_bug_on(sbi, type == NO_CHECK_TYPE); in __f2fs_replace_block()
2861 f2fs_bug_on(sbi, !IS_DATASEG(type)); in __f2fs_replace_block()
2862 curseg = CURSEG_I(sbi, type); in __f2fs_replace_block()
2873 change_curseg(sbi, type); in __f2fs_replace_block()
2876 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); in __f2fs_replace_block()
2877 __add_sum_entry(sbi, type, sum); in __f2fs_replace_block()
2880 update_sit_entry(sbi, new_blkaddr, 1); in __f2fs_replace_block()
2881 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) in __f2fs_replace_block()
2882 update_sit_entry(sbi, old_blkaddr, -1); in __f2fs_replace_block()
2884 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); in __f2fs_replace_block()
2885 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr)); in __f2fs_replace_block()
2887 locate_dirty_segment(sbi, old_cursegno); in __f2fs_replace_block()
2892 change_curseg(sbi, type); in __f2fs_replace_block()
2899 up_write(&SM_I(sbi)->curseg_lock); in __f2fs_replace_block()
2902 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, in f2fs_replace_block() argument
2911 __f2fs_replace_block(sbi, &sum, old_addr, new_addr, in f2fs_replace_block()
2921 struct f2fs_sb_info *sbi = F2FS_P_SB(page); in f2fs_wait_on_page_writeback() local
2923 f2fs_submit_merged_write_cond(sbi, page->mapping->host, in f2fs_wait_on_page_writeback()
2932 void f2fs_wait_on_block_writeback(struct f2fs_sb_info *sbi, block_t blkaddr) in f2fs_wait_on_block_writeback() argument
2939 cpage = find_lock_page(META_MAPPING(sbi), blkaddr); in f2fs_wait_on_block_writeback()
2946 static void read_compacted_summaries(struct f2fs_sb_info *sbi) in read_compacted_summaries() argument
2948 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in read_compacted_summaries()
2955 start = start_sum_block(sbi); in read_compacted_summaries()
2957 page = get_meta_page(sbi, start++); in read_compacted_summaries()
2961 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); in read_compacted_summaries()
2965 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); in read_compacted_summaries()
2974 seg_i = CURSEG_I(sbi, i); in read_compacted_summaries()
2978 reset_curseg(sbi, i, 0); in read_compacted_summaries()
2983 blk_off = sbi->blocks_per_seg; in read_compacted_summaries()
2997 page = get_meta_page(sbi, start++); in read_compacted_summaries()
3005 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) in read_normal_summaries() argument
3007 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in read_normal_summaries()
3020 if (__exist_node_summaries(sbi)) in read_normal_summaries()
3021 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type); in read_normal_summaries()
3023 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); in read_normal_summaries()
3029 if (__exist_node_summaries(sbi)) in read_normal_summaries()
3030 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, in read_normal_summaries()
3033 blk_addr = GET_SUM_BLOCK(sbi, segno); in read_normal_summaries()
3036 new = get_meta_page(sbi, blk_addr); in read_normal_summaries()
3040 if (__exist_node_summaries(sbi)) { in read_normal_summaries()
3043 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) { in read_normal_summaries()
3048 restore_node_summary(sbi, segno, sum); in read_normal_summaries()
3053 curseg = CURSEG_I(sbi, type); in read_normal_summaries()
3064 reset_curseg(sbi, type, 0); in read_normal_summaries()
3072 static int restore_curseg_summaries(struct f2fs_sb_info *sbi) in restore_curseg_summaries() argument
3074 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal; in restore_curseg_summaries()
3075 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal; in restore_curseg_summaries()
3079 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) { in restore_curseg_summaries()
3080 int npages = npages_for_summary_flush(sbi, true); in restore_curseg_summaries()
3083 ra_meta_pages(sbi, start_sum_block(sbi), npages, in restore_curseg_summaries()
3087 read_compacted_summaries(sbi); in restore_curseg_summaries()
3091 if (__exist_node_summaries(sbi)) in restore_curseg_summaries()
3092 ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type), in restore_curseg_summaries()
3096 err = read_normal_summaries(sbi, type); in restore_curseg_summaries()
3109 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) in write_compacted_summaries() argument
3118 page = grab_meta_page(sbi, blkaddr++); in write_compacted_summaries()
3122 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); in write_compacted_summaries()
3127 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); in write_compacted_summaries()
3134 seg_i = CURSEG_I(sbi, i); in write_compacted_summaries()
3135 if (sbi->ckpt->alloc_type[i] == SSR) in write_compacted_summaries()
3136 blkoff = sbi->blocks_per_seg; in write_compacted_summaries()
3138 blkoff = curseg_blkoff(sbi, i); in write_compacted_summaries()
3142 page = grab_meta_page(sbi, blkaddr++); in write_compacted_summaries()
3165 static void write_normal_summaries(struct f2fs_sb_info *sbi, in write_normal_summaries() argument
3175 write_current_sum_page(sbi, i, blkaddr + (i - type)); in write_normal_summaries()
3178 void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) in write_data_summaries() argument
3180 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) in write_data_summaries()
3181 write_compacted_summaries(sbi, start_blk); in write_data_summaries()
3183 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); in write_data_summaries()
3186 void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) in write_node_summaries() argument
3188 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); in write_node_summaries()
3213 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, in get_current_sit_page() argument
3216 return get_meta_page(sbi, current_sit_addr(sbi, segno)); in get_current_sit_page()
3219 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, in get_next_sit_page() argument
3222 struct sit_info *sit_i = SIT_I(sbi); in get_next_sit_page()
3226 src_off = current_sit_addr(sbi, start); in get_next_sit_page()
3227 dst_off = next_sit_addr(sbi, src_off); in get_next_sit_page()
3229 page = grab_meta_page(sbi, dst_off); in get_next_sit_page()
3230 seg_info_to_sit_page(sbi, page, start); in get_next_sit_page()
3289 static void add_sits_in_set(struct f2fs_sb_info *sbi) in add_sits_in_set() argument
3291 struct f2fs_sm_info *sm_info = SM_I(sbi); in add_sits_in_set()
3293 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; in add_sits_in_set()
3296 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) in add_sits_in_set()
3300 static void remove_sits_in_journal(struct f2fs_sb_info *sbi) in remove_sits_in_journal() argument
3302 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); in remove_sits_in_journal()
3312 dirtied = __mark_sit_entry_dirty(sbi, segno); in remove_sits_in_journal()
3315 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); in remove_sits_in_journal()
3325 void flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) in flush_sit_entries() argument
3327 struct sit_info *sit_i = SIT_I(sbi); in flush_sit_entries()
3329 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); in flush_sit_entries()
3332 struct list_head *head = &SM_I(sbi)->sit_entry_set; in flush_sit_entries()
3345 add_sits_in_set(sbi); in flush_sit_entries()
3353 remove_sits_in_journal(sbi); in flush_sit_entries()
3365 (unsigned long)MAIN_SEGS(sbi)); in flush_sit_entries()
3375 page = get_next_sit_page(sbi, start_segno); in flush_sit_entries()
3383 se = get_seg_entry(sbi, segno); in flush_sit_entries()
3388 add_discard_addrs(sbi, cpc, false); in flush_sit_entries()
3394 f2fs_bug_on(sbi, offset < 0); in flush_sit_entries()
3415 f2fs_bug_on(sbi, ses->entry_cnt); in flush_sit_entries()
3419 f2fs_bug_on(sbi, !list_empty(head)); in flush_sit_entries()
3420 f2fs_bug_on(sbi, sit_i->dirty_sentries); in flush_sit_entries()
3426 add_discard_addrs(sbi, cpc, false); in flush_sit_entries()
3432 set_prefree_as_free_segments(sbi); in flush_sit_entries()
3435 static int build_sit_info(struct f2fs_sb_info *sbi) in build_sit_info() argument
3437 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); in build_sit_info()
3444 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL); in build_sit_info()
3448 SM_I(sbi)->sit_info = sit_i; in build_sit_info()
3450 sit_i->sentries = f2fs_kvzalloc(sbi, MAIN_SEGS(sbi) * in build_sit_info()
3455 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); in build_sit_info()
3456 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, bitmap_size, in build_sit_info()
3461 for (start = 0; start < MAIN_SEGS(sbi); start++) { in build_sit_info()
3463 = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); in build_sit_info()
3465 = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); in build_sit_info()
3472 = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); in build_sit_info()
3477 if (f2fs_discard_en(sbi)) { in build_sit_info()
3479 = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, in build_sit_info()
3486 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); in build_sit_info()
3490 if (sbi->segs_per_sec > 1) { in build_sit_info()
3491 sit_i->sec_entries = f2fs_kvzalloc(sbi, MAIN_SECS(sbi) * in build_sit_info()
3501 bitmap_size = __bitmap_size(sbi, SIT_BITMAP); in build_sit_info()
3502 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); in build_sit_info()
3518 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; in build_sit_info()
3523 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); in build_sit_info()
3529 static int build_free_segmap(struct f2fs_sb_info *sbi) in build_free_segmap() argument
3535 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL); in build_free_segmap()
3539 SM_I(sbi)->free_info = free_i; in build_free_segmap()
3541 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); in build_free_segmap()
3542 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL); in build_free_segmap()
3546 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); in build_free_segmap()
3547 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL); in build_free_segmap()
3556 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi)); in build_free_segmap()
3563 static int build_curseg(struct f2fs_sb_info *sbi) in build_curseg() argument
3568 array = f2fs_kzalloc(sbi, sizeof(*array) * NR_CURSEG_TYPE, GFP_KERNEL); in build_curseg()
3572 SM_I(sbi)->curseg_array = array; in build_curseg()
3576 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL); in build_curseg()
3580 array[i].journal = f2fs_kzalloc(sbi, in build_curseg()
3587 return restore_curseg_summaries(sbi); in build_curseg()
3590 static int build_sit_entries(struct f2fs_sb_info *sbi) in build_sit_entries() argument
3592 struct sit_info *sit_i = SIT_I(sbi); in build_sit_entries()
3593 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); in build_sit_entries()
3597 int sit_blk_cnt = SIT_BLK_CNT(sbi); in build_sit_entries()
3603 readed = ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES, in build_sit_entries()
3609 for (; start < end && start < MAIN_SEGS(sbi); start++) { in build_sit_entries()
3614 page = get_current_sit_page(sbi, start); in build_sit_entries()
3619 err = check_block_count(sbi, start, &sit); in build_sit_entries()
3625 if (f2fs_discard_en(sbi)) { in build_sit_entries()
3626 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { in build_sit_entries()
3633 sbi->discard_blks += in build_sit_entries()
3634 sbi->blocks_per_seg - in build_sit_entries()
3639 if (sbi->segs_per_sec > 1) in build_sit_entries()
3640 get_sec_entry(sbi, start)->valid_blocks += in build_sit_entries()
3656 err = check_block_count(sbi, start, &sit); in build_sit_entries()
3661 if (f2fs_discard_en(sbi)) { in build_sit_entries()
3662 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { in build_sit_entries()
3668 sbi->discard_blks += old_valid_blocks - in build_sit_entries()
3673 if (sbi->segs_per_sec > 1) in build_sit_entries()
3674 get_sec_entry(sbi, start)->valid_blocks += in build_sit_entries()
3681 static void init_free_segmap(struct f2fs_sb_info *sbi) in init_free_segmap() argument
3686 for (start = 0; start < MAIN_SEGS(sbi); start++) { in init_free_segmap()
3687 struct seg_entry *sentry = get_seg_entry(sbi, start); in init_free_segmap()
3689 __set_free(sbi, start); in init_free_segmap()
3691 SIT_I(sbi)->written_valid_blocks += in init_free_segmap()
3697 struct curseg_info *curseg_t = CURSEG_I(sbi, type); in init_free_segmap()
3698 __set_test_and_inuse(sbi, curseg_t->segno); in init_free_segmap()
3702 static void init_dirty_segmap(struct f2fs_sb_info *sbi) in init_dirty_segmap() argument
3704 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in init_dirty_segmap()
3705 struct free_segmap_info *free_i = FREE_I(sbi); in init_dirty_segmap()
3711 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset); in init_dirty_segmap()
3712 if (segno >= MAIN_SEGS(sbi)) in init_dirty_segmap()
3715 valid_blocks = get_valid_blocks(sbi, segno, false); in init_dirty_segmap()
3716 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks) in init_dirty_segmap()
3718 if (valid_blocks > sbi->blocks_per_seg) { in init_dirty_segmap()
3719 f2fs_bug_on(sbi, 1); in init_dirty_segmap()
3723 __locate_dirty_segment(sbi, segno, DIRTY); in init_dirty_segmap()
3728 static int init_victim_secmap(struct f2fs_sb_info *sbi) in init_victim_secmap() argument
3730 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in init_victim_secmap()
3731 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); in init_victim_secmap()
3733 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); in init_victim_secmap()
3739 static int build_dirty_segmap(struct f2fs_sb_info *sbi) in build_dirty_segmap() argument
3745 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info), in build_dirty_segmap()
3750 SM_I(sbi)->dirty_info = dirty_i; in build_dirty_segmap()
3753 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); in build_dirty_segmap()
3756 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size, in build_dirty_segmap()
3762 init_dirty_segmap(sbi); in build_dirty_segmap()
3763 return init_victim_secmap(sbi); in build_dirty_segmap()
3769 static void init_min_max_mtime(struct f2fs_sb_info *sbi) in init_min_max_mtime() argument
3771 struct sit_info *sit_i = SIT_I(sbi); in init_min_max_mtime()
3778 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { in init_min_max_mtime()
3782 for (i = 0; i < sbi->segs_per_sec; i++) in init_min_max_mtime()
3783 mtime += get_seg_entry(sbi, segno + i)->mtime; in init_min_max_mtime()
3785 mtime = div_u64(mtime, sbi->segs_per_sec); in init_min_max_mtime()
3790 sit_i->max_mtime = get_mtime(sbi); in init_min_max_mtime()
3794 int build_segment_manager(struct f2fs_sb_info *sbi) in build_segment_manager() argument
3796 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); in build_segment_manager()
3797 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in build_segment_manager()
3801 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL); in build_segment_manager()
3806 sbi->sm_info = sm_info; in build_segment_manager()
3819 if (!test_opt(sbi, LFS)) in build_segment_manager()
3824 sm_info->min_ssr_sections = reserved_sections(sbi); in build_segment_manager()
3832 if (!f2fs_readonly(sbi->sb)) { in build_segment_manager()
3833 err = create_flush_cmd_control(sbi); in build_segment_manager()
3838 err = create_discard_cmd_control(sbi); in build_segment_manager()
3842 err = build_sit_info(sbi); in build_segment_manager()
3845 err = build_free_segmap(sbi); in build_segment_manager()
3848 err = build_curseg(sbi); in build_segment_manager()
3853 err = build_sit_entries(sbi); in build_segment_manager()
3857 init_free_segmap(sbi); in build_segment_manager()
3858 err = build_dirty_segmap(sbi); in build_segment_manager()
3862 init_min_max_mtime(sbi); in build_segment_manager()
3866 static void discard_dirty_segmap(struct f2fs_sb_info *sbi, in discard_dirty_segmap() argument
3869 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in discard_dirty_segmap()
3877 static void destroy_victim_secmap(struct f2fs_sb_info *sbi) in destroy_victim_secmap() argument
3879 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in destroy_victim_secmap()
3883 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) in destroy_dirty_segmap() argument
3885 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in destroy_dirty_segmap()
3893 discard_dirty_segmap(sbi, i); in destroy_dirty_segmap()
3895 destroy_victim_secmap(sbi); in destroy_dirty_segmap()
3896 SM_I(sbi)->dirty_info = NULL; in destroy_dirty_segmap()
3900 static void destroy_curseg(struct f2fs_sb_info *sbi) in destroy_curseg() argument
3902 struct curseg_info *array = SM_I(sbi)->curseg_array; in destroy_curseg()
3907 SM_I(sbi)->curseg_array = NULL; in destroy_curseg()
3915 static void destroy_free_segmap(struct f2fs_sb_info *sbi) in destroy_free_segmap() argument
3917 struct free_segmap_info *free_i = SM_I(sbi)->free_info; in destroy_free_segmap()
3920 SM_I(sbi)->free_info = NULL; in destroy_free_segmap()
3926 static void destroy_sit_info(struct f2fs_sb_info *sbi) in destroy_sit_info() argument
3928 struct sit_info *sit_i = SIT_I(sbi); in destroy_sit_info()
3935 for (start = 0; start < MAIN_SEGS(sbi); start++) { in destroy_sit_info()
3950 SM_I(sbi)->sit_info = NULL; in destroy_sit_info()
3958 void destroy_segment_manager(struct f2fs_sb_info *sbi) in destroy_segment_manager() argument
3960 struct f2fs_sm_info *sm_info = SM_I(sbi); in destroy_segment_manager()
3964 destroy_flush_cmd_control(sbi, true); in destroy_segment_manager()
3965 destroy_discard_cmd_control(sbi); in destroy_segment_manager()
3966 destroy_dirty_segmap(sbi); in destroy_segment_manager()
3967 destroy_curseg(sbi); in destroy_segment_manager()
3968 destroy_free_segmap(sbi); in destroy_segment_manager()
3969 destroy_sit_info(sbi); in destroy_segment_manager()
3970 sbi->sm_info = NULL; in destroy_segment_manager()