171 bool f2fs_need_SSR(struct f2fs_sb_info *sbi)  in f2fs_need_SSR()  argument
173 	int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);  in f2fs_need_SSR()
174 	int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);  in f2fs_need_SSR()
175 	int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);  in f2fs_need_SSR()
177 	if (f2fs_lfs_mode(sbi))  in f2fs_need_SSR()
179 	if (sbi->gc_mode == GC_URGENT_HIGH)  in f2fs_need_SSR()
181 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))  in f2fs_need_SSR()
184 	return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +  in f2fs_need_SSR()
185 			SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));  in f2fs_need_SSR()
215 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in __replace_atomic_write_block()  local
231 	err = f2fs_get_node_info(sbi, dn.nid, &ni, false);  in __replace_atomic_write_block()
241 				dec_valid_block_count(sbi, inode, 1);  in __replace_atomic_write_block()
242 			f2fs_invalidate_blocks(sbi, dn.data_blkaddr);  in __replace_atomic_write_block()
245 			f2fs_replace_block(sbi, &dn, dn.data_blkaddr,  in __replace_atomic_write_block()
251 		err = inc_valid_block_count(sbi, inode, &count);  in __replace_atomic_write_block()
259 		dec_valid_block_count(sbi, F2FS_I(inode)->cow_inode, count);  in __replace_atomic_write_block()
261 		f2fs_replace_block(sbi, &dn, dn.data_blkaddr, new_addr,  in __replace_atomic_write_block()
298 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in __f2fs_commit_atomic_write()  local
333 			} else if (!f2fs_is_valid_blkaddr(sbi, blkaddr,  in __f2fs_commit_atomic_write()
337 				f2fs_handle_error(sbi,  in __f2fs_commit_atomic_write()
365 		sbi->revoked_atomic_block += fi->atomic_write_cnt;  in __f2fs_commit_atomic_write()
367 		sbi->committed_atomic_block += fi->atomic_write_cnt;  in __f2fs_commit_atomic_write()
378 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_commit_atomic_write()  local
387 	f2fs_lock_op(sbi);  in f2fs_commit_atomic_write()
391 	f2fs_unlock_op(sbi);  in f2fs_commit_atomic_write()
401 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)  in f2fs_balance_fs()  argument
403 	if (time_to_inject(sbi, FAULT_CHECKPOINT))  in f2fs_balance_fs()
404 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT);  in f2fs_balance_fs()
407 	if (need && excess_cached_nats(sbi))  in f2fs_balance_fs()
408 		f2fs_balance_fs_bg(sbi, false);  in f2fs_balance_fs()
410 	if (!f2fs_is_checkpoint_ready(sbi))  in f2fs_balance_fs()
417 	if (has_enough_free_secs(sbi, 0, 0))  in f2fs_balance_fs()
420 	if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&  in f2fs_balance_fs()
421 				sbi->gc_thread->f2fs_gc_task) {  in f2fs_balance_fs()
424 		prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,  in f2fs_balance_fs()
426 		wake_up(&sbi->gc_thread->gc_wait_queue_head);  in f2fs_balance_fs()
428 		finish_wait(&sbi->gc_thread->fggc_wq, &wait);  in f2fs_balance_fs()
437 		f2fs_down_write(&sbi->gc_lock);  in f2fs_balance_fs()
438 		stat_inc_gc_call_count(sbi, FOREGROUND);  in f2fs_balance_fs()
439 		f2fs_gc(sbi, &gc_control);  in f2fs_balance_fs()
443 static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi)  in excess_dirty_threshold()  argument
445 	int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2;  in excess_dirty_threshold()
446 	unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS);  in excess_dirty_threshold()
447 	unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA);  in excess_dirty_threshold()
448 	unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES);  in excess_dirty_threshold()
449 	unsigned int meta = get_pages(sbi, F2FS_DIRTY_META);  in excess_dirty_threshold()
450 	unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA);  in excess_dirty_threshold()
451 	unsigned int threshold = sbi->blocks_per_seg * factor *  in excess_dirty_threshold()
462 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)  in f2fs_balance_fs_bg()  argument
464 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))  in f2fs_balance_fs_bg()
468 	if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE))  in f2fs_balance_fs_bg()
469 		f2fs_shrink_read_extent_tree(sbi,  in f2fs_balance_fs_bg()
473 	if (!f2fs_available_free_memory(sbi, AGE_EXTENT_CACHE))  in f2fs_balance_fs_bg()
474 		f2fs_shrink_age_extent_tree(sbi,  in f2fs_balance_fs_bg()
478 	if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))  in f2fs_balance_fs_bg()
479 		f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);  in f2fs_balance_fs_bg()
481 	if (!f2fs_available_free_memory(sbi, FREE_NIDS))  in f2fs_balance_fs_bg()
482 		f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);  in f2fs_balance_fs_bg()
484 		f2fs_build_free_nids(sbi, false, false);  in f2fs_balance_fs_bg()
486 	if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) ||  in f2fs_balance_fs_bg()
487 		excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi))  in f2fs_balance_fs_bg()
491 	if (is_inflight_io(sbi, REQ_TIME) ||  in f2fs_balance_fs_bg()
492 		(!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem)))  in f2fs_balance_fs_bg()
496 	if (f2fs_time_over(sbi, CP_TIME))  in f2fs_balance_fs_bg()
500 	if (f2fs_available_free_memory(sbi, NAT_ENTRIES) &&  in f2fs_balance_fs_bg()
501 		f2fs_available_free_memory(sbi, INO_ENTRIES))  in f2fs_balance_fs_bg()
505 	if (test_opt(sbi, DATA_FLUSH) && from_bg) {  in f2fs_balance_fs_bg()
508 		mutex_lock(&sbi->flush_lock);  in f2fs_balance_fs_bg()
511 		f2fs_sync_dirty_inodes(sbi, FILE_INODE, false);  in f2fs_balance_fs_bg()
514 		mutex_unlock(&sbi->flush_lock);  in f2fs_balance_fs_bg()
516 	stat_inc_cp_call_count(sbi, BACKGROUND);  in f2fs_balance_fs_bg()
517 	f2fs_sync_fs(sbi->sb, 1);  in f2fs_balance_fs_bg()
520 static int __submit_flush_wait(struct f2fs_sb_info *sbi,  in __submit_flush_wait()  argument
525 	trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),  in __submit_flush_wait()
526 				test_opt(sbi, FLUSH_MERGE), ret);  in __submit_flush_wait()
528 		f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0);  in __submit_flush_wait()
532 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)  in submit_flush_wait()  argument
537 	if (!f2fs_is_multi_device(sbi))  in submit_flush_wait()
538 		return __submit_flush_wait(sbi, sbi->sb->s_bdev);  in submit_flush_wait()
540 	for (i = 0; i < sbi->s_ndevs; i++) {  in submit_flush_wait()
541 		if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))  in submit_flush_wait()
543 		ret = __submit_flush_wait(sbi, FDEV(i).bdev);  in submit_flush_wait()
552 	struct f2fs_sb_info *sbi = data;  in issue_flush_thread()  local
553 	struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;  in issue_flush_thread()
568 		ret = submit_flush_wait(sbi, cmd->ino);  in issue_flush_thread()
584 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)  in f2fs_issue_flush()  argument
586 	struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;  in f2fs_issue_flush()
590 	if (test_opt(sbi, NOBARRIER))  in f2fs_issue_flush()
593 	if (!test_opt(sbi, FLUSH_MERGE)) {  in f2fs_issue_flush()
595 		ret = submit_flush_wait(sbi, ino);  in f2fs_issue_flush()
602 	    f2fs_is_multi_device(sbi)) {  in f2fs_issue_flush()
603 		ret = submit_flush_wait(sbi, ino);  in f2fs_issue_flush()
638 			ret = submit_flush_wait(sbi, ino);  in f2fs_issue_flush()
655 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)  in f2fs_create_flush_cmd_control()  argument
657 	dev_t dev = sbi->sb->s_bdev->bd_dev;  in f2fs_create_flush_cmd_control()
660 	if (SM_I(sbi)->fcc_info) {  in f2fs_create_flush_cmd_control()
661 		fcc = SM_I(sbi)->fcc_info;  in f2fs_create_flush_cmd_control()
667 	fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);  in f2fs_create_flush_cmd_control()
674 	SM_I(sbi)->fcc_info = fcc;  in f2fs_create_flush_cmd_control()
675 	if (!test_opt(sbi, FLUSH_MERGE))  in f2fs_create_flush_cmd_control()
679 	fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,  in f2fs_create_flush_cmd_control()
691 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)  in f2fs_destroy_flush_cmd_control()  argument
693 	struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;  in f2fs_destroy_flush_cmd_control()
703 		SM_I(sbi)->fcc_info = NULL;  in f2fs_destroy_flush_cmd_control()
707 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)  in f2fs_flush_device_cache()  argument
711 	if (!f2fs_is_multi_device(sbi))  in f2fs_flush_device_cache()
714 	if (test_opt(sbi, NOBARRIER))  in f2fs_flush_device_cache()
717 	for (i = 1; i < sbi->s_ndevs; i++) {  in f2fs_flush_device_cache()
720 		if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))  in f2fs_flush_device_cache()
724 			ret = __submit_flush_wait(sbi, FDEV(i).bdev);  in f2fs_flush_device_cache()
730 			f2fs_stop_checkpoint(sbi, false,  in f2fs_flush_device_cache()
735 		spin_lock(&sbi->dev_lock);  in f2fs_flush_device_cache()
736 		f2fs_clear_bit(i, (char *)&sbi->dirty_device);  in f2fs_flush_device_cache()
737 		spin_unlock(&sbi->dev_lock);  in f2fs_flush_device_cache()
743 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,  in __locate_dirty_segment()  argument
746 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in __locate_dirty_segment()
749 	if (IS_CURSEG(sbi, segno))  in __locate_dirty_segment()
756 		struct seg_entry *sentry = get_seg_entry(sbi, segno);  in __locate_dirty_segment()
760 			f2fs_bug_on(sbi, 1);  in __locate_dirty_segment()
766 		if (__is_large_section(sbi)) {  in __locate_dirty_segment()
767 			unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);  in __locate_dirty_segment()
769 				get_valid_blocks(sbi, segno, true);  in __locate_dirty_segment()
771 			f2fs_bug_on(sbi, unlikely(!valid_blocks ||  in __locate_dirty_segment()
772 					valid_blocks == CAP_BLKS_PER_SEC(sbi)));  in __locate_dirty_segment()
774 			if (!IS_CURSEC(sbi, secno))  in __locate_dirty_segment()
780 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,  in __remove_dirty_segment()  argument
783 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in __remove_dirty_segment()
790 		struct seg_entry *sentry = get_seg_entry(sbi, segno);  in __remove_dirty_segment()
796 		valid_blocks = get_valid_blocks(sbi, segno, true);  in __remove_dirty_segment()
798 			clear_bit(GET_SEC_FROM_SEG(sbi, segno),  in __remove_dirty_segment()
801 			clear_bit(segno, SIT_I(sbi)->invalid_segmap);  in __remove_dirty_segment()
804 		if (__is_large_section(sbi)) {  in __remove_dirty_segment()
805 			unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);  in __remove_dirty_segment()
808 					valid_blocks == CAP_BLKS_PER_SEC(sbi)) {  in __remove_dirty_segment()
813 			if (!IS_CURSEC(sbi, secno))  in __remove_dirty_segment()
824 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)  in locate_dirty_segment()  argument
826 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in locate_dirty_segment()
830 	if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))  in locate_dirty_segment()
833 	usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);  in locate_dirty_segment()
836 	valid_blocks = get_valid_blocks(sbi, segno, false);  in locate_dirty_segment()
837 	ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);  in locate_dirty_segment()
839 	if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||  in locate_dirty_segment()
841 		__locate_dirty_segment(sbi, segno, PRE);  in locate_dirty_segment()
842 		__remove_dirty_segment(sbi, segno, DIRTY);  in locate_dirty_segment()
844 		__locate_dirty_segment(sbi, segno, DIRTY);  in locate_dirty_segment()
847 		__remove_dirty_segment(sbi, segno, DIRTY);  in locate_dirty_segment()
854 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)  in f2fs_dirty_to_prefree()  argument
856 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in f2fs_dirty_to_prefree()
860 	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {  in f2fs_dirty_to_prefree()
861 		if (get_valid_blocks(sbi, segno, false))  in f2fs_dirty_to_prefree()
863 		if (IS_CURSEG(sbi, segno))  in f2fs_dirty_to_prefree()
865 		__locate_dirty_segment(sbi, segno, PRE);  in f2fs_dirty_to_prefree()
866 		__remove_dirty_segment(sbi, segno, DIRTY);  in f2fs_dirty_to_prefree()
871 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)  in f2fs_get_unusable_blocks()  argument
874 		(overprovision_segments(sbi) - reserved_segments(sbi));  in f2fs_get_unusable_blocks()
875 	block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;  in f2fs_get_unusable_blocks()
876 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in f2fs_get_unusable_blocks()
883 	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {  in f2fs_get_unusable_blocks()
884 		se = get_seg_entry(sbi, segno);  in f2fs_get_unusable_blocks()
886 			holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -  in f2fs_get_unusable_blocks()
889 			holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -  in f2fs_get_unusable_blocks()
900 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)  in f2fs_disable_cp_again()  argument
903 		(overprovision_segments(sbi) - reserved_segments(sbi));  in f2fs_disable_cp_again()
904 	if (unusable > F2FS_OPTION(sbi).unusable_cap)  in f2fs_disable_cp_again()
906 	if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&  in f2fs_disable_cp_again()
907 		dirty_segments(sbi) > ovp_hole_segs)  in f2fs_disable_cp_again()
913 static unsigned int get_free_segment(struct f2fs_sb_info *sbi)  in get_free_segment()  argument
915 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in get_free_segment()
919 	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {  in get_free_segment()
920 		if (get_valid_blocks(sbi, segno, false))  in get_free_segment()
922 		if (get_ckpt_valid_blocks(sbi, segno, false))  in get_free_segment()
931 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,  in __create_discard_cmd()  argument
935 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __create_discard_cmd()
939 	f2fs_bug_on(sbi, !len);  in __create_discard_cmd()
963 static bool f2fs_check_discard_tree(struct f2fs_sb_info *sbi)  in f2fs_check_discard_tree()  argument
966 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_check_discard_tree()
979 			f2fs_info(sbi, "broken discard_rbtree, "  in f2fs_check_discard_tree()
991 static struct discard_cmd *__lookup_discard_cmd(struct f2fs_sb_info *sbi,  in __lookup_discard_cmd()  argument
994 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __lookup_discard_cmd()
1083 static void __remove_discard_cmd(struct f2fs_sb_info *sbi,  in __remove_discard_cmd()  argument
1086 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __remove_discard_cmd()
1098 	f2fs_bug_on(sbi, dc->ref);  in __remove_discard_cmd()
1106 			KERN_INFO, sbi->sb->s_id,  in __remove_discard_cmd()
1128 static void __check_sit_bitmap(struct f2fs_sb_info *sbi,  in __check_sit_bitmap()  argument
1135 	unsigned long offset, size, max_blocks = sbi->blocks_per_seg;  in __check_sit_bitmap()
1139 		segno = GET_SEGNO(sbi, blk);  in __check_sit_bitmap()
1140 		sentry = get_seg_entry(sbi, segno);  in __check_sit_bitmap()
1141 		offset = GET_BLKOFF_FROM_SEG0(sbi, blk);  in __check_sit_bitmap()
1143 		if (end < START_BLOCK(sbi, segno + 1))  in __check_sit_bitmap()
1144 			size = GET_BLKOFF_FROM_SEG0(sbi, end);  in __check_sit_bitmap()
1149 		f2fs_bug_on(sbi, offset != size);  in __check_sit_bitmap()
1150 		blk = START_BLOCK(sbi, segno + 1);  in __check_sit_bitmap()
1155 static void __init_discard_policy(struct f2fs_sb_info *sbi,  in __init_discard_policy()  argument
1159 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __init_discard_policy()
1178 		if (utilization(sbi) > dcc->discard_urgent_util) {  in __init_discard_policy()
1199 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1204 static void __submit_zone_reset_cmd(struct f2fs_sb_info *sbi,  in __submit_zone_reset_cmd()  argument
1209 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __submit_zone_reset_cmd()
1229 	__check_sit_bitmap(sbi, dc->di.lstart, dc->di.lstart + dc->di.len);  in __submit_zone_reset_cmd()
1237 	f2fs_update_iostat(sbi, NULL, FS_ZONE_RESET_IO, dc->di.len * F2FS_BLKSIZE);  in __submit_zone_reset_cmd()
1242 static int __submit_discard_cmd(struct f2fs_sb_info *sbi,  in __submit_discard_cmd()  argument
1249 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __submit_discard_cmd()
1259 	if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))  in __submit_discard_cmd()
1263 	if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) {  in __submit_discard_cmd()
1264 		int devi = f2fs_bdev_index(sbi, bdev);  in __submit_discard_cmd()
1269 		if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) {  in __submit_discard_cmd()
1270 			__submit_zone_reset_cmd(sbi, dc, flag,  in __submit_discard_cmd()
1302 		if (time_to_inject(sbi, FAULT_DISCARD)) {  in __submit_discard_cmd()
1319 		f2fs_bug_on(sbi, !bio);  in __submit_discard_cmd()
1338 		__check_sit_bitmap(sbi, lstart, lstart + len);  in __submit_discard_cmd()
1347 		f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, len * F2FS_BLKSIZE);  in __submit_discard_cmd()
1357 		__update_discard_tree_range(sbi, bdev, lstart, start, len);  in __submit_discard_cmd()
1362 static void __insert_discard_cmd(struct f2fs_sb_info *sbi,  in __insert_discard_cmd()  argument
1366 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __insert_discard_cmd()
1383 			f2fs_bug_on(sbi, 1);  in __insert_discard_cmd()
1387 	dc = __create_discard_cmd(sbi, bdev, lstart, start, len);  in __insert_discard_cmd()
1399 static void __punch_discard_cmd(struct f2fs_sb_info *sbi,  in __punch_discard_cmd()  argument
1402 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __punch_discard_cmd()
1407 		__remove_discard_cmd(sbi, dc);  in __punch_discard_cmd()
1422 			__insert_discard_cmd(sbi, dc->bdev, blkaddr + 1,  in __punch_discard_cmd()
1435 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,  in __update_discard_tree_range()  argument
1439 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __update_discard_tree_range()
1504 				__remove_discard_cmd(sbi, tdc);  in __update_discard_tree_range()
1509 			__insert_discard_cmd(sbi, bdev,  in __update_discard_tree_range()
1522 static void __queue_zone_reset_cmd(struct f2fs_sb_info *sbi,  in __queue_zone_reset_cmd()  argument
1528 	mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);  in __queue_zone_reset_cmd()
1529 	__insert_discard_cmd(sbi, bdev, lblkstart, blkstart, blklen);  in __queue_zone_reset_cmd()
1530 	mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);  in __queue_zone_reset_cmd()
1534 static void __queue_discard_cmd(struct f2fs_sb_info *sbi,  in __queue_discard_cmd()  argument
1544 	if (f2fs_is_multi_device(sbi)) {  in __queue_discard_cmd()
1545 		int devi = f2fs_target_device_index(sbi, blkstart);  in __queue_discard_cmd()
1549 	mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);  in __queue_discard_cmd()
1550 	__update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);  in __queue_discard_cmd()
1551 	mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);  in __queue_discard_cmd()
1554 static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,  in __issue_discard_cmd_orderly()  argument
1557 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd_orderly()
1579 		if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {  in __issue_discard_cmd_orderly()
1585 		err = __submit_discard_cmd(sbi, dpolicy, dc, issued);  in __issue_discard_cmd_orderly()
1592 			__remove_discard_cmd(sbi, dc);  in __issue_discard_cmd_orderly()
1606 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1609 static int __issue_discard_cmd(struct f2fs_sb_info *sbi,  in __issue_discard_cmd()  argument
1612 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd()
1620 		f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);  in __issue_discard_cmd()
1626 				f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))  in __issue_discard_cmd()
1633 			__issue_discard_cmd_orderly(sbi, dpolicy, &issued);  in __issue_discard_cmd()
1643 			f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi));  in __issue_discard_cmd()
1646 			f2fs_bug_on(sbi, dc->state != D_PREP);  in __issue_discard_cmd()
1649 				f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))  in __issue_discard_cmd()
1653 						!is_idle(sbi, DISCARD_TIME)) {  in __issue_discard_cmd()
1658 			__submit_discard_cmd(sbi, dpolicy, dc, &issued);  in __issue_discard_cmd()
1672 		__wait_all_discard_cmd(sbi, dpolicy);  in __issue_discard_cmd()
1682 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)  in __drop_discard_cmd()  argument
1684 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __drop_discard_cmd()
1694 			f2fs_bug_on(sbi, dc->state != D_PREP);  in __drop_discard_cmd()
1695 			__remove_discard_cmd(sbi, dc);  in __drop_discard_cmd()
1704 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)  in f2fs_drop_discard_cmd()  argument
1706 	__drop_discard_cmd(sbi);  in f2fs_drop_discard_cmd()
1709 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,  in __wait_one_discard_bio()  argument
1712 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __wait_one_discard_bio()
1717 	f2fs_bug_on(sbi, dc->state != D_DONE);  in __wait_one_discard_bio()
1722 		__remove_discard_cmd(sbi, dc);  in __wait_one_discard_bio()
1729 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,  in __wait_discard_cmd_range()  argument
1733 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __wait_discard_cmd_range()
1753 			__remove_discard_cmd(sbi, iter);  in __wait_discard_cmd_range()
1763 		trimmed += __wait_one_discard_bio(sbi, dc);  in __wait_discard_cmd_range()
1770 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,  in __wait_all_discard_cmd()  argument
1777 		return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);  in __wait_all_discard_cmd()
1780 	__init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, MIN_DISCARD_GRANULARITY);  in __wait_all_discard_cmd()
1781 	discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);  in __wait_all_discard_cmd()
1782 	__init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, MIN_DISCARD_GRANULARITY);  in __wait_all_discard_cmd()
1783 	discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);  in __wait_all_discard_cmd()
1789 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)  in f2fs_wait_discard_bio()  argument
1791 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_wait_discard_bio()
1796 	dc = __lookup_discard_cmd(sbi, blkaddr);  in f2fs_wait_discard_bio()
1798 	if (dc && f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(dc->bdev)) {  in f2fs_wait_discard_bio()
1799 		int devi = f2fs_bdev_index(sbi, dc->bdev);  in f2fs_wait_discard_bio()
1806 		if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) {  in f2fs_wait_discard_bio()
1809 				__submit_zone_reset_cmd(sbi, dc, REQ_SYNC,  in f2fs_wait_discard_bio()
1814 			__wait_one_discard_bio(sbi, dc);  in f2fs_wait_discard_bio()
1821 			__punch_discard_cmd(sbi, dc, blkaddr);  in f2fs_wait_discard_bio()
1830 		__wait_one_discard_bio(sbi, dc);  in f2fs_wait_discard_bio()
1833 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)  in f2fs_stop_discard_thread()  argument
1835 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_stop_discard_thread()
1847  * @sbi: the f2fs_sb_info data for discard cmd to issue
1853 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)  in f2fs_issue_discard_timeout()  argument
1855 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_issue_discard_timeout()
1862 	__init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,  in f2fs_issue_discard_timeout()
1864 	__issue_discard_cmd(sbi, &dpolicy);  in f2fs_issue_discard_timeout()
1865 	dropped = __drop_discard_cmd(sbi);  in f2fs_issue_discard_timeout()
1868 	__wait_all_discard_cmd(sbi, NULL);  in f2fs_issue_discard_timeout()
1870 	f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));  in f2fs_issue_discard_timeout()
1876 	struct f2fs_sb_info *sbi = data;  in issue_discard_thread()  local
1877 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in issue_discard_thread()
1891 		if (sbi->gc_mode == GC_URGENT_HIGH ||  in issue_discard_thread()
1892 			!f2fs_available_free_memory(sbi, DISCARD_CACHE))  in issue_discard_thread()
1893 			__init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE,  in issue_discard_thread()
1896 			__init_discard_policy(sbi, &dpolicy, DPOLICY_BG,  in issue_discard_thread()
1904 			__wait_all_discard_cmd(sbi, NULL);  in issue_discard_thread()
1908 		if (f2fs_readonly(sbi->sb))  in issue_discard_thread()
1912 		if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) ||  in issue_discard_thread()
1918 		sb_start_intwrite(sbi->sb);  in issue_discard_thread()
1920 		issued = __issue_discard_cmd(sbi, &dpolicy);  in issue_discard_thread()
1922 			__wait_all_discard_cmd(sbi, &dpolicy);  in issue_discard_thread()
1925 			wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);  in issue_discard_thread()
1934 		sb_end_intwrite(sbi->sb);  in issue_discard_thread()
1941 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,  in __f2fs_issue_discard_zone()  argument
1949 	if (f2fs_is_multi_device(sbi)) {  in __f2fs_issue_discard_zone()
1950 		devi = f2fs_target_device_index(sbi, blkstart);  in __f2fs_issue_discard_zone()
1953 			f2fs_err(sbi, "Invalid block %x", blkstart);  in __f2fs_issue_discard_zone()
1960 	if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {  in __f2fs_issue_discard_zone()
1966 			f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",  in __f2fs_issue_discard_zone()
1967 				 devi, sbi->s_ndevs ? FDEV(devi).path : "",  in __f2fs_issue_discard_zone()
1972 		if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) {  in __f2fs_issue_discard_zone()
1978 		__queue_zone_reset_cmd(sbi, bdev, blkstart, lblkstart, blklen);  in __f2fs_issue_discard_zone()
1983 	__queue_discard_cmd(sbi, bdev, lblkstart, blklen);  in __f2fs_issue_discard_zone()
1988 static int __issue_discard_async(struct f2fs_sb_info *sbi,  in __issue_discard_async()  argument
1992 	if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))  in __issue_discard_async()
1993 		return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);  in __issue_discard_async()
1995 	__queue_discard_cmd(sbi, bdev, blkstart, blklen);  in __issue_discard_async()
1999 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,  in f2fs_issue_discard()  argument
2009 	bdev = f2fs_target_device(sbi, blkstart, NULL);  in f2fs_issue_discard()
2014 				f2fs_target_device(sbi, i, NULL);  in f2fs_issue_discard()
2017 				err = __issue_discard_async(sbi, bdev,  in f2fs_issue_discard()
2027 		se = get_seg_entry(sbi, GET_SEGNO(sbi, i));  in f2fs_issue_discard()
2028 		offset = GET_BLKOFF_FROM_SEG0(sbi, i);  in f2fs_issue_discard()
2030 		if (f2fs_block_unit_discard(sbi) &&  in f2fs_issue_discard()
2032 			sbi->discard_blks--;  in f2fs_issue_discard()
2036 		err = __issue_discard_async(sbi, bdev, start, len);  in f2fs_issue_discard()
2040 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,  in add_discard_addrs()  argument
2044 	int max_blocks = sbi->blocks_per_seg;  in add_discard_addrs()
2045 	struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);  in add_discard_addrs()
2049 	unsigned long *dmap = SIT_I(sbi)->tmp_map;  in add_discard_addrs()
2053 	struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;  in add_discard_addrs()
2056 	if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi) ||  in add_discard_addrs()
2057 			!f2fs_block_unit_discard(sbi))  in add_discard_addrs()
2061 		if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||  in add_discard_addrs()
2062 			SM_I(sbi)->dcc_info->nr_discards >=  in add_discard_addrs()
2063 				SM_I(sbi)->dcc_info->max_discards)  in add_discard_addrs()
2072 	while (force || SM_I(sbi)->dcc_info->nr_discards <=  in add_discard_addrs()
2073 				SM_I(sbi)->dcc_info->max_discards) {  in add_discard_addrs()
2089 			de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);  in add_discard_addrs()
2096 		SM_I(sbi)->dcc_info->nr_discards += end - start;  in add_discard_addrs()
2107 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)  in f2fs_release_discard_addrs()  argument
2109 	struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);  in f2fs_release_discard_addrs()
2120 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)  in set_prefree_as_free_segments()  argument
2122 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in set_prefree_as_free_segments()
2126 	for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))  in set_prefree_as_free_segments()
2127 		__set_test_and_free(sbi, segno, false);  in set_prefree_as_free_segments()
2131 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,  in f2fs_clear_prefree_segments()  argument
2134 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_clear_prefree_segments()
2137 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in f2fs_clear_prefree_segments()
2142 	bool section_alignment = F2FS_OPTION(sbi).discard_unit ==  in f2fs_clear_prefree_segments()
2145 	if (f2fs_lfs_mode(sbi) && __is_large_section(sbi))  in f2fs_clear_prefree_segments()
2155 		start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);  in f2fs_clear_prefree_segments()
2156 		if (start >= MAIN_SEGS(sbi))  in f2fs_clear_prefree_segments()
2158 		end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),  in f2fs_clear_prefree_segments()
2162 			start = rounddown(start, sbi->segs_per_sec);  in f2fs_clear_prefree_segments()
2163 			end = roundup(end, sbi->segs_per_sec);  in f2fs_clear_prefree_segments()
2171 		if (!f2fs_realtime_discard_enable(sbi))  in f2fs_clear_prefree_segments()
2179 		if (!f2fs_sb_has_blkzoned(sbi) &&  in f2fs_clear_prefree_segments()
2180 		    (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi))) {  in f2fs_clear_prefree_segments()
2181 			f2fs_issue_discard(sbi, START_BLOCK(sbi, start),  in f2fs_clear_prefree_segments()
2182 				(end - start) << sbi->log_blocks_per_seg);  in f2fs_clear_prefree_segments()
2186 		secno = GET_SEC_FROM_SEG(sbi, start);  in f2fs_clear_prefree_segments()
2187 		start_segno = GET_SEG_FROM_SEC(sbi, secno);  in f2fs_clear_prefree_segments()
2188 		if (!IS_CURSEC(sbi, secno) &&  in f2fs_clear_prefree_segments()
2189 			!get_valid_blocks(sbi, start, true))  in f2fs_clear_prefree_segments()
2190 			f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),  in f2fs_clear_prefree_segments()
2191 				sbi->segs_per_sec << sbi->log_blocks_per_seg);  in f2fs_clear_prefree_segments()
2193 		start = start_segno + sbi->segs_per_sec;  in f2fs_clear_prefree_segments()
2201 	if (!f2fs_block_unit_discard(sbi))  in f2fs_clear_prefree_segments()
2212 					sbi->blocks_per_seg, cur_pos);  in f2fs_clear_prefree_segments()
2215 			if (f2fs_sb_has_blkzoned(sbi) ||  in f2fs_clear_prefree_segments()
2219 			f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,  in f2fs_clear_prefree_segments()
2224 					sbi->blocks_per_seg, cur_pos);  in f2fs_clear_prefree_segments()
2230 		if (cur_pos < sbi->blocks_per_seg)  in f2fs_clear_prefree_segments()
2238 	wake_up_discard_thread(sbi, false);  in f2fs_clear_prefree_segments()
2241 int f2fs_start_discard_thread(struct f2fs_sb_info *sbi)  in f2fs_start_discard_thread()  argument
2243 	dev_t dev = sbi->sb->s_bdev->bd_dev;  in f2fs_start_discard_thread()
2244 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_start_discard_thread()
2247 	if (!f2fs_realtime_discard_enable(sbi))  in f2fs_start_discard_thread()
2250 	dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,  in f2fs_start_discard_thread()
2260 static int create_discard_cmd_control(struct f2fs_sb_info *sbi)  in create_discard_cmd_control()  argument
2265 	if (SM_I(sbi)->dcc_info) {  in create_discard_cmd_control()
2266 		dcc = SM_I(sbi)->dcc_info;  in create_discard_cmd_control()
2270 	dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);  in create_discard_cmd_control()
2277 	if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT)  in create_discard_cmd_control()
2278 		dcc->discard_granularity = sbi->blocks_per_seg;  in create_discard_cmd_control()
2279 	else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION)  in create_discard_cmd_control()
2280 		dcc->discard_granularity = BLKS_PER_SEC(sbi);  in create_discard_cmd_control()
2292 	dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;  in create_discard_cmd_control()
2304 	SM_I(sbi)->dcc_info = dcc;  in create_discard_cmd_control()
2306 	err = f2fs_start_discard_thread(sbi);  in create_discard_cmd_control()
2309 		SM_I(sbi)->dcc_info = NULL;  in create_discard_cmd_control()
2315 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)  in destroy_discard_cmd_control()  argument
2317 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in destroy_discard_cmd_control()
2322 	f2fs_stop_discard_thread(sbi);  in destroy_discard_cmd_control()
2328 	f2fs_issue_discard_timeout(sbi);  in destroy_discard_cmd_control()
2331 	SM_I(sbi)->dcc_info = NULL;  in destroy_discard_cmd_control()
2334 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)  in __mark_sit_entry_dirty()  argument
2336 	struct sit_info *sit_i = SIT_I(sbi);  in __mark_sit_entry_dirty()
2346 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,  in __set_sit_entry_type()  argument
2349 	struct seg_entry *se = get_seg_entry(sbi, segno);  in __set_sit_entry_type()
2353 		__mark_sit_entry_dirty(sbi, segno);  in __set_sit_entry_type()
2356 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,  in get_segment_mtime()  argument
2359 	unsigned int segno = GET_SEGNO(sbi, blkaddr);  in get_segment_mtime()
2363 	return get_seg_entry(sbi, segno)->mtime;  in get_segment_mtime()
2366 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,  in update_segment_mtime()  argument
2370 	unsigned int segno = GET_SEGNO(sbi, blkaddr);  in update_segment_mtime()
2371 	unsigned long long ctime = get_mtime(sbi, false);  in update_segment_mtime()
2377 	se = get_seg_entry(sbi, segno);  in update_segment_mtime()
2385 	if (ctime > SIT_I(sbi)->max_mtime)  in update_segment_mtime()
2386 		SIT_I(sbi)->max_mtime = ctime;  in update_segment_mtime()
2389 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)  in update_sit_entry()  argument
2399 	segno = GET_SEGNO(sbi, blkaddr);  in update_sit_entry()
2401 	se = get_seg_entry(sbi, segno);  in update_sit_entry()
2403 	offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);  in update_sit_entry()
2405 	f2fs_bug_on(sbi, (new_vblocks < 0 ||  in update_sit_entry()
2406 			(new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));  in update_sit_entry()
2417 			f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",  in update_sit_entry()
2419 			f2fs_bug_on(sbi, 1);  in update_sit_entry()
2423 			f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",  in update_sit_entry()
2425 			f2fs_bug_on(sbi, 1);  in update_sit_entry()
2430 		if (f2fs_block_unit_discard(sbi) &&  in update_sit_entry()
2432 			sbi->discard_blks--;  in update_sit_entry()
2438 		if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {  in update_sit_entry()
2448 			f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",  in update_sit_entry()
2450 			f2fs_bug_on(sbi, 1);  in update_sit_entry()
2454 			f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",  in update_sit_entry()
2456 			f2fs_bug_on(sbi, 1);  in update_sit_entry()
2459 		} else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {  in update_sit_entry()
2467 				spin_lock(&sbi->stat_lock);  in update_sit_entry()
2468 				sbi->unusable_block_count++;  in update_sit_entry()
2469 				spin_unlock(&sbi->stat_lock);  in update_sit_entry()
2473 		if (f2fs_block_unit_discard(sbi) &&  in update_sit_entry()
2475 			sbi->discard_blks++;  in update_sit_entry()
2480 	__mark_sit_entry_dirty(sbi, segno);  in update_sit_entry()
2483 	SIT_I(sbi)->written_valid_blocks += del;  in update_sit_entry()
2485 	if (__is_large_section(sbi))  in update_sit_entry()
2486 		get_sec_entry(sbi, segno)->valid_blocks += del;  in update_sit_entry()
2489 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)  in f2fs_invalidate_blocks()  argument
2491 	unsigned int segno = GET_SEGNO(sbi, addr);  in f2fs_invalidate_blocks()
2492 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_invalidate_blocks()
2494 	f2fs_bug_on(sbi, addr == NULL_ADDR);  in f2fs_invalidate_blocks()
2498 	invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);  in f2fs_invalidate_blocks()
2499 	f2fs_invalidate_compress_page(sbi, addr);  in f2fs_invalidate_blocks()
2504 	update_segment_mtime(sbi, addr, 0);  in f2fs_invalidate_blocks()
2505 	update_sit_entry(sbi, addr, -1);  in f2fs_invalidate_blocks()
2508 	locate_dirty_segment(sbi, segno);  in f2fs_invalidate_blocks()
2513 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)  in f2fs_is_checkpointed_data()  argument
2515 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_is_checkpointed_data()
2525 	segno = GET_SEGNO(sbi, blkaddr);  in f2fs_is_checkpointed_data()
2526 	se = get_seg_entry(sbi, segno);  in f2fs_is_checkpointed_data()
2527 	offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);  in f2fs_is_checkpointed_data()
2537 static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int type)  in f2fs_curseg_valid_blocks()  argument
2539 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in f2fs_curseg_valid_blocks()
2541 	if (sbi->ckpt->alloc_type[type] == SSR)  in f2fs_curseg_valid_blocks()
2542 		return sbi->blocks_per_seg;  in f2fs_curseg_valid_blocks()
2549 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)  in f2fs_npages_for_summary_flush()  argument
2555 		if (sbi->ckpt->alloc_type[i] != SSR && for_ra)  in f2fs_npages_for_summary_flush()
2557 				le16_to_cpu(F2FS_CKPT(sbi)->cur_data_blkoff[i]);  in f2fs_npages_for_summary_flush()
2559 			valid_sum_count += f2fs_curseg_valid_blocks(sbi, i);  in f2fs_npages_for_summary_flush()
2575 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)  in f2fs_get_sum_page()  argument
2577 	if (unlikely(f2fs_cp_error(sbi)))  in f2fs_get_sum_page()
2579 	return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));  in f2fs_get_sum_page()
2582 void f2fs_update_meta_page(struct f2fs_sb_info *sbi,  in f2fs_update_meta_page()  argument
2585 	struct page *page = f2fs_grab_meta_page(sbi, blk_addr);  in f2fs_update_meta_page()
2592 static void write_sum_page(struct f2fs_sb_info *sbi,  in write_sum_page()  argument
2595 	f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);  in write_sum_page()
2598 static void write_current_sum_page(struct f2fs_sb_info *sbi,  in write_current_sum_page()  argument
2601 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in write_current_sum_page()
2602 	struct page *page = f2fs_grab_meta_page(sbi, blk_addr);  in write_current_sum_page()
2624 static int is_next_segment_free(struct f2fs_sb_info *sbi,  in is_next_segment_free()  argument
2628 	struct free_segmap_info *free_i = FREE_I(sbi);  in is_next_segment_free()
2630 	if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)  in is_next_segment_free()
2639 static void get_new_segment(struct f2fs_sb_info *sbi,  in get_new_segment()  argument
2642 	struct free_segmap_info *free_i = FREE_I(sbi);  in get_new_segment()
2644 	unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;  in get_new_segment()
2645 	unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);  in get_new_segment()
2646 	unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);  in get_new_segment()
2654 	if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {  in get_new_segment()
2656 			GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);  in get_new_segment()
2657 		if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))  in get_new_segment()
2661 	secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);  in get_new_segment()
2662 	if (secno >= MAIN_SECS(sbi)) {  in get_new_segment()
2665 							MAIN_SECS(sbi));  in get_new_segment()
2666 			f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));  in get_new_segment()
2681 							MAIN_SECS(sbi));  in get_new_segment()
2682 		f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));  in get_new_segment()
2687 	segno = GET_SEG_FROM_SEC(sbi, secno);  in get_new_segment()
2688 	zoneno = GET_ZONE_FROM_SEC(sbi, secno);  in get_new_segment()
2693 	if (sbi->secs_per_zone == 1)  in get_new_segment()
2704 		if (CURSEG_I(sbi, i)->zone == zoneno)  in get_new_segment()
2710 			hint = zoneno * sbi->secs_per_zone - 1;  in get_new_segment()
2714 			hint = (zoneno + 1) * sbi->secs_per_zone;  in get_new_segment()
2720 	f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));  in get_new_segment()
2721 	__set_inuse(sbi, segno);  in get_new_segment()
2726 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)  in reset_curseg()  argument
2728 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in reset_curseg()
2734 	curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);  in reset_curseg()
2741 	sanity_check_seg_type(sbi, seg_type);  in reset_curseg()
2747 	__set_sit_entry_type(sbi, seg_type, curseg->segno, modified);  in reset_curseg()
2750 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)  in __get_next_segno()  argument
2752 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __get_next_segno()
2755 	sanity_check_seg_type(sbi, seg_type);  in __get_next_segno()
2756 	if (f2fs_need_rand_seg(sbi))  in __get_next_segno()
2757 		return get_random_u32_below(MAIN_SECS(sbi) * sbi->segs_per_sec);  in __get_next_segno()
2760 	if (__is_large_section(sbi))  in __get_next_segno()
2767 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))  in __get_next_segno()
2770 	if (test_opt(sbi, NOHEAP) &&  in __get_next_segno()
2774 	if (SIT_I(sbi)->last_victim[ALLOC_NEXT])  in __get_next_segno()
2775 		return SIT_I(sbi)->last_victim[ALLOC_NEXT];  in __get_next_segno()
2778 	if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)  in __get_next_segno()
2788 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)  in new_curseg()  argument
2790 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in new_curseg()
2796 		write_sum_page(sbi, curseg->sum_blk,  in new_curseg()
2797 				GET_SUM_BLOCK(sbi, segno));  in new_curseg()
2801 	if (test_opt(sbi, NOHEAP))  in new_curseg()
2804 	segno = __get_next_segno(sbi, type);  in new_curseg()
2805 	get_new_segment(sbi, &segno, new_sec, dir);  in new_curseg()
2807 	reset_curseg(sbi, type, 1);  in new_curseg()
2809 	if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)  in new_curseg()
2811 				get_random_u32_inclusive(1, sbi->max_fragment_chunk);  in new_curseg()
2814 static int __next_free_blkoff(struct f2fs_sb_info *sbi,  in __next_free_blkoff()  argument
2817 	struct seg_entry *se = get_seg_entry(sbi, segno);  in __next_free_blkoff()
2819 	unsigned long *target_map = SIT_I(sbi)->tmp_map;  in __next_free_blkoff()
2827 	return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);  in __next_free_blkoff()
2830 static int f2fs_find_next_ssr_block(struct f2fs_sb_info *sbi,  in f2fs_find_next_ssr_block()  argument
2833 	return __next_free_blkoff(sbi, seg->segno, seg->next_blkoff + 1);  in f2fs_find_next_ssr_block()
2836 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)  in f2fs_segment_has_free_slot()  argument
2838 	return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg;  in f2fs_segment_has_free_slot()
2845 static void change_curseg(struct f2fs_sb_info *sbi, int type)  in change_curseg()  argument
2847 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in change_curseg()
2848 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in change_curseg()
2853 	write_sum_page(sbi, curseg->sum_blk, GET_SUM_BLOCK(sbi, curseg->segno));  in change_curseg()
2855 	__set_test_and_inuse(sbi, new_segno);  in change_curseg()
2858 	__remove_dirty_segment(sbi, new_segno, PRE);  in change_curseg()
2859 	__remove_dirty_segment(sbi, new_segno, DIRTY);  in change_curseg()
2862 	reset_curseg(sbi, type, 1);  in change_curseg()
2864 	curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);  in change_curseg()
2866 	sum_page = f2fs_get_sum_page(sbi, new_segno);  in change_curseg()
2877 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2880 static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,  in get_atssr_segment()  argument
2884 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in get_atssr_segment()
2888 	if (get_ssr_segment(sbi, type, alloc_mode, age)) {  in get_atssr_segment()
2889 		struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);  in get_atssr_segment()
2892 		change_curseg(sbi, type);  in get_atssr_segment()
2896 		new_curseg(sbi, type, true);  in get_atssr_segment()
2898 	stat_inc_seg_type(sbi, curseg);  in get_atssr_segment()
2901 static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)  in __f2fs_init_atgc_curseg()  argument
2903 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);  in __f2fs_init_atgc_curseg()
2905 	if (!sbi->am.atgc_enabled)  in __f2fs_init_atgc_curseg()
2908 	f2fs_down_read(&SM_I(sbi)->curseg_lock);  in __f2fs_init_atgc_curseg()
2911 	down_write(&SIT_I(sbi)->sentry_lock);  in __f2fs_init_atgc_curseg()
2913 	get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);  in __f2fs_init_atgc_curseg()
2915 	up_write(&SIT_I(sbi)->sentry_lock);  in __f2fs_init_atgc_curseg()
2918 	f2fs_up_read(&SM_I(sbi)->curseg_lock);  in __f2fs_init_atgc_curseg()
2921 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)  in f2fs_init_inmem_curseg()  argument
2923 	__f2fs_init_atgc_curseg(sbi);  in f2fs_init_inmem_curseg()
2926 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)  in __f2fs_save_inmem_curseg()  argument
2928 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __f2fs_save_inmem_curseg()
2934 	if (get_valid_blocks(sbi, curseg->segno, false)) {  in __f2fs_save_inmem_curseg()
2935 		write_sum_page(sbi, curseg->sum_blk,  in __f2fs_save_inmem_curseg()
2936 				GET_SUM_BLOCK(sbi, curseg->segno));  in __f2fs_save_inmem_curseg()
2938 		mutex_lock(&DIRTY_I(sbi)->seglist_lock);  in __f2fs_save_inmem_curseg()
2939 		__set_test_and_free(sbi, curseg->segno, true);  in __f2fs_save_inmem_curseg()
2940 		mutex_unlock(&DIRTY_I(sbi)->seglist_lock);  in __f2fs_save_inmem_curseg()
2946 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)  in f2fs_save_inmem_curseg()  argument
2948 	__f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);  in f2fs_save_inmem_curseg()
2950 	if (sbi->am.atgc_enabled)  in f2fs_save_inmem_curseg()
2951 		__f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);  in f2fs_save_inmem_curseg()
2954 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)  in __f2fs_restore_inmem_curseg()  argument
2956 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __f2fs_restore_inmem_curseg()
2961 	if (get_valid_blocks(sbi, curseg->segno, false))  in __f2fs_restore_inmem_curseg()
2964 	mutex_lock(&DIRTY_I(sbi)->seglist_lock);  in __f2fs_restore_inmem_curseg()
2965 	__set_test_and_inuse(sbi, curseg->segno);  in __f2fs_restore_inmem_curseg()
2966 	mutex_unlock(&DIRTY_I(sbi)->seglist_lock);  in __f2fs_restore_inmem_curseg()
2971 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)  in f2fs_restore_inmem_curseg()  argument
2973 	__f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);  in f2fs_restore_inmem_curseg()
2975 	if (sbi->am.atgc_enabled)  in f2fs_restore_inmem_curseg()
2976 		__f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);  in f2fs_restore_inmem_curseg()
2979 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,  in get_ssr_segment()  argument
2982 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in get_ssr_segment()
2988 	sanity_check_seg_type(sbi, seg_type);  in get_ssr_segment()
2991 	if (!f2fs_get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {  in get_ssr_segment()
3018 		if (!f2fs_get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {  in get_ssr_segment()
3025 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {  in get_ssr_segment()
3026 		segno = get_free_segment(sbi);  in get_ssr_segment()
3035 static bool need_new_seg(struct f2fs_sb_info *sbi, int type)  in need_new_seg()  argument
3037 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in need_new_seg()
3039 	if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&  in need_new_seg()
3043 	    is_next_segment_free(sbi, curseg, type) &&  in need_new_seg()
3044 	    likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))  in need_new_seg()
3046 	if (!f2fs_need_SSR(sbi) || !get_ssr_segment(sbi, type, SSR, 0))  in need_new_seg()
3051 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,  in f2fs_allocate_segment_for_resize()  argument
3054 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in f2fs_allocate_segment_for_resize()
3057 	f2fs_down_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_segment_for_resize()
3059 	down_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_segment_for_resize()
3061 	segno = CURSEG_I(sbi, type)->segno;  in f2fs_allocate_segment_for_resize()
3065 	if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))  in f2fs_allocate_segment_for_resize()
3066 		change_curseg(sbi, type);  in f2fs_allocate_segment_for_resize()
3068 		new_curseg(sbi, type, true);  in f2fs_allocate_segment_for_resize()
3070 	stat_inc_seg_type(sbi, curseg);  in f2fs_allocate_segment_for_resize()
3072 	locate_dirty_segment(sbi, segno);  in f2fs_allocate_segment_for_resize()
3074 	up_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_segment_for_resize()
3077 		f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",  in f2fs_allocate_segment_for_resize()
3081 	f2fs_up_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_segment_for_resize()
3084 static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type,  in __allocate_new_segment()  argument
3087 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __allocate_new_segment()
3092 	    !get_valid_blocks(sbi, curseg->segno, new_sec) &&  in __allocate_new_segment()
3093 	    !get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))  in __allocate_new_segment()
3097 	new_curseg(sbi, type, true);  in __allocate_new_segment()
3098 	stat_inc_seg_type(sbi, curseg);  in __allocate_new_segment()
3099 	locate_dirty_segment(sbi, old_segno);  in __allocate_new_segment()
3102 void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)  in f2fs_allocate_new_section()  argument
3104 	f2fs_down_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_new_section()
3105 	down_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_new_section()
3106 	__allocate_new_segment(sbi, type, true, force);  in f2fs_allocate_new_section()
3107 	up_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_new_section()
3108 	f2fs_up_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_new_section()
3111 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)  in f2fs_allocate_new_segments()  argument
3115 	f2fs_down_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_new_segments()
3116 	down_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_new_segments()
3118 		__allocate_new_segment(sbi, i, false, false);  in f2fs_allocate_new_segments()
3119 	up_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_new_segments()
3120 	f2fs_up_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_new_segments()
3123 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,  in f2fs_exist_trim_candidates()  argument
3129 	down_write(&SIT_I(sbi)->sentry_lock);  in f2fs_exist_trim_candidates()
3131 		if (add_discard_addrs(sbi, cpc, true)) {  in f2fs_exist_trim_candidates()
3136 	up_write(&SIT_I(sbi)->sentry_lock);  in f2fs_exist_trim_candidates()
3142 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,  in __issue_discard_cmd_range()  argument
3146 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd_range()
3159 		f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi));  in __issue_discard_cmd_range()
3180 		err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);  in __issue_discard_cmd_range()
3186 				__remove_discard_cmd(sbi, dc);  in __issue_discard_cmd_range()
3190 			trimmed += __wait_all_discard_cmd(sbi, NULL);  in __issue_discard_cmd_range()
3197 			__remove_discard_cmd(sbi, dc);  in __issue_discard_cmd_range()
3210 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)  in f2fs_trim_fs()  argument
3220 	bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);  in f2fs_trim_fs()
3222 	if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)  in f2fs_trim_fs()
3225 	if (end < MAIN_BLKADDR(sbi))  in f2fs_trim_fs()
3228 	if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {  in f2fs_trim_fs()
3229 		f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");  in f2fs_trim_fs()
3234 	start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);  in f2fs_trim_fs()
3235 	end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :  in f2fs_trim_fs()
3236 						GET_SEGNO(sbi, end);  in f2fs_trim_fs()
3238 		start_segno = rounddown(start_segno, sbi->segs_per_sec);  in f2fs_trim_fs()
3239 		end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;  in f2fs_trim_fs()
3247 	if (sbi->discard_blks == 0)  in f2fs_trim_fs()
3250 	f2fs_down_write(&sbi->gc_lock);  in f2fs_trim_fs()
3251 	stat_inc_cp_call_count(sbi, TOTAL_CALL);  in f2fs_trim_fs()
3252 	err = f2fs_write_checkpoint(sbi, &cpc);  in f2fs_trim_fs()
3253 	f2fs_up_write(&sbi->gc_lock);  in f2fs_trim_fs()
3263 	if (f2fs_realtime_discard_enable(sbi))  in f2fs_trim_fs()
3266 	start_block = START_BLOCK(sbi, start_segno);  in f2fs_trim_fs()
3267 	end_block = START_BLOCK(sbi, end_segno + 1);  in f2fs_trim_fs()
3269 	__init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);  in f2fs_trim_fs()
3270 	trimmed = __issue_discard_cmd_range(sbi, &dpolicy,  in f2fs_trim_fs()
3273 	trimmed += __wait_discard_cmd_range(sbi, &dpolicy,  in f2fs_trim_fs()
3320 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in __get_age_segment_type()  local
3326 		if (ei.age <= sbi->hot_data_age_threshold)  in __get_age_segment_type()
3328 		if (ei.age <= sbi->warm_data_age_threshold)  in __get_age_segment_type()
3345 			if (fio->sbi->am.atgc_enabled &&  in __get_segment_type_6()
3347 				(fio->sbi->gc_mode != GC_URGENT_HIGH))  in __get_segment_type_6()
3376 	switch (F2FS_OPTION(fio->sbi).active_logs) {  in __get_segment_type()
3387 		f2fs_bug_on(fio->sbi, true);  in __get_segment_type()
3399 static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi,  in f2fs_randomize_chunk()  argument
3407 		get_random_u32_inclusive(1, sbi->max_fragment_chunk);  in f2fs_randomize_chunk()
3409 		get_random_u32_inclusive(1, sbi->max_fragment_hole);  in f2fs_randomize_chunk()
3412 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,  in f2fs_allocate_data_block()  argument
3417 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_allocate_data_block()
3418 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in f2fs_allocate_data_block()
3424 	f2fs_down_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_data_block()
3430 		f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);  in f2fs_allocate_data_block()
3431 		se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));  in f2fs_allocate_data_block()
3432 		sanity_check_seg_type(sbi, se->type);  in f2fs_allocate_data_block()
3433 		f2fs_bug_on(sbi, IS_NODESEG(se->type));  in f2fs_allocate_data_block()
3435 	*new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);  in f2fs_allocate_data_block()
3437 	f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);  in f2fs_allocate_data_block()
3439 	f2fs_wait_discard_bio(sbi, *new_blkaddr);  in f2fs_allocate_data_block()
3443 		curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg);  in f2fs_allocate_data_block()
3446 		if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)  in f2fs_allocate_data_block()
3447 			f2fs_randomize_chunk(sbi, curseg);  in f2fs_allocate_data_block()
3449 	if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno))  in f2fs_allocate_data_block()
3451 	stat_inc_block_count(sbi, curseg);  in f2fs_allocate_data_block()
3454 		old_mtime = get_segment_mtime(sbi, old_blkaddr);  in f2fs_allocate_data_block()
3456 		update_segment_mtime(sbi, old_blkaddr, 0);  in f2fs_allocate_data_block()
3459 	update_segment_mtime(sbi, *new_blkaddr, old_mtime);  in f2fs_allocate_data_block()
3465 	update_sit_entry(sbi, *new_blkaddr, 1);  in f2fs_allocate_data_block()
3466 	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)  in f2fs_allocate_data_block()
3467 		update_sit_entry(sbi, old_blkaddr, -1);  in f2fs_allocate_data_block()
3475 			get_atssr_segment(sbi, type, se->type,  in f2fs_allocate_data_block()
3478 			if (need_new_seg(sbi, type))  in f2fs_allocate_data_block()
3479 				new_curseg(sbi, type, false);  in f2fs_allocate_data_block()
3481 				change_curseg(sbi, type);  in f2fs_allocate_data_block()
3482 			stat_inc_seg_type(sbi, curseg);  in f2fs_allocate_data_block()
3490 	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));  in f2fs_allocate_data_block()
3491 	locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));  in f2fs_allocate_data_block()
3494 		atomic64_inc(&sbi->allocated_data_blocks);  in f2fs_allocate_data_block()
3499 		fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));  in f2fs_allocate_data_block()
3501 		f2fs_inode_chksum_set(sbi, page);  in f2fs_allocate_data_block()
3507 		if (F2FS_IO_ALIGNED(sbi))  in f2fs_allocate_data_block()
3512 		io = sbi->write_io[fio->type] + fio->temp;  in f2fs_allocate_data_block()
3520 	f2fs_up_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_data_block()
3523 void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,  in f2fs_update_device_state()  argument
3526 	if (!f2fs_is_multi_device(sbi))  in f2fs_update_device_state()
3530 		unsigned int devidx = f2fs_target_device_index(sbi, blkaddr);  in f2fs_update_device_state()
3534 		f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO);  in f2fs_update_device_state()
3537 		if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {  in f2fs_update_device_state()
3538 			spin_lock(&sbi->dev_lock);  in f2fs_update_device_state()
3539 			f2fs_set_bit(devidx, (char *)&sbi->dirty_device);  in f2fs_update_device_state()
3540 			spin_unlock(&sbi->dev_lock);  in f2fs_update_device_state()
3553 	bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);  in do_write_page()
3556 		f2fs_down_read(&fio->sbi->io_order_lock);  in do_write_page()
3558 	f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,  in do_write_page()
3560 	if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) {  in do_write_page()
3561 		invalidate_mapping_pages(META_MAPPING(fio->sbi),  in do_write_page()
3563 		f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr);  in do_write_page()
3573 	f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1);  in do_write_page()
3576 		f2fs_up_read(&fio->sbi->io_order_lock);  in do_write_page()
3579 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,  in f2fs_do_write_meta_page()  argument
3583 		.sbi = sbi,  in f2fs_do_write_meta_page()
3595 	if (unlikely(page->index >= MAIN_BLKADDR(sbi)))  in f2fs_do_write_meta_page()
3601 	stat_inc_meta_count(sbi, page->index);  in f2fs_do_write_meta_page()
3602 	f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE);  in f2fs_do_write_meta_page()
3612 	f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE);  in f2fs_do_write_node_page()
3618 	struct f2fs_sb_info *sbi = fio->sbi;  in f2fs_outplace_write_data()  local
3621 	f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);  in f2fs_outplace_write_data()
3628 	f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE);  in f2fs_outplace_write_data()
3634 	struct f2fs_sb_info *sbi = fio->sbi;  in f2fs_inplace_write_data()  local
3641 	segno = GET_SEGNO(sbi, fio->new_blkaddr);  in f2fs_inplace_write_data()
3643 	if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {  in f2fs_inplace_write_data()
3644 		set_sbi_flag(sbi, SBI_NEED_FSCK);  in f2fs_inplace_write_data()
3645 		f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",  in f2fs_inplace_write_data()
3648 		f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);  in f2fs_inplace_write_data()
3652 	if (f2fs_cp_error(sbi)) {  in f2fs_inplace_write_data()
3658 		invalidate_mapping_pages(META_MAPPING(sbi),  in f2fs_inplace_write_data()
3661 	stat_inc_inplace_blocks(fio->sbi);  in f2fs_inplace_write_data()
3663 	if (fio->bio && !IS_F2FS_IPU_NOCACHE(sbi))  in f2fs_inplace_write_data()
3668 		f2fs_update_device_state(fio->sbi, fio->ino,  in f2fs_inplace_write_data()
3670 		f2fs_update_iostat(fio->sbi, fio->page->mapping->host,  in f2fs_inplace_write_data()
3686 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,  in __f2fs_get_curseg()  argument
3692 		if (CURSEG_I(sbi, i)->segno == segno)  in __f2fs_get_curseg()
3698 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,  in f2fs_do_replace_block()  argument
3703 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_do_replace_block()
3711 	segno = GET_SEGNO(sbi, new_blkaddr);  in f2fs_do_replace_block()
3712 	se = get_seg_entry(sbi, segno);  in f2fs_do_replace_block()
3715 	f2fs_down_write(&SM_I(sbi)->curseg_lock);  in f2fs_do_replace_block()
3719 		if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {  in f2fs_do_replace_block()
3726 		if (IS_CURSEG(sbi, segno)) {  in f2fs_do_replace_block()
3728 			type = __f2fs_get_curseg(sbi, segno);  in f2fs_do_replace_block()
3729 			f2fs_bug_on(sbi, type == NO_CHECK_TYPE);  in f2fs_do_replace_block()
3735 	f2fs_bug_on(sbi, !IS_DATASEG(type));  in f2fs_do_replace_block()
3736 	curseg = CURSEG_I(sbi, type);  in f2fs_do_replace_block()
3748 		change_curseg(sbi, type);  in f2fs_do_replace_block()
3751 	curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);  in f2fs_do_replace_block()
3756 			update_segment_mtime(sbi, new_blkaddr, 0);  in f2fs_do_replace_block()
3757 		update_sit_entry(sbi, new_blkaddr, 1);  in f2fs_do_replace_block()
3759 	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {  in f2fs_do_replace_block()
3760 		invalidate_mapping_pages(META_MAPPING(sbi),  in f2fs_do_replace_block()
3762 		f2fs_invalidate_compress_page(sbi, old_blkaddr);  in f2fs_do_replace_block()
3764 			update_segment_mtime(sbi, old_blkaddr, 0);  in f2fs_do_replace_block()
3765 		update_sit_entry(sbi, old_blkaddr, -1);  in f2fs_do_replace_block()
3768 	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));  in f2fs_do_replace_block()
3769 	locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));  in f2fs_do_replace_block()
3771 	locate_dirty_segment(sbi, old_cursegno);  in f2fs_do_replace_block()
3776 			change_curseg(sbi, type);  in f2fs_do_replace_block()
3784 	f2fs_up_write(&SM_I(sbi)->curseg_lock);  in f2fs_do_replace_block()
3787 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,  in f2fs_replace_block()  argument
3796 	f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,  in f2fs_replace_block()
3806 		struct f2fs_sb_info *sbi = F2FS_P_SB(page);  in f2fs_wait_on_page_writeback()  local
3809 		f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);  in f2fs_wait_on_page_writeback()
3811 		f2fs_submit_merged_ipu_write(sbi, NULL, page);  in f2fs_wait_on_page_writeback()
3814 			f2fs_bug_on(sbi, locked && PageWriteback(page));  in f2fs_wait_on_page_writeback()
3823 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_wait_on_block_writeback()  local
3832 	cpage = find_lock_page(META_MAPPING(sbi), blkaddr);  in f2fs_wait_on_block_writeback()
3842 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_wait_on_block_writeback_range()  local
3851 	invalidate_mapping_pages(META_MAPPING(sbi), blkaddr, blkaddr + len - 1);  in f2fs_wait_on_block_writeback_range()
3854 static int read_compacted_summaries(struct f2fs_sb_info *sbi)  in read_compacted_summaries()  argument
3856 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);  in read_compacted_summaries()
3863 	start = start_sum_block(sbi);  in read_compacted_summaries()
3865 	page = f2fs_get_meta_page(sbi, start++);  in read_compacted_summaries()
3871 	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);  in read_compacted_summaries()
3875 	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);  in read_compacted_summaries()
3884 		seg_i = CURSEG_I(sbi, i);  in read_compacted_summaries()
3888 		reset_curseg(sbi, i, 0);  in read_compacted_summaries()
3893 			blk_off = sbi->blocks_per_seg;  in read_compacted_summaries()
3908 			page = f2fs_get_meta_page(sbi, start++);  in read_compacted_summaries()
3919 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)  in read_normal_summaries()  argument
3921 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);  in read_normal_summaries()
3935 		if (__exist_node_summaries(sbi))  in read_normal_summaries()
3936 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);  in read_normal_summaries()
3938 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);  in read_normal_summaries()
3944 		if (__exist_node_summaries(sbi))  in read_normal_summaries()
3945 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,  in read_normal_summaries()
3948 			blk_addr = GET_SUM_BLOCK(sbi, segno);  in read_normal_summaries()
3951 	new = f2fs_get_meta_page(sbi, blk_addr);  in read_normal_summaries()
3957 		if (__exist_node_summaries(sbi)) {  in read_normal_summaries()
3961 			for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {  in read_normal_summaries()
3966 			err = f2fs_restore_node_summary(sbi, segno, sum);  in read_normal_summaries()
3973 	curseg = CURSEG_I(sbi, type);  in read_normal_summaries()
3984 	reset_curseg(sbi, type, 0);  in read_normal_summaries()
3993 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)  in restore_curseg_summaries()  argument
3995 	struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;  in restore_curseg_summaries()
3996 	struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;  in restore_curseg_summaries()
4000 	if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {  in restore_curseg_summaries()
4001 		int npages = f2fs_npages_for_summary_flush(sbi, true);  in restore_curseg_summaries()
4004 			f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,  in restore_curseg_summaries()
4008 		err = read_compacted_summaries(sbi);  in restore_curseg_summaries()
4014 	if (__exist_node_summaries(sbi))  in restore_curseg_summaries()
4015 		f2fs_ra_meta_pages(sbi,  in restore_curseg_summaries()
4016 				sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),  in restore_curseg_summaries()
4020 		err = read_normal_summaries(sbi, type);  in restore_curseg_summaries()
4028 		f2fs_err(sbi, "invalid journal entries nats %u sits %u",  in restore_curseg_summaries()
4036 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)  in write_compacted_summaries()  argument
4045 	page = f2fs_grab_meta_page(sbi, blkaddr++);  in write_compacted_summaries()
4050 	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);  in write_compacted_summaries()
4055 	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);  in write_compacted_summaries()
4061 		seg_i = CURSEG_I(sbi, i);  in write_compacted_summaries()
4062 		for (j = 0; j < f2fs_curseg_valid_blocks(sbi, i); j++) {  in write_compacted_summaries()
4064 				page = f2fs_grab_meta_page(sbi, blkaddr++);  in write_compacted_summaries()
4088 static void write_normal_summaries(struct f2fs_sb_info *sbi,  in write_normal_summaries()  argument
4099 		write_current_sum_page(sbi, i, blkaddr + (i - type));  in write_normal_summaries()
4102 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)  in f2fs_write_data_summaries()  argument
4104 	if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))  in f2fs_write_data_summaries()
4105 		write_compacted_summaries(sbi, start_blk);  in f2fs_write_data_summaries()
4107 		write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);  in f2fs_write_data_summaries()
4110 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)  in f2fs_write_node_summaries()  argument
4112 	write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);  in f2fs_write_node_summaries()
4137 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,  in get_current_sit_page()  argument
4140 	return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));  in get_current_sit_page()
4143 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,  in get_next_sit_page()  argument
4146 	struct sit_info *sit_i = SIT_I(sbi);  in get_next_sit_page()
4150 	src_off = current_sit_addr(sbi, start);  in get_next_sit_page()
4151 	dst_off = next_sit_addr(sbi, src_off);  in get_next_sit_page()
4153 	page = f2fs_grab_meta_page(sbi, dst_off);  in get_next_sit_page()
4154 	seg_info_to_sit_page(sbi, page, start);  in get_next_sit_page()
4216 static void add_sits_in_set(struct f2fs_sb_info *sbi)  in add_sits_in_set()  argument
4218 	struct f2fs_sm_info *sm_info = SM_I(sbi);  in add_sits_in_set()
4220 	unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;  in add_sits_in_set()
4223 	for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))  in add_sits_in_set()
4227 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)  in remove_sits_in_journal()  argument
4229 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);  in remove_sits_in_journal()
4239 		dirtied = __mark_sit_entry_dirty(sbi, segno);  in remove_sits_in_journal()
4242 			add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);  in remove_sits_in_journal()
4252 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)  in f2fs_flush_sit_entries()  argument
4254 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_flush_sit_entries()
4256 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);  in f2fs_flush_sit_entries()
4259 	struct list_head *head = &SM_I(sbi)->sit_entry_set;  in f2fs_flush_sit_entries()
4260 	bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);  in f2fs_flush_sit_entries()
4272 	add_sits_in_set(sbi);  in f2fs_flush_sit_entries()
4281 		remove_sits_in_journal(sbi);  in f2fs_flush_sit_entries()
4293 						(unsigned long)MAIN_SEGS(sbi));  in f2fs_flush_sit_entries()
4303 			page = get_next_sit_page(sbi, start_segno);  in f2fs_flush_sit_entries()
4311 			se = get_seg_entry(sbi, segno);  in f2fs_flush_sit_entries()
4315 				f2fs_bug_on(sbi, 1);  in f2fs_flush_sit_entries()
4321 				add_discard_addrs(sbi, cpc, false);  in f2fs_flush_sit_entries()
4327 				f2fs_bug_on(sbi, offset < 0);  in f2fs_flush_sit_entries()
4332 				check_block_count(sbi, segno,  in f2fs_flush_sit_entries()
4338 				check_block_count(sbi, segno,  in f2fs_flush_sit_entries()
4352 		f2fs_bug_on(sbi, ses->entry_cnt);  in f2fs_flush_sit_entries()
4356 	f2fs_bug_on(sbi, !list_empty(head));  in f2fs_flush_sit_entries()
4357 	f2fs_bug_on(sbi, sit_i->dirty_sentries);  in f2fs_flush_sit_entries()
4363 			add_discard_addrs(sbi, cpc, false);  in f2fs_flush_sit_entries()
4369 	set_prefree_as_free_segments(sbi);  in f2fs_flush_sit_entries()
4372 static int build_sit_info(struct f2fs_sb_info *sbi)  in build_sit_info()  argument
4374 	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);  in build_sit_info()
4379 	unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0;  in build_sit_info()
4382 	sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);  in build_sit_info()
4386 	SM_I(sbi)->sit_info = sit_i;  in build_sit_info()
4389 		f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),  in build_sit_info()
4390 					      MAIN_SEGS(sbi)),  in build_sit_info()
4395 	main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));  in build_sit_info()
4396 	sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,  in build_sit_info()
4402 	bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map);  in build_sit_info()
4404 	bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map);  in build_sit_info()
4406 	sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);  in build_sit_info()
4412 	for (start = 0; start < MAIN_SEGS(sbi); start++) {  in build_sit_info()
4430 	sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);  in build_sit_info()
4434 	if (__is_large_section(sbi)) {  in build_sit_info()
4436 			f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),  in build_sit_info()
4437 						      MAIN_SECS(sbi)),  in build_sit_info()
4447 	sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);  in build_sit_info()
4448 	src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);  in build_sit_info()
4460 	sit_i->invalid_segmap = f2fs_kvzalloc(sbi,  in build_sit_info()
4467 	sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;  in build_sit_info()
4472 	sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);  in build_sit_info()
4478 static int build_free_segmap(struct f2fs_sb_info *sbi)  in build_free_segmap()  argument
4484 	free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);  in build_free_segmap()
4488 	SM_I(sbi)->free_info = free_i;  in build_free_segmap()
4490 	bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));  in build_free_segmap()
4491 	free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);  in build_free_segmap()
4495 	sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));  in build_free_segmap()
4496 	free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);  in build_free_segmap()
4505 	free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));  in build_free_segmap()
4512 static int build_curseg(struct f2fs_sb_info *sbi)  in build_curseg()  argument
4517 	array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,  in build_curseg()
4522 	SM_I(sbi)->curseg_array = array;  in build_curseg()
4526 		array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);  in build_curseg()
4530 		array[i].journal = f2fs_kzalloc(sbi,  in build_curseg()
4544 	return restore_curseg_summaries(sbi);  in build_curseg()
4547 static int build_sit_entries(struct f2fs_sb_info *sbi)  in build_sit_entries()  argument
4549 	struct sit_info *sit_i = SIT_I(sbi);  in build_sit_entries()
4550 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);  in build_sit_entries()
4554 	int sit_blk_cnt = SIT_BLK_CNT(sbi);  in build_sit_entries()
4561 		readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS,  in build_sit_entries()
4567 		for (; start < end && start < MAIN_SEGS(sbi); start++) {  in build_sit_entries()
4572 			page = get_current_sit_page(sbi, start);  in build_sit_entries()
4579 			err = check_block_count(sbi, start, &sit);  in build_sit_entries()
4585 				f2fs_err(sbi, "Invalid segment type: %u, segno: %u",  in build_sit_entries()
4587 				f2fs_handle_error(sbi,  in build_sit_entries()
4594 			if (f2fs_block_unit_discard(sbi)) {  in build_sit_entries()
4596 				if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {  in build_sit_entries()
4603 					sbi->discard_blks +=  in build_sit_entries()
4604 						sbi->blocks_per_seg -  in build_sit_entries()
4609 			if (__is_large_section(sbi))  in build_sit_entries()
4610 				get_sec_entry(sbi, start)->valid_blocks +=  in build_sit_entries()
4621 		if (start >= MAIN_SEGS(sbi)) {  in build_sit_entries()
4622 			f2fs_err(sbi, "Wrong journal entry on segno %u",  in build_sit_entries()
4625 			f2fs_handle_error(sbi, ERROR_CORRUPTED_JOURNAL);  in build_sit_entries()
4636 		err = check_block_count(sbi, start, &sit);  in build_sit_entries()
4642 			f2fs_err(sbi, "Invalid segment type: %u, segno: %u",  in build_sit_entries()
4645 			f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);  in build_sit_entries()
4651 		if (f2fs_block_unit_discard(sbi)) {  in build_sit_entries()
4652 			if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {  in build_sit_entries()
4657 				sbi->discard_blks += old_valid_blocks;  in build_sit_entries()
4658 				sbi->discard_blks -= se->valid_blocks;  in build_sit_entries()
4662 		if (__is_large_section(sbi)) {  in build_sit_entries()
4663 			get_sec_entry(sbi, start)->valid_blocks +=  in build_sit_entries()
4665 			get_sec_entry(sbi, start)->valid_blocks -=  in build_sit_entries()
4674 	if (sit_valid_blocks[NODE] != valid_node_count(sbi)) {  in build_sit_entries()
4675 		f2fs_err(sbi, "SIT is corrupted node# %u vs %u",  in build_sit_entries()
4676 			 sit_valid_blocks[NODE], valid_node_count(sbi));  in build_sit_entries()
4677 		f2fs_handle_error(sbi, ERROR_INCONSISTENT_NODE_COUNT);  in build_sit_entries()
4682 				valid_user_blocks(sbi)) {  in build_sit_entries()
4683 		f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u",  in build_sit_entries()
4685 			 valid_user_blocks(sbi));  in build_sit_entries()
4686 		f2fs_handle_error(sbi, ERROR_INCONSISTENT_BLOCK_COUNT);  in build_sit_entries()
4693 static void init_free_segmap(struct f2fs_sb_info *sbi)  in init_free_segmap()  argument
4699 	for (start = 0; start < MAIN_SEGS(sbi); start++) {  in init_free_segmap()
4700 		if (f2fs_usable_blks_in_seg(sbi, start) == 0)  in init_free_segmap()
4702 		sentry = get_seg_entry(sbi, start);  in init_free_segmap()
4704 			__set_free(sbi, start);  in init_free_segmap()
4706 			SIT_I(sbi)->written_valid_blocks +=  in init_free_segmap()
4712 		struct curseg_info *curseg_t = CURSEG_I(sbi, type);  in init_free_segmap()
4714 		__set_test_and_inuse(sbi, curseg_t->segno);  in init_free_segmap()
4718 static void init_dirty_segmap(struct f2fs_sb_info *sbi)  in init_dirty_segmap()  argument
4720 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in init_dirty_segmap()
4721 	struct free_segmap_info *free_i = FREE_I(sbi);  in init_dirty_segmap()
4727 		segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);  in init_dirty_segmap()
4728 		if (segno >= MAIN_SEGS(sbi))  in init_dirty_segmap()
4731 		valid_blocks = get_valid_blocks(sbi, segno, false);  in init_dirty_segmap()
4732 		usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);  in init_dirty_segmap()
4736 			f2fs_bug_on(sbi, 1);  in init_dirty_segmap()
4740 		__locate_dirty_segment(sbi, segno, DIRTY);  in init_dirty_segmap()
4744 	if (!__is_large_section(sbi))  in init_dirty_segmap()
4748 	for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {  in init_dirty_segmap()
4749 		valid_blocks = get_valid_blocks(sbi, segno, true);  in init_dirty_segmap()
4750 		secno = GET_SEC_FROM_SEG(sbi, segno);  in init_dirty_segmap()
4752 		if (!valid_blocks || valid_blocks == CAP_BLKS_PER_SEC(sbi))  in init_dirty_segmap()
4754 		if (IS_CURSEC(sbi, secno))  in init_dirty_segmap()
4761 static int init_victim_secmap(struct f2fs_sb_info *sbi)  in init_victim_secmap()  argument
4763 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in init_victim_secmap()
4764 	unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));  in init_victim_secmap()
4766 	dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);  in init_victim_secmap()
4770 	dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);  in init_victim_secmap()
4779 static int build_dirty_segmap(struct f2fs_sb_info *sbi)  in build_dirty_segmap()  argument
4785 	dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),  in build_dirty_segmap()
4790 	SM_I(sbi)->dirty_info = dirty_i;  in build_dirty_segmap()
4793 	bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));  in build_dirty_segmap()
4796 		dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,  in build_dirty_segmap()
4802 	if (__is_large_section(sbi)) {  in build_dirty_segmap()
4803 		bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));  in build_dirty_segmap()
4804 		dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,  in build_dirty_segmap()
4810 	init_dirty_segmap(sbi);  in build_dirty_segmap()
4811 	return init_victim_secmap(sbi);  in build_dirty_segmap()
4814 static int sanity_check_curseg(struct f2fs_sb_info *sbi)  in sanity_check_curseg()  argument
4823 		struct curseg_info *curseg = CURSEG_I(sbi, i);  in sanity_check_curseg()
4824 		struct seg_entry *se = get_seg_entry(sbi, curseg->segno);  in sanity_check_curseg()
4827 		if (f2fs_sb_has_readonly(sbi) &&  in sanity_check_curseg()
4831 		sanity_check_seg_type(sbi, curseg->seg_type);  in sanity_check_curseg()
4834 			f2fs_err(sbi,  in sanity_check_curseg()
4837 			f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);  in sanity_check_curseg()
4847 		for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {  in sanity_check_curseg()
4851 			f2fs_err(sbi,  in sanity_check_curseg()
4855 			f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);  in sanity_check_curseg()
4864 static int check_zone_write_pointer(struct f2fs_sb_info *sbi,  in check_zone_write_pointer()  argument
4870 	unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;  in check_zone_write_pointer()
4878 	wp_segno = GET_SEGNO(sbi, wp_block);  in check_zone_write_pointer()
4879 	wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);  in check_zone_write_pointer()
4881 	zone_segno = GET_SEGNO(sbi, zone_block);  in check_zone_write_pointer()
4882 	zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);  in check_zone_write_pointer()
4884 	if (zone_segno >= MAIN_SEGS(sbi))  in check_zone_write_pointer()
4892 		if (zone_secno == GET_SEC_FROM_SEG(sbi,  in check_zone_write_pointer()
4893 						   CURSEG_I(sbi, i)->segno))  in check_zone_write_pointer()
4900 	for (s = sbi->segs_per_sec - 1; s >= 0; s--) {  in check_zone_write_pointer()
4902 		se = get_seg_entry(sbi, segno);  in check_zone_write_pointer()
4903 		for (b = sbi->blocks_per_seg - 1; b >= 0; b--)  in check_zone_write_pointer()
4905 				last_valid_block = START_BLOCK(sbi, segno) + b;  in check_zone_write_pointer()
4925 		f2fs_notice(sbi,  in check_zone_write_pointer()
4929 		ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,  in check_zone_write_pointer()
4932 			f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",  in check_zone_write_pointer()
4945 	f2fs_notice(sbi, "Valid blocks are not aligned with write pointer: "  in check_zone_write_pointer()
4947 		    GET_SEGNO(sbi, last_valid_block),  in check_zone_write_pointer()
4948 		    GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),  in check_zone_write_pointer()
4958 			f2fs_err(sbi, "Fill up zone failed: %s (errno=%d)",  in check_zone_write_pointer()
4961 		f2fs_err(sbi, "Finishing zone failed: %s (errno=%d)",  in check_zone_write_pointer()
4968 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,  in get_target_zoned_dev()  argument
4973 	for (i = 0; i < sbi->s_ndevs; i++) {  in get_target_zoned_dev()
4976 		if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&  in get_target_zoned_dev()
4991 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)  in fix_curseg_write_pointer()  argument
4993 	struct curseg_info *cs = CURSEG_I(sbi, type);  in fix_curseg_write_pointer()
4998 	unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;  in fix_curseg_write_pointer()
5002 	cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);  in fix_curseg_write_pointer()
5003 	cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));  in fix_curseg_write_pointer()
5005 	zbd = get_target_zoned_dev(sbi, cs_zone_block);  in fix_curseg_write_pointer()
5015 		f2fs_err(sbi, "Report zone failed: %s errno=(%d)",  in fix_curseg_write_pointer()
5024 	wp_segno = GET_SEGNO(sbi, wp_block);  in fix_curseg_write_pointer()
5025 	wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);  in fix_curseg_write_pointer()
5032 	f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "  in fix_curseg_write_pointer()
5036 	f2fs_notice(sbi, "Assign new section to curseg[%d]: "  in fix_curseg_write_pointer()
5039 	f2fs_allocate_new_section(sbi, type, true);  in fix_curseg_write_pointer()
5042 	if (check_zone_write_pointer(sbi, zbd, &zone))  in fix_curseg_write_pointer()
5046 	cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);  in fix_curseg_write_pointer()
5047 	cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));  in fix_curseg_write_pointer()
5049 	zbd = get_target_zoned_dev(sbi, cs_zone_block);  in fix_curseg_write_pointer()
5058 		f2fs_err(sbi, "Report zone failed: %s errno=(%d)",  in fix_curseg_write_pointer()
5067 		f2fs_notice(sbi,  in fix_curseg_write_pointer()
5071 		err = __f2fs_issue_discard_zone(sbi, zbd->bdev,	cs_zone_block,  in fix_curseg_write_pointer()
5074 			f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",  in fix_curseg_write_pointer()
5083 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)  in f2fs_fix_curseg_write_pointer()  argument
5088 		ret = fix_curseg_write_pointer(sbi, i);  in f2fs_fix_curseg_write_pointer()
5097 	struct f2fs_sb_info *sbi;  member
5108 	return check_zone_write_pointer(args->sbi, args->fdev, zone);  in check_zone_write_pointer_cb()
5111 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)  in f2fs_check_write_pointer()  argument
5116 	for (i = 0; i < sbi->s_ndevs; i++) {  in f2fs_check_write_pointer()
5120 		args.sbi = sbi;  in f2fs_check_write_pointer()
5140 			struct f2fs_sb_info *sbi, unsigned int segno)  in f2fs_usable_zone_blks_in_seg()  argument
5145 	if (!sbi->unusable_blocks_per_sec)  in f2fs_usable_zone_blks_in_seg()
5146 		return sbi->blocks_per_seg;  in f2fs_usable_zone_blks_in_seg()
5148 	secno = GET_SEC_FROM_SEG(sbi, segno);  in f2fs_usable_zone_blks_in_seg()
5149 	seg_start = START_BLOCK(sbi, segno);  in f2fs_usable_zone_blks_in_seg()
5150 	sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));  in f2fs_usable_zone_blks_in_seg()
5151 	sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi);  in f2fs_usable_zone_blks_in_seg()
5161 	if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)  in f2fs_usable_zone_blks_in_seg()
5164 	return sbi->blocks_per_seg;  in f2fs_usable_zone_blks_in_seg()
5167 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)  in f2fs_fix_curseg_write_pointer()  argument
5172 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)  in f2fs_check_write_pointer()  argument
5177 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,  in f2fs_usable_zone_blks_in_seg()  argument
5184 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,  in f2fs_usable_blks_in_seg()  argument
5187 	if (f2fs_sb_has_blkzoned(sbi))  in f2fs_usable_blks_in_seg()
5188 		return f2fs_usable_zone_blks_in_seg(sbi, segno);  in f2fs_usable_blks_in_seg()
5190 	return sbi->blocks_per_seg;  in f2fs_usable_blks_in_seg()
5193 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,  in f2fs_usable_segs_in_sec()  argument
5196 	if (f2fs_sb_has_blkzoned(sbi))  in f2fs_usable_segs_in_sec()
5197 		return CAP_SEGS_PER_SEC(sbi);  in f2fs_usable_segs_in_sec()
5199 	return sbi->segs_per_sec;  in f2fs_usable_segs_in_sec()
5205 static void init_min_max_mtime(struct f2fs_sb_info *sbi)  in init_min_max_mtime()  argument
5207 	struct sit_info *sit_i = SIT_I(sbi);  in init_min_max_mtime()
5214 	for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {  in init_min_max_mtime()
5218 		for (i = 0; i < sbi->segs_per_sec; i++)  in init_min_max_mtime()
5219 			mtime += get_seg_entry(sbi, segno + i)->mtime;  in init_min_max_mtime()
5221 		mtime = div_u64(mtime, sbi->segs_per_sec);  in init_min_max_mtime()
5226 	sit_i->max_mtime = get_mtime(sbi, false);  in init_min_max_mtime()
5231 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)  in f2fs_build_segment_manager()  argument
5233 	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);  in f2fs_build_segment_manager()
5234 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);  in f2fs_build_segment_manager()
5238 	sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);  in f2fs_build_segment_manager()
5243 	sbi->sm_info = sm_info;  in f2fs_build_segment_manager()
5256 	if (!f2fs_lfs_mode(sbi))  in f2fs_build_segment_manager()
5260 	sm_info->min_seq_blocks = sbi->blocks_per_seg;  in f2fs_build_segment_manager()
5262 	sm_info->min_ssr_sections = reserved_sections(sbi);  in f2fs_build_segment_manager()
5268 	err = f2fs_create_flush_cmd_control(sbi);  in f2fs_build_segment_manager()
5272 	err = create_discard_cmd_control(sbi);  in f2fs_build_segment_manager()
5276 	err = build_sit_info(sbi);  in f2fs_build_segment_manager()
5279 	err = build_free_segmap(sbi);  in f2fs_build_segment_manager()
5282 	err = build_curseg(sbi);  in f2fs_build_segment_manager()
5287 	err = build_sit_entries(sbi);  in f2fs_build_segment_manager()
5291 	init_free_segmap(sbi);  in f2fs_build_segment_manager()
5292 	err = build_dirty_segmap(sbi);  in f2fs_build_segment_manager()
5296 	err = sanity_check_curseg(sbi);  in f2fs_build_segment_manager()
5300 	init_min_max_mtime(sbi);  in f2fs_build_segment_manager()
5304 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,  in discard_dirty_segmap()  argument
5307 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in discard_dirty_segmap()
5315 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)  in destroy_victim_secmap()  argument
5317 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in destroy_victim_secmap()
5323 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)  in destroy_dirty_segmap()  argument
5325 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in destroy_dirty_segmap()
5333 		discard_dirty_segmap(sbi, i);  in destroy_dirty_segmap()
5335 	if (__is_large_section(sbi)) {  in destroy_dirty_segmap()
5341 	destroy_victim_secmap(sbi);  in destroy_dirty_segmap()
5342 	SM_I(sbi)->dirty_info = NULL;  in destroy_dirty_segmap()
5346 static void destroy_curseg(struct f2fs_sb_info *sbi)  in destroy_curseg()  argument
5348 	struct curseg_info *array = SM_I(sbi)->curseg_array;  in destroy_curseg()
5353 	SM_I(sbi)->curseg_array = NULL;  in destroy_curseg()
5361 static void destroy_free_segmap(struct f2fs_sb_info *sbi)  in destroy_free_segmap()  argument
5363 	struct free_segmap_info *free_i = SM_I(sbi)->free_info;  in destroy_free_segmap()
5367 	SM_I(sbi)->free_info = NULL;  in destroy_free_segmap()
5373 static void destroy_sit_info(struct f2fs_sb_info *sbi)  in destroy_sit_info()  argument
5375 	struct sit_info *sit_i = SIT_I(sbi);  in destroy_sit_info()
5388 	SM_I(sbi)->sit_info = NULL;  in destroy_sit_info()
5397 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)  in f2fs_destroy_segment_manager()  argument
5399 	struct f2fs_sm_info *sm_info = SM_I(sbi);  in f2fs_destroy_segment_manager()
5403 	f2fs_destroy_flush_cmd_control(sbi, true);  in f2fs_destroy_segment_manager()
5404 	destroy_discard_cmd_control(sbi);  in f2fs_destroy_segment_manager()
5405 	destroy_dirty_segmap(sbi);  in f2fs_destroy_segment_manager()
5406 	destroy_curseg(sbi);  in f2fs_destroy_segment_manager()
5407 	destroy_free_segmap(sbi);  in f2fs_destroy_segment_manager()
5408 	destroy_sit_info(sbi);  in f2fs_destroy_segment_manager()
5409 	sbi->sm_info = NULL;  in f2fs_destroy_segment_manager()