65 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
69 int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info)  in btrfs_init_dev_replace()  argument
72 	struct btrfs_root *dev_root = fs_info->dev_root;  in btrfs_init_dev_replace()
73 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_init_dev_replace()
98 		if (btrfs_find_device(fs_info->fs_devices,  in btrfs_init_dev_replace()
100 			btrfs_err(fs_info,  in btrfs_init_dev_replace()
130 		btrfs_warn(fs_info,  in btrfs_init_dev_replace()
161 		if (btrfs_find_device(fs_info->fs_devices,  in btrfs_init_dev_replace()
163 			btrfs_err(fs_info,  in btrfs_init_dev_replace()
173 		dev_replace->srcdev = btrfs_find_device(fs_info->fs_devices,  in btrfs_init_dev_replace()
175 		dev_replace->tgtdev = btrfs_find_device(fs_info->fs_devices,  in btrfs_init_dev_replace()
183 		    !btrfs_test_opt(fs_info, DEGRADED)) {  in btrfs_init_dev_replace()
185 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
187 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
192 		    !btrfs_test_opt(fs_info, DEGRADED)) {  in btrfs_init_dev_replace()
194 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
196 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
216 			WARN_ON(fs_info->fs_devices->rw_devices == 0);  in btrfs_init_dev_replace()
217 			dev_replace->tgtdev->io_width = fs_info->sectorsize;  in btrfs_init_dev_replace()
218 			dev_replace->tgtdev->io_align = fs_info->sectorsize;  in btrfs_init_dev_replace()
219 			dev_replace->tgtdev->sector_size = fs_info->sectorsize;  in btrfs_init_dev_replace()
220 			dev_replace->tgtdev->fs_info = fs_info;  in btrfs_init_dev_replace()
238 static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,  in btrfs_init_dev_replace_tgtdev()  argument
251 		btrfs_err(fs_info, "the filesystem is a seed filesystem!");  in btrfs_init_dev_replace_tgtdev()
256 				  fs_info->bdev_holder);  in btrfs_init_dev_replace_tgtdev()
258 		btrfs_err(fs_info, "target device %s is invalid!", device_path);  in btrfs_init_dev_replace_tgtdev()
264 	list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) {  in btrfs_init_dev_replace_tgtdev()
266 			btrfs_err(fs_info,  in btrfs_init_dev_replace_tgtdev()
276 		btrfs_err(fs_info,  in btrfs_init_dev_replace_tgtdev()
299 	device->io_width = fs_info->sectorsize;  in btrfs_init_dev_replace_tgtdev()
300 	device->io_align = fs_info->sectorsize;  in btrfs_init_dev_replace_tgtdev()
301 	device->sector_size = fs_info->sectorsize;  in btrfs_init_dev_replace_tgtdev()
307 	device->fs_info = fs_info;  in btrfs_init_dev_replace_tgtdev()
314 	device->fs_devices = fs_info->fs_devices;  in btrfs_init_dev_replace_tgtdev()
316 	mutex_lock(&fs_info->fs_devices->device_list_mutex);  in btrfs_init_dev_replace_tgtdev()
317 	list_add(&device->dev_list, &fs_info->fs_devices->devices);  in btrfs_init_dev_replace_tgtdev()
318 	fs_info->fs_devices->num_devices++;  in btrfs_init_dev_replace_tgtdev()
319 	fs_info->fs_devices->open_devices++;  in btrfs_init_dev_replace_tgtdev()
320 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_init_dev_replace_tgtdev()
336 	struct btrfs_fs_info *fs_info = trans->fs_info;  in btrfs_run_dev_replace()  local
338 	struct btrfs_root *dev_root = fs_info->dev_root;  in btrfs_run_dev_replace()
343 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_run_dev_replace()
364 		btrfs_warn(fs_info,  in btrfs_run_dev_replace()
385 			btrfs_warn(fs_info,  in btrfs_run_dev_replace()
399 			btrfs_warn(fs_info,  in btrfs_run_dev_replace()
450 static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_start()  argument
454 	struct btrfs_root *root = fs_info->dev_root;  in btrfs_dev_replace_start()
456 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_start()
461 	src_device = btrfs_find_device_by_devspec(fs_info, srcdevid,  in btrfs_dev_replace_start()
466 	if (btrfs_pinned_by_swapfile(fs_info, src_device)) {  in btrfs_dev_replace_start()
467 		btrfs_warn_in_rcu(fs_info,  in btrfs_dev_replace_start()
486 	ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name,  in btrfs_dev_replace_start()
509 	btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_start()
533 		btrfs_err(fs_info, "kobj add dev failed %d", ret);  in btrfs_dev_replace_start()
535 	btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);  in btrfs_dev_replace_start()
554 	ret = btrfs_scrub_dev(fs_info, src_device->devid, 0,  in btrfs_dev_replace_start()
558 	ret = btrfs_dev_replace_finishing(fs_info, ret);  in btrfs_dev_replace_start()
569 int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_by_ioctl()  argument
586 	ret = btrfs_dev_replace_start(fs_info, args->start.tgtdev_name,  in btrfs_dev_replace_by_ioctl()
602 static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info)  in btrfs_rm_dev_replace_blocked()  argument
604 	set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);  in btrfs_rm_dev_replace_blocked()
605 	wait_event(fs_info->dev_replace.replace_wait, !percpu_counter_sum(  in btrfs_rm_dev_replace_blocked()
606 		   &fs_info->dev_replace.bio_counter));  in btrfs_rm_dev_replace_blocked()
612 static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info)  in btrfs_rm_dev_replace_unblocked()  argument
614 	clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);  in btrfs_rm_dev_replace_unblocked()
615 	wake_up(&fs_info->dev_replace.replace_wait);  in btrfs_rm_dev_replace_unblocked()
633 	lockdep_assert_held(&srcdev->fs_info->chunk_mutex);  in btrfs_set_target_alloc_state()
650 						struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_update_device_in_mapping_tree()  argument
654 	struct extent_map_tree *em_tree = &fs_info->mapping_tree;  in btrfs_dev_replace_update_device_in_mapping_tree()
675 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_finishing()  argument
678 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_finishing()
681 	struct btrfs_root *root = fs_info->tree_root;  in btrfs_dev_replace_finishing()
706 	ret = btrfs_start_delalloc_roots(fs_info, U64_MAX, false);  in btrfs_dev_replace_finishing()
711 	btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);  in btrfs_dev_replace_finishing()
732 		mutex_lock(&fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
734 		mutex_lock(&fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
737 			mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
738 			mutex_unlock(&fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
761 		btrfs_dev_replace_update_device_in_mapping_tree(fs_info,  in btrfs_dev_replace_finishing()
766 			btrfs_err_in_rcu(fs_info,  in btrfs_dev_replace_finishing()
773 		mutex_unlock(&fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
774 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
776 		btrfs_rm_dev_replace_blocked(fs_info);  in btrfs_dev_replace_finishing()
779 		btrfs_rm_dev_replace_unblocked(fs_info);  in btrfs_dev_replace_finishing()
785 	btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_finishing()
804 	list_add(&tgt_device->dev_alloc_list, &fs_info->fs_devices->alloc_list);  in btrfs_dev_replace_finishing()
805 	fs_info->fs_devices->rw_devices++;  in btrfs_dev_replace_finishing()
808 	btrfs_rm_dev_replace_blocked(fs_info);  in btrfs_dev_replace_finishing()
812 	btrfs_rm_dev_replace_unblocked(fs_info);  in btrfs_dev_replace_finishing()
827 	mutex_unlock(&fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
828 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
834 		btrfs_scratch_superblocks(fs_info, src_device->bdev,  in btrfs_dev_replace_finishing()
854 static u64 btrfs_dev_replace_progress(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_progress()  argument
856 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_progress()
878 void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_status()  argument
881 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_status()
894 	args->status.progress_1000 = btrfs_dev_replace_progress(fs_info);  in btrfs_dev_replace_status()
898 int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_cancel()  argument
900 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_cancel()
904 	struct btrfs_root *root = fs_info->tree_root;  in btrfs_dev_replace_cancel()
908 	if (sb_rdonly(fs_info->sb))  in btrfs_dev_replace_cancel()
924 		ret = btrfs_scrub_cancel(fs_info);  in btrfs_dev_replace_cancel()
933 			btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_cancel()
957 		ret = btrfs_scrub_cancel(fs_info);  in btrfs_dev_replace_cancel()
968 		btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_cancel()
985 void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_suspend_for_unmount()  argument
987 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_suspend_for_unmount()
1003 		btrfs_info(fs_info, "suspending dev_replace for unmount");  in btrfs_dev_replace_suspend_for_unmount()
1012 int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)  in btrfs_resume_dev_replace_async()  argument
1015 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_resume_dev_replace_async()
1033 		btrfs_info(fs_info,  in btrfs_resume_dev_replace_async()
1035 		btrfs_info(fs_info,  in btrfs_resume_dev_replace_async()
1049 	if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) {  in btrfs_resume_dev_replace_async()
1054 		btrfs_info(fs_info,  in btrfs_resume_dev_replace_async()
1059 	task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl");  in btrfs_resume_dev_replace_async()
1065 	struct btrfs_fs_info *fs_info = data;  in btrfs_dev_replace_kthread()  local
1066 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_kthread()
1070 	progress = btrfs_dev_replace_progress(fs_info);  in btrfs_dev_replace_kthread()
1072 	btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_kthread()
1079 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,  in btrfs_dev_replace_kthread()
1083 	ret = btrfs_dev_replace_finishing(fs_info, ret);  in btrfs_dev_replace_kthread()
1086 	btrfs_exclop_finish(fs_info);  in btrfs_dev_replace_kthread()
1117 void btrfs_bio_counter_inc_noblocked(struct btrfs_fs_info *fs_info)  in btrfs_bio_counter_inc_noblocked()  argument
1119 	percpu_counter_inc(&fs_info->dev_replace.bio_counter);  in btrfs_bio_counter_inc_noblocked()
1122 void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount)  in btrfs_bio_counter_sub()  argument
1124 	percpu_counter_sub(&fs_info->dev_replace.bio_counter, amount);  in btrfs_bio_counter_sub()
1125 	cond_wake_up_nomb(&fs_info->dev_replace.replace_wait);  in btrfs_bio_counter_sub()
1128 void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info)  in btrfs_bio_counter_inc_blocked()  argument
1131 		percpu_counter_inc(&fs_info->dev_replace.bio_counter);  in btrfs_bio_counter_inc_blocked()
1133 				     &fs_info->fs_state)))  in btrfs_bio_counter_inc_blocked()
1136 		btrfs_bio_counter_dec(fs_info);  in btrfs_bio_counter_inc_blocked()
1137 		wait_event(fs_info->dev_replace.replace_wait,  in btrfs_bio_counter_inc_blocked()
1139 				     &fs_info->fs_state));  in btrfs_bio_counter_inc_blocked()