• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) STRATO AG 2012.  All rights reserved.
4  */
5 
6 #include <linux/sched.h>
7 #include <linux/bio.h>
8 #include <linux/slab.h>
9 #include <linux/blkdev.h>
10 #include <linux/kthread.h>
11 #include <linux/math64.h>
12 #include "misc.h"
13 #include "ctree.h"
14 #include "extent_map.h"
15 #include "disk-io.h"
16 #include "transaction.h"
17 #include "print-tree.h"
18 #include "volumes.h"
19 #include "async-thread.h"
20 #include "check-integrity.h"
21 #include "rcu-string.h"
22 #include "dev-replace.h"
23 #include "sysfs.h"
24 
25 /*
26  * Device replace overview
27  *
28  * [Objective]
29  * To copy all extents (both new and on-disk) from source device to target
30  * device, while still keeping the filesystem read-write.
31  *
32  * [Method]
33  * There are two main methods involved:
34  *
35  * - Write duplication
36  *
37  *   All new writes will be written to both target and source devices, so even
38  *   if replace gets canceled, sources device still contans up-to-date data.
39  *
40  *   Location:		handle_ops_on_dev_replace() from __btrfs_map_block()
41  *   Start:		btrfs_dev_replace_start()
42  *   End:		btrfs_dev_replace_finishing()
43  *   Content:		Latest data/metadata
44  *
45  * - Copy existing extents
46  *
47  *   This happens by re-using scrub facility, as scrub also iterates through
48  *   existing extents from commit root.
49  *
50  *   Location:		scrub_write_block_to_dev_replace() from
51  *   			scrub_block_complete()
52  *   Content:		Data/meta from commit root.
53  *
54  * Due to the content difference, we need to avoid nocow write when dev-replace
55  * is happening.  This is done by marking the block group read-only and waiting
56  * for NOCOW writes.
57  *
58  * After replace is done, the finishing part is done by swapping the target and
59  * source devices.
60  *
61  *   Location:		btrfs_dev_replace_update_device_in_mapping_tree() from
62  *   			btrfs_dev_replace_finishing()
63  */
64 
65 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
66 				       int scrub_ret);
67 static int btrfs_dev_replace_kthread(void *data);
68 
btrfs_init_dev_replace(struct btrfs_fs_info * fs_info)69 int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info)
70 {
71 	struct btrfs_key key;
72 	struct btrfs_root *dev_root = fs_info->dev_root;
73 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
74 	struct extent_buffer *eb;
75 	int slot;
76 	int ret = 0;
77 	struct btrfs_path *path = NULL;
78 	int item_size;
79 	struct btrfs_dev_replace_item *ptr;
80 	u64 src_devid;
81 
82 	path = btrfs_alloc_path();
83 	if (!path) {
84 		ret = -ENOMEM;
85 		goto out;
86 	}
87 
88 	key.objectid = 0;
89 	key.type = BTRFS_DEV_REPLACE_KEY;
90 	key.offset = 0;
91 	ret = btrfs_search_slot(NULL, dev_root, &key, path, 0, 0);
92 	if (ret) {
93 no_valid_dev_replace_entry_found:
94 		/*
95 		 * We don't have a replace item or it's corrupted.  If there is
96 		 * a replace target, fail the mount.
97 		 */
98 		if (btrfs_find_device(fs_info->fs_devices,
99 				      BTRFS_DEV_REPLACE_DEVID, NULL, NULL, false)) {
100 			btrfs_err(fs_info,
101 			"found replace target device without a valid replace item");
102 			ret = -EUCLEAN;
103 			goto out;
104 		}
105 		ret = 0;
106 		dev_replace->replace_state =
107 			BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED;
108 		dev_replace->cont_reading_from_srcdev_mode =
109 		    BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_ALWAYS;
110 		dev_replace->time_started = 0;
111 		dev_replace->time_stopped = 0;
112 		atomic64_set(&dev_replace->num_write_errors, 0);
113 		atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
114 		dev_replace->cursor_left = 0;
115 		dev_replace->committed_cursor_left = 0;
116 		dev_replace->cursor_left_last_write_of_item = 0;
117 		dev_replace->cursor_right = 0;
118 		dev_replace->srcdev = NULL;
119 		dev_replace->tgtdev = NULL;
120 		dev_replace->is_valid = 0;
121 		dev_replace->item_needs_writeback = 0;
122 		goto out;
123 	}
124 	slot = path->slots[0];
125 	eb = path->nodes[0];
126 	item_size = btrfs_item_size_nr(eb, slot);
127 	ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_replace_item);
128 
129 	if (item_size != sizeof(struct btrfs_dev_replace_item)) {
130 		btrfs_warn(fs_info,
131 			"dev_replace entry found has unexpected size, ignore entry");
132 		goto no_valid_dev_replace_entry_found;
133 	}
134 
135 	src_devid = btrfs_dev_replace_src_devid(eb, ptr);
136 	dev_replace->cont_reading_from_srcdev_mode =
137 		btrfs_dev_replace_cont_reading_from_srcdev_mode(eb, ptr);
138 	dev_replace->replace_state = btrfs_dev_replace_replace_state(eb, ptr);
139 	dev_replace->time_started = btrfs_dev_replace_time_started(eb, ptr);
140 	dev_replace->time_stopped =
141 		btrfs_dev_replace_time_stopped(eb, ptr);
142 	atomic64_set(&dev_replace->num_write_errors,
143 		     btrfs_dev_replace_num_write_errors(eb, ptr));
144 	atomic64_set(&dev_replace->num_uncorrectable_read_errors,
145 		     btrfs_dev_replace_num_uncorrectable_read_errors(eb, ptr));
146 	dev_replace->cursor_left = btrfs_dev_replace_cursor_left(eb, ptr);
147 	dev_replace->committed_cursor_left = dev_replace->cursor_left;
148 	dev_replace->cursor_left_last_write_of_item = dev_replace->cursor_left;
149 	dev_replace->cursor_right = btrfs_dev_replace_cursor_right(eb, ptr);
150 	dev_replace->is_valid = 1;
151 
152 	dev_replace->item_needs_writeback = 0;
153 	switch (dev_replace->replace_state) {
154 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
155 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
156 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
157 		/*
158 		 * We don't have an active replace item but if there is a
159 		 * replace target, fail the mount.
160 		 */
161 		if (btrfs_find_device(fs_info->fs_devices,
162 				      BTRFS_DEV_REPLACE_DEVID, NULL, NULL, false)) {
163 			btrfs_err(fs_info,
164 "replace without active item, run 'device scan --forget' on the target device");
165 			ret = -EUCLEAN;
166 		} else {
167 			dev_replace->srcdev = NULL;
168 			dev_replace->tgtdev = NULL;
169 		}
170 		break;
171 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
172 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
173 		dev_replace->srcdev = btrfs_find_device(fs_info->fs_devices,
174 						src_devid, NULL, NULL, true);
175 		dev_replace->tgtdev = btrfs_find_device(fs_info->fs_devices,
176 							BTRFS_DEV_REPLACE_DEVID,
177 							NULL, NULL, true);
178 		/*
179 		 * allow 'btrfs dev replace_cancel' if src/tgt device is
180 		 * missing
181 		 */
182 		if (!dev_replace->srcdev &&
183 		    !btrfs_test_opt(fs_info, DEGRADED)) {
184 			ret = -EIO;
185 			btrfs_warn(fs_info,
186 			   "cannot mount because device replace operation is ongoing and");
187 			btrfs_warn(fs_info,
188 			   "srcdev (devid %llu) is missing, need to run 'btrfs dev scan'?",
189 			   src_devid);
190 		}
191 		if (!dev_replace->tgtdev &&
192 		    !btrfs_test_opt(fs_info, DEGRADED)) {
193 			ret = -EIO;
194 			btrfs_warn(fs_info,
195 			   "cannot mount because device replace operation is ongoing and");
196 			btrfs_warn(fs_info,
197 			   "tgtdev (devid %llu) is missing, need to run 'btrfs dev scan'?",
198 				BTRFS_DEV_REPLACE_DEVID);
199 		}
200 		if (dev_replace->tgtdev) {
201 			if (dev_replace->srcdev) {
202 				dev_replace->tgtdev->total_bytes =
203 					dev_replace->srcdev->total_bytes;
204 				dev_replace->tgtdev->disk_total_bytes =
205 					dev_replace->srcdev->disk_total_bytes;
206 				dev_replace->tgtdev->commit_total_bytes =
207 					dev_replace->srcdev->commit_total_bytes;
208 				dev_replace->tgtdev->bytes_used =
209 					dev_replace->srcdev->bytes_used;
210 				dev_replace->tgtdev->commit_bytes_used =
211 					dev_replace->srcdev->commit_bytes_used;
212 			}
213 			set_bit(BTRFS_DEV_STATE_REPLACE_TGT,
214 				&dev_replace->tgtdev->dev_state);
215 
216 			WARN_ON(fs_info->fs_devices->rw_devices == 0);
217 			dev_replace->tgtdev->io_width = fs_info->sectorsize;
218 			dev_replace->tgtdev->io_align = fs_info->sectorsize;
219 			dev_replace->tgtdev->sector_size = fs_info->sectorsize;
220 			dev_replace->tgtdev->fs_info = fs_info;
221 			set_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
222 				&dev_replace->tgtdev->dev_state);
223 		}
224 		break;
225 	}
226 
227 out:
228 	btrfs_free_path(path);
229 	return ret;
230 }
231 
232 /*
233  * Initialize a new device for device replace target from a given source dev
234  * and path.
235  *
236  * Return 0 and new device in @device_out, otherwise return < 0
237  */
btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info * fs_info,const char * device_path,struct btrfs_device * srcdev,struct btrfs_device ** device_out)238 static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
239 				  const char *device_path,
240 				  struct btrfs_device *srcdev,
241 				  struct btrfs_device **device_out)
242 {
243 	struct btrfs_device *device;
244 	struct block_device *bdev;
245 	struct rcu_string *name;
246 	u64 devid = BTRFS_DEV_REPLACE_DEVID;
247 	int ret = 0;
248 
249 	*device_out = NULL;
250 	if (srcdev->fs_devices->seeding) {
251 		btrfs_err(fs_info, "the filesystem is a seed filesystem!");
252 		return -EINVAL;
253 	}
254 
255 	bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
256 				  fs_info->bdev_holder);
257 	if (IS_ERR(bdev)) {
258 		btrfs_err(fs_info, "target device %s is invalid!", device_path);
259 		return PTR_ERR(bdev);
260 	}
261 
262 	sync_blockdev(bdev);
263 
264 	list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) {
265 		if (device->bdev == bdev) {
266 			btrfs_err(fs_info,
267 				  "target device is in the filesystem!");
268 			ret = -EEXIST;
269 			goto error;
270 		}
271 	}
272 
273 
274 	if (i_size_read(bdev->bd_inode) <
275 	    btrfs_device_get_total_bytes(srcdev)) {
276 		btrfs_err(fs_info,
277 			  "target device is smaller than source device!");
278 		ret = -EINVAL;
279 		goto error;
280 	}
281 
282 
283 	device = btrfs_alloc_device(NULL, &devid, NULL);
284 	if (IS_ERR(device)) {
285 		ret = PTR_ERR(device);
286 		goto error;
287 	}
288 
289 	name = rcu_string_strdup(device_path, GFP_KERNEL);
290 	if (!name) {
291 		btrfs_free_device(device);
292 		ret = -ENOMEM;
293 		goto error;
294 	}
295 	rcu_assign_pointer(device->name, name);
296 
297 	set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
298 	device->generation = 0;
299 	device->io_width = fs_info->sectorsize;
300 	device->io_align = fs_info->sectorsize;
301 	device->sector_size = fs_info->sectorsize;
302 	device->total_bytes = btrfs_device_get_total_bytes(srcdev);
303 	device->disk_total_bytes = btrfs_device_get_disk_total_bytes(srcdev);
304 	device->bytes_used = btrfs_device_get_bytes_used(srcdev);
305 	device->commit_total_bytes = srcdev->commit_total_bytes;
306 	device->commit_bytes_used = device->bytes_used;
307 	device->fs_info = fs_info;
308 	device->bdev = bdev;
309 	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
310 	set_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
311 	device->mode = FMODE_EXCL;
312 	device->dev_stats_valid = 1;
313 	set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE);
314 	device->fs_devices = fs_info->fs_devices;
315 
316 	mutex_lock(&fs_info->fs_devices->device_list_mutex);
317 	list_add(&device->dev_list, &fs_info->fs_devices->devices);
318 	fs_info->fs_devices->num_devices++;
319 	fs_info->fs_devices->open_devices++;
320 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
321 
322 	*device_out = device;
323 	return 0;
324 
325 error:
326 	blkdev_put(bdev, FMODE_EXCL);
327 	return ret;
328 }
329 
330 /*
331  * called from commit_transaction. Writes changed device replace state to
332  * disk.
333  */
btrfs_run_dev_replace(struct btrfs_trans_handle * trans)334 int btrfs_run_dev_replace(struct btrfs_trans_handle *trans)
335 {
336 	struct btrfs_fs_info *fs_info = trans->fs_info;
337 	int ret;
338 	struct btrfs_root *dev_root = fs_info->dev_root;
339 	struct btrfs_path *path;
340 	struct btrfs_key key;
341 	struct extent_buffer *eb;
342 	struct btrfs_dev_replace_item *ptr;
343 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
344 
345 	down_read(&dev_replace->rwsem);
346 	if (!dev_replace->is_valid ||
347 	    !dev_replace->item_needs_writeback) {
348 		up_read(&dev_replace->rwsem);
349 		return 0;
350 	}
351 	up_read(&dev_replace->rwsem);
352 
353 	key.objectid = 0;
354 	key.type = BTRFS_DEV_REPLACE_KEY;
355 	key.offset = 0;
356 
357 	path = btrfs_alloc_path();
358 	if (!path) {
359 		ret = -ENOMEM;
360 		goto out;
361 	}
362 	ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
363 	if (ret < 0) {
364 		btrfs_warn(fs_info,
365 			   "error %d while searching for dev_replace item!",
366 			   ret);
367 		goto out;
368 	}
369 
370 	if (ret == 0 &&
371 	    btrfs_item_size_nr(path->nodes[0], path->slots[0]) < sizeof(*ptr)) {
372 		/*
373 		 * need to delete old one and insert a new one.
374 		 * Since no attempt is made to recover any old state, if the
375 		 * dev_replace state is 'running', the data on the target
376 		 * drive is lost.
377 		 * It would be possible to recover the state: just make sure
378 		 * that the beginning of the item is never changed and always
379 		 * contains all the essential information. Then read this
380 		 * minimal set of information and use it as a base for the
381 		 * new state.
382 		 */
383 		ret = btrfs_del_item(trans, dev_root, path);
384 		if (ret != 0) {
385 			btrfs_warn(fs_info,
386 				   "delete too small dev_replace item failed %d!",
387 				   ret);
388 			goto out;
389 		}
390 		ret = 1;
391 	}
392 
393 	if (ret == 1) {
394 		/* need to insert a new item */
395 		btrfs_release_path(path);
396 		ret = btrfs_insert_empty_item(trans, dev_root, path,
397 					      &key, sizeof(*ptr));
398 		if (ret < 0) {
399 			btrfs_warn(fs_info,
400 				   "insert dev_replace item failed %d!", ret);
401 			goto out;
402 		}
403 	}
404 
405 	eb = path->nodes[0];
406 	ptr = btrfs_item_ptr(eb, path->slots[0],
407 			     struct btrfs_dev_replace_item);
408 
409 	down_write(&dev_replace->rwsem);
410 	if (dev_replace->srcdev)
411 		btrfs_set_dev_replace_src_devid(eb, ptr,
412 			dev_replace->srcdev->devid);
413 	else
414 		btrfs_set_dev_replace_src_devid(eb, ptr, (u64)-1);
415 	btrfs_set_dev_replace_cont_reading_from_srcdev_mode(eb, ptr,
416 		dev_replace->cont_reading_from_srcdev_mode);
417 	btrfs_set_dev_replace_replace_state(eb, ptr,
418 		dev_replace->replace_state);
419 	btrfs_set_dev_replace_time_started(eb, ptr, dev_replace->time_started);
420 	btrfs_set_dev_replace_time_stopped(eb, ptr, dev_replace->time_stopped);
421 	btrfs_set_dev_replace_num_write_errors(eb, ptr,
422 		atomic64_read(&dev_replace->num_write_errors));
423 	btrfs_set_dev_replace_num_uncorrectable_read_errors(eb, ptr,
424 		atomic64_read(&dev_replace->num_uncorrectable_read_errors));
425 	dev_replace->cursor_left_last_write_of_item =
426 		dev_replace->cursor_left;
427 	btrfs_set_dev_replace_cursor_left(eb, ptr,
428 		dev_replace->cursor_left_last_write_of_item);
429 	btrfs_set_dev_replace_cursor_right(eb, ptr,
430 		dev_replace->cursor_right);
431 	dev_replace->item_needs_writeback = 0;
432 	up_write(&dev_replace->rwsem);
433 
434 	btrfs_mark_buffer_dirty(eb);
435 
436 out:
437 	btrfs_free_path(path);
438 
439 	return ret;
440 }
441 
btrfs_dev_name(struct btrfs_device * device)442 static char* btrfs_dev_name(struct btrfs_device *device)
443 {
444 	if (!device || test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
445 		return "<missing disk>";
446 	else
447 		return rcu_str_deref(device->name);
448 }
449 
btrfs_dev_replace_start(struct btrfs_fs_info * fs_info,const char * tgtdev_name,u64 srcdevid,const char * srcdev_name,int read_src)450 static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,
451 		const char *tgtdev_name, u64 srcdevid, const char *srcdev_name,
452 		int read_src)
453 {
454 	struct btrfs_root *root = fs_info->dev_root;
455 	struct btrfs_trans_handle *trans;
456 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
457 	int ret;
458 	struct btrfs_device *tgt_device = NULL;
459 	struct btrfs_device *src_device = NULL;
460 
461 	src_device = btrfs_find_device_by_devspec(fs_info, srcdevid,
462 						  srcdev_name);
463 	if (IS_ERR(src_device))
464 		return PTR_ERR(src_device);
465 
466 	if (btrfs_pinned_by_swapfile(fs_info, src_device)) {
467 		btrfs_warn_in_rcu(fs_info,
468 	  "cannot replace device %s (devid %llu) due to active swapfile",
469 			btrfs_dev_name(src_device), src_device->devid);
470 		return -ETXTBSY;
471 	}
472 
473 	/*
474 	 * Here we commit the transaction to make sure commit_total_bytes
475 	 * of all the devices are updated.
476 	 */
477 	trans = btrfs_attach_transaction(root);
478 	if (!IS_ERR(trans)) {
479 		ret = btrfs_commit_transaction(trans);
480 		if (ret)
481 			return ret;
482 	} else if (PTR_ERR(trans) != -ENOENT) {
483 		return PTR_ERR(trans);
484 	}
485 
486 	ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name,
487 					    src_device, &tgt_device);
488 	if (ret)
489 		return ret;
490 
491 	down_write(&dev_replace->rwsem);
492 	switch (dev_replace->replace_state) {
493 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
494 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
495 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
496 		break;
497 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
498 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
499 		ASSERT(0);
500 		ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_ALREADY_STARTED;
501 		up_write(&dev_replace->rwsem);
502 		goto leave;
503 	}
504 
505 	dev_replace->cont_reading_from_srcdev_mode = read_src;
506 	dev_replace->srcdev = src_device;
507 	dev_replace->tgtdev = tgt_device;
508 
509 	btrfs_info_in_rcu(fs_info,
510 		      "dev_replace from %s (devid %llu) to %s started",
511 		      btrfs_dev_name(src_device),
512 		      src_device->devid,
513 		      rcu_str_deref(tgt_device->name));
514 
515 	/*
516 	 * from now on, the writes to the srcdev are all duplicated to
517 	 * go to the tgtdev as well (refer to btrfs_map_block()).
518 	 */
519 	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;
520 	dev_replace->time_started = ktime_get_real_seconds();
521 	dev_replace->cursor_left = 0;
522 	dev_replace->committed_cursor_left = 0;
523 	dev_replace->cursor_left_last_write_of_item = 0;
524 	dev_replace->cursor_right = 0;
525 	dev_replace->is_valid = 1;
526 	dev_replace->item_needs_writeback = 1;
527 	atomic64_set(&dev_replace->num_write_errors, 0);
528 	atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
529 	up_write(&dev_replace->rwsem);
530 
531 	ret = btrfs_sysfs_add_device(tgt_device);
532 	if (ret)
533 		btrfs_err(fs_info, "kobj add dev failed %d", ret);
534 
535 	btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
536 
537 	/* Commit dev_replace state and reserve 1 item for it. */
538 	trans = btrfs_start_transaction(root, 1);
539 	if (IS_ERR(trans)) {
540 		ret = PTR_ERR(trans);
541 		down_write(&dev_replace->rwsem);
542 		dev_replace->replace_state =
543 			BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED;
544 		dev_replace->srcdev = NULL;
545 		dev_replace->tgtdev = NULL;
546 		up_write(&dev_replace->rwsem);
547 		goto leave;
548 	}
549 
550 	ret = btrfs_commit_transaction(trans);
551 	WARN_ON(ret);
552 
553 	/* the disk copy procedure reuses the scrub code */
554 	ret = btrfs_scrub_dev(fs_info, src_device->devid, 0,
555 			      btrfs_device_get_total_bytes(src_device),
556 			      &dev_replace->scrub_progress, 0, 1);
557 
558 	ret = btrfs_dev_replace_finishing(fs_info, ret);
559 	if (ret == -EINPROGRESS)
560 		ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS;
561 
562 	return ret;
563 
564 leave:
565 	btrfs_destroy_dev_replace_tgtdev(tgt_device);
566 	return ret;
567 }
568 
btrfs_dev_replace_by_ioctl(struct btrfs_fs_info * fs_info,struct btrfs_ioctl_dev_replace_args * args)569 int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info,
570 			    struct btrfs_ioctl_dev_replace_args *args)
571 {
572 	int ret;
573 
574 	switch (args->start.cont_reading_from_srcdev_mode) {
575 	case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_ALWAYS:
576 	case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_AVOID:
577 		break;
578 	default:
579 		return -EINVAL;
580 	}
581 
582 	if ((args->start.srcdevid == 0 && args->start.srcdev_name[0] == '\0') ||
583 	    args->start.tgtdev_name[0] == '\0')
584 		return -EINVAL;
585 
586 	ret = btrfs_dev_replace_start(fs_info, args->start.tgtdev_name,
587 					args->start.srcdevid,
588 					args->start.srcdev_name,
589 					args->start.cont_reading_from_srcdev_mode);
590 	args->result = ret;
591 	/* don't warn if EINPROGRESS, someone else might be running scrub */
592 	if (ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS ||
593 	    ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR)
594 		return 0;
595 
596 	return ret;
597 }
598 
599 /*
600  * blocked until all in-flight bios operations are finished.
601  */
btrfs_rm_dev_replace_blocked(struct btrfs_fs_info * fs_info)602 static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info)
603 {
604 	set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);
605 	wait_event(fs_info->dev_replace.replace_wait, !percpu_counter_sum(
606 		   &fs_info->dev_replace.bio_counter));
607 }
608 
609 /*
610  * we have removed target device, it is safe to allow new bios request.
611  */
btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info * fs_info)612 static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info)
613 {
614 	clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);
615 	wake_up(&fs_info->dev_replace.replace_wait);
616 }
617 
618 /*
619  * When finishing the device replace, before swapping the source device with the
620  * target device we must update the chunk allocation state in the target device,
621  * as it is empty because replace works by directly copying the chunks and not
622  * through the normal chunk allocation path.
623  */
btrfs_set_target_alloc_state(struct btrfs_device * srcdev,struct btrfs_device * tgtdev)624 static int btrfs_set_target_alloc_state(struct btrfs_device *srcdev,
625 					struct btrfs_device *tgtdev)
626 {
627 	struct extent_state *cached_state = NULL;
628 	u64 start = 0;
629 	u64 found_start;
630 	u64 found_end;
631 	int ret = 0;
632 
633 	lockdep_assert_held(&srcdev->fs_info->chunk_mutex);
634 
635 	while (!find_first_extent_bit(&srcdev->alloc_state, start,
636 				      &found_start, &found_end,
637 				      CHUNK_ALLOCATED, &cached_state)) {
638 		ret = set_extent_bits(&tgtdev->alloc_state, found_start,
639 				      found_end, CHUNK_ALLOCATED);
640 		if (ret)
641 			break;
642 		start = found_end + 1;
643 	}
644 
645 	free_extent_state(cached_state);
646 	return ret;
647 }
648 
btrfs_dev_replace_update_device_in_mapping_tree(struct btrfs_fs_info * fs_info,struct btrfs_device * srcdev,struct btrfs_device * tgtdev)649 static void btrfs_dev_replace_update_device_in_mapping_tree(
650 						struct btrfs_fs_info *fs_info,
651 						struct btrfs_device *srcdev,
652 						struct btrfs_device *tgtdev)
653 {
654 	struct extent_map_tree *em_tree = &fs_info->mapping_tree;
655 	struct extent_map *em;
656 	struct map_lookup *map;
657 	u64 start = 0;
658 	int i;
659 
660 	write_lock(&em_tree->lock);
661 	do {
662 		em = lookup_extent_mapping(em_tree, start, (u64)-1);
663 		if (!em)
664 			break;
665 		map = em->map_lookup;
666 		for (i = 0; i < map->num_stripes; i++)
667 			if (srcdev == map->stripes[i].dev)
668 				map->stripes[i].dev = tgtdev;
669 		start = em->start + em->len;
670 		free_extent_map(em);
671 	} while (start);
672 	write_unlock(&em_tree->lock);
673 }
674 
btrfs_dev_replace_finishing(struct btrfs_fs_info * fs_info,int scrub_ret)675 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
676 				       int scrub_ret)
677 {
678 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
679 	struct btrfs_device *tgt_device;
680 	struct btrfs_device *src_device;
681 	struct btrfs_root *root = fs_info->tree_root;
682 	u8 uuid_tmp[BTRFS_UUID_SIZE];
683 	struct btrfs_trans_handle *trans;
684 	int ret = 0;
685 
686 	/* don't allow cancel or unmount to disturb the finishing procedure */
687 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
688 
689 	down_read(&dev_replace->rwsem);
690 	/* was the operation canceled, or is it finished? */
691 	if (dev_replace->replace_state !=
692 	    BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED) {
693 		up_read(&dev_replace->rwsem);
694 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
695 		return 0;
696 	}
697 
698 	tgt_device = dev_replace->tgtdev;
699 	src_device = dev_replace->srcdev;
700 	up_read(&dev_replace->rwsem);
701 
702 	/*
703 	 * flush all outstanding I/O and inode extent mappings before the
704 	 * copy operation is declared as being finished
705 	 */
706 	ret = btrfs_start_delalloc_roots(fs_info, U64_MAX, false);
707 	if (ret) {
708 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
709 		return ret;
710 	}
711 	btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
712 
713 	if (!scrub_ret)
714 		btrfs_reada_remove_dev(src_device);
715 
716 	/*
717 	 * We have to use this loop approach because at this point src_device
718 	 * has to be available for transaction commit to complete, yet new
719 	 * chunks shouldn't be allocated on the device.
720 	 */
721 	while (1) {
722 		trans = btrfs_start_transaction(root, 0);
723 		if (IS_ERR(trans)) {
724 			btrfs_reada_undo_remove_dev(src_device);
725 			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
726 			return PTR_ERR(trans);
727 		}
728 		ret = btrfs_commit_transaction(trans);
729 		WARN_ON(ret);
730 
731 		/* Prevent write_all_supers() during the finishing procedure */
732 		mutex_lock(&fs_info->fs_devices->device_list_mutex);
733 		/* Prevent new chunks being allocated on the source device */
734 		mutex_lock(&fs_info->chunk_mutex);
735 
736 		if (!list_empty(&src_device->post_commit_list)) {
737 			mutex_unlock(&fs_info->fs_devices->device_list_mutex);
738 			mutex_unlock(&fs_info->chunk_mutex);
739 		} else {
740 			break;
741 		}
742 	}
743 
744 	down_write(&dev_replace->rwsem);
745 	dev_replace->replace_state =
746 		scrub_ret ? BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED
747 			  : BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED;
748 	dev_replace->tgtdev = NULL;
749 	dev_replace->srcdev = NULL;
750 	dev_replace->time_stopped = ktime_get_real_seconds();
751 	dev_replace->item_needs_writeback = 1;
752 
753 	/*
754 	 * Update allocation state in the new device and replace the old device
755 	 * with the new one in the mapping tree.
756 	 */
757 	if (!scrub_ret) {
758 		scrub_ret = btrfs_set_target_alloc_state(src_device, tgt_device);
759 		if (scrub_ret)
760 			goto error;
761 		btrfs_dev_replace_update_device_in_mapping_tree(fs_info,
762 								src_device,
763 								tgt_device);
764 	} else {
765 		if (scrub_ret != -ECANCELED)
766 			btrfs_err_in_rcu(fs_info,
767 				 "btrfs_scrub_dev(%s, %llu, %s) failed %d",
768 				 btrfs_dev_name(src_device),
769 				 src_device->devid,
770 				 rcu_str_deref(tgt_device->name), scrub_ret);
771 error:
772 		up_write(&dev_replace->rwsem);
773 		mutex_unlock(&fs_info->chunk_mutex);
774 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
775 		btrfs_reada_undo_remove_dev(src_device);
776 		btrfs_rm_dev_replace_blocked(fs_info);
777 		if (tgt_device)
778 			btrfs_destroy_dev_replace_tgtdev(tgt_device);
779 		btrfs_rm_dev_replace_unblocked(fs_info);
780 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
781 
782 		return scrub_ret;
783 	}
784 
785 	btrfs_info_in_rcu(fs_info,
786 			  "dev_replace from %s (devid %llu) to %s finished",
787 			  btrfs_dev_name(src_device),
788 			  src_device->devid,
789 			  rcu_str_deref(tgt_device->name));
790 	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &tgt_device->dev_state);
791 	tgt_device->devid = src_device->devid;
792 	src_device->devid = BTRFS_DEV_REPLACE_DEVID;
793 	memcpy(uuid_tmp, tgt_device->uuid, sizeof(uuid_tmp));
794 	memcpy(tgt_device->uuid, src_device->uuid, sizeof(tgt_device->uuid));
795 	memcpy(src_device->uuid, uuid_tmp, sizeof(src_device->uuid));
796 	btrfs_device_set_total_bytes(tgt_device, src_device->total_bytes);
797 	btrfs_device_set_disk_total_bytes(tgt_device,
798 					  src_device->disk_total_bytes);
799 	btrfs_device_set_bytes_used(tgt_device, src_device->bytes_used);
800 	tgt_device->commit_bytes_used = src_device->bytes_used;
801 
802 	btrfs_assign_next_active_device(src_device, tgt_device);
803 
804 	list_add(&tgt_device->dev_alloc_list, &fs_info->fs_devices->alloc_list);
805 	fs_info->fs_devices->rw_devices++;
806 
807 	up_write(&dev_replace->rwsem);
808 	btrfs_rm_dev_replace_blocked(fs_info);
809 
810 	btrfs_rm_dev_replace_remove_srcdev(src_device);
811 
812 	btrfs_rm_dev_replace_unblocked(fs_info);
813 
814 	/*
815 	 * Increment dev_stats_ccnt so that btrfs_run_dev_stats() will
816 	 * update on-disk dev stats value during commit transaction
817 	 */
818 	atomic_inc(&tgt_device->dev_stats_ccnt);
819 
820 	/*
821 	 * this is again a consistent state where no dev_replace procedure
822 	 * is running, the target device is part of the filesystem, the
823 	 * source device is not part of the filesystem anymore and its 1st
824 	 * superblock is scratched out so that it is no longer marked to
825 	 * belong to this filesystem.
826 	 */
827 	mutex_unlock(&fs_info->chunk_mutex);
828 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
829 
830 	/* replace the sysfs entry */
831 	btrfs_sysfs_remove_device(src_device);
832 	btrfs_sysfs_update_devid(tgt_device);
833 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &src_device->dev_state))
834 		btrfs_scratch_superblocks(fs_info, src_device->bdev,
835 					  src_device->name->str);
836 
837 	/* write back the superblocks */
838 	trans = btrfs_start_transaction(root, 0);
839 	if (!IS_ERR(trans))
840 		btrfs_commit_transaction(trans);
841 
842 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
843 
844 	btrfs_rm_dev_replace_free_srcdev(src_device);
845 
846 	return 0;
847 }
848 
849 /*
850  * Read progress of device replace status according to the state and last
851  * stored position. The value format is the same as for
852  * btrfs_dev_replace::progress_1000
853  */
btrfs_dev_replace_progress(struct btrfs_fs_info * fs_info)854 static u64 btrfs_dev_replace_progress(struct btrfs_fs_info *fs_info)
855 {
856 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
857 	u64 ret = 0;
858 
859 	switch (dev_replace->replace_state) {
860 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
861 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
862 		ret = 0;
863 		break;
864 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
865 		ret = 1000;
866 		break;
867 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
868 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
869 		ret = div64_u64(dev_replace->cursor_left,
870 				div_u64(btrfs_device_get_total_bytes(
871 						dev_replace->srcdev), 1000));
872 		break;
873 	}
874 
875 	return ret;
876 }
877 
btrfs_dev_replace_status(struct btrfs_fs_info * fs_info,struct btrfs_ioctl_dev_replace_args * args)878 void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info,
879 			      struct btrfs_ioctl_dev_replace_args *args)
880 {
881 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
882 
883 	down_read(&dev_replace->rwsem);
884 	/* even if !dev_replace_is_valid, the values are good enough for
885 	 * the replace_status ioctl */
886 	args->result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
887 	args->status.replace_state = dev_replace->replace_state;
888 	args->status.time_started = dev_replace->time_started;
889 	args->status.time_stopped = dev_replace->time_stopped;
890 	args->status.num_write_errors =
891 		atomic64_read(&dev_replace->num_write_errors);
892 	args->status.num_uncorrectable_read_errors =
893 		atomic64_read(&dev_replace->num_uncorrectable_read_errors);
894 	args->status.progress_1000 = btrfs_dev_replace_progress(fs_info);
895 	up_read(&dev_replace->rwsem);
896 }
897 
btrfs_dev_replace_cancel(struct btrfs_fs_info * fs_info)898 int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info)
899 {
900 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
901 	struct btrfs_device *tgt_device = NULL;
902 	struct btrfs_device *src_device = NULL;
903 	struct btrfs_trans_handle *trans;
904 	struct btrfs_root *root = fs_info->tree_root;
905 	int result;
906 	int ret;
907 
908 	if (sb_rdonly(fs_info->sb))
909 		return -EROFS;
910 
911 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
912 	down_write(&dev_replace->rwsem);
913 	switch (dev_replace->replace_state) {
914 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
915 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
916 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
917 		result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED;
918 		up_write(&dev_replace->rwsem);
919 		break;
920 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
921 		tgt_device = dev_replace->tgtdev;
922 		src_device = dev_replace->srcdev;
923 		up_write(&dev_replace->rwsem);
924 		ret = btrfs_scrub_cancel(fs_info);
925 		if (ret < 0) {
926 			result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED;
927 		} else {
928 			result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
929 			/*
930 			 * btrfs_dev_replace_finishing() will handle the
931 			 * cleanup part
932 			 */
933 			btrfs_info_in_rcu(fs_info,
934 				"dev_replace from %s (devid %llu) to %s canceled",
935 				btrfs_dev_name(src_device), src_device->devid,
936 				btrfs_dev_name(tgt_device));
937 		}
938 		break;
939 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
940 		/*
941 		 * Scrub doing the replace isn't running so we need to do the
942 		 * cleanup step of btrfs_dev_replace_finishing() here
943 		 */
944 		result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
945 		tgt_device = dev_replace->tgtdev;
946 		src_device = dev_replace->srcdev;
947 		dev_replace->tgtdev = NULL;
948 		dev_replace->srcdev = NULL;
949 		dev_replace->replace_state =
950 				BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED;
951 		dev_replace->time_stopped = ktime_get_real_seconds();
952 		dev_replace->item_needs_writeback = 1;
953 
954 		up_write(&dev_replace->rwsem);
955 
956 		/* Scrub for replace must not be running in suspended state */
957 		btrfs_scrub_cancel(fs_info);
958 
959 		trans = btrfs_start_transaction(root, 0);
960 		if (IS_ERR(trans)) {
961 			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
962 			return PTR_ERR(trans);
963 		}
964 		ret = btrfs_commit_transaction(trans);
965 		WARN_ON(ret);
966 
967 		btrfs_info_in_rcu(fs_info,
968 		"suspended dev_replace from %s (devid %llu) to %s canceled",
969 			btrfs_dev_name(src_device), src_device->devid,
970 			btrfs_dev_name(tgt_device));
971 
972 		if (tgt_device)
973 			btrfs_destroy_dev_replace_tgtdev(tgt_device);
974 		break;
975 	default:
976 		up_write(&dev_replace->rwsem);
977 		result = -EINVAL;
978 	}
979 
980 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
981 	return result;
982 }
983 
btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info * fs_info)984 void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info)
985 {
986 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
987 
988 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
989 	down_write(&dev_replace->rwsem);
990 
991 	switch (dev_replace->replace_state) {
992 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
993 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
994 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
995 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
996 		break;
997 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
998 		dev_replace->replace_state =
999 			BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
1000 		dev_replace->time_stopped = ktime_get_real_seconds();
1001 		dev_replace->item_needs_writeback = 1;
1002 		btrfs_info(fs_info, "suspending dev_replace for unmount");
1003 		break;
1004 	}
1005 
1006 	up_write(&dev_replace->rwsem);
1007 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
1008 }
1009 
1010 /* resume dev_replace procedure that was interrupted by unmount */
btrfs_resume_dev_replace_async(struct btrfs_fs_info * fs_info)1011 int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)
1012 {
1013 	struct task_struct *task;
1014 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1015 
1016 	down_write(&dev_replace->rwsem);
1017 
1018 	switch (dev_replace->replace_state) {
1019 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
1020 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
1021 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
1022 		up_write(&dev_replace->rwsem);
1023 		return 0;
1024 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1025 		break;
1026 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1027 		dev_replace->replace_state =
1028 			BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;
1029 		break;
1030 	}
1031 	if (!dev_replace->tgtdev || !dev_replace->tgtdev->bdev) {
1032 		btrfs_info(fs_info,
1033 			   "cannot continue dev_replace, tgtdev is missing");
1034 		btrfs_info(fs_info,
1035 			   "you may cancel the operation after 'mount -o degraded'");
1036 		dev_replace->replace_state =
1037 					BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
1038 		up_write(&dev_replace->rwsem);
1039 		return 0;
1040 	}
1041 	up_write(&dev_replace->rwsem);
1042 
1043 	/*
1044 	 * This could collide with a paused balance, but the exclusive op logic
1045 	 * should never allow both to start and pause. We don't want to allow
1046 	 * dev-replace to start anyway.
1047 	 */
1048 	if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) {
1049 		down_write(&dev_replace->rwsem);
1050 		dev_replace->replace_state =
1051 					BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
1052 		up_write(&dev_replace->rwsem);
1053 		btrfs_info(fs_info,
1054 		"cannot resume dev-replace, other exclusive operation running");
1055 		return 0;
1056 	}
1057 
1058 	task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl");
1059 	return PTR_ERR_OR_ZERO(task);
1060 }
1061 
btrfs_dev_replace_kthread(void * data)1062 static int btrfs_dev_replace_kthread(void *data)
1063 {
1064 	struct btrfs_fs_info *fs_info = data;
1065 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1066 	u64 progress;
1067 	int ret;
1068 
1069 	progress = btrfs_dev_replace_progress(fs_info);
1070 	progress = div_u64(progress, 10);
1071 	btrfs_info_in_rcu(fs_info,
1072 		"continuing dev_replace from %s (devid %llu) to target %s @%u%%",
1073 		btrfs_dev_name(dev_replace->srcdev),
1074 		dev_replace->srcdev->devid,
1075 		btrfs_dev_name(dev_replace->tgtdev),
1076 		(unsigned int)progress);
1077 
1078 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,
1079 			      dev_replace->committed_cursor_left,
1080 			      btrfs_device_get_total_bytes(dev_replace->srcdev),
1081 			      &dev_replace->scrub_progress, 0, 1);
1082 	ret = btrfs_dev_replace_finishing(fs_info, ret);
1083 	WARN_ON(ret && ret != -ECANCELED);
1084 
1085 	btrfs_exclop_finish(fs_info);
1086 	return 0;
1087 }
1088 
btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace * dev_replace)1089 int __pure btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace)
1090 {
1091 	if (!dev_replace->is_valid)
1092 		return 0;
1093 
1094 	switch (dev_replace->replace_state) {
1095 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
1096 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
1097 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
1098 		return 0;
1099 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1100 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1101 		/*
1102 		 * return true even if tgtdev is missing (this is
1103 		 * something that can happen if the dev_replace
1104 		 * procedure is suspended by an umount and then
1105 		 * the tgtdev is missing (or "btrfs dev scan") was
1106 		 * not called and the filesystem is remounted
1107 		 * in degraded state. This does not stop the
1108 		 * dev_replace procedure. It needs to be canceled
1109 		 * manually if the cancellation is wanted.
1110 		 */
1111 		break;
1112 	}
1113 	return 1;
1114 }
1115 
btrfs_bio_counter_inc_noblocked(struct btrfs_fs_info * fs_info)1116 void btrfs_bio_counter_inc_noblocked(struct btrfs_fs_info *fs_info)
1117 {
1118 	percpu_counter_inc(&fs_info->dev_replace.bio_counter);
1119 }
1120 
btrfs_bio_counter_sub(struct btrfs_fs_info * fs_info,s64 amount)1121 void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount)
1122 {
1123 	percpu_counter_sub(&fs_info->dev_replace.bio_counter, amount);
1124 	cond_wake_up_nomb(&fs_info->dev_replace.replace_wait);
1125 }
1126 
btrfs_bio_counter_inc_blocked(struct btrfs_fs_info * fs_info)1127 void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info)
1128 {
1129 	while (1) {
1130 		percpu_counter_inc(&fs_info->dev_replace.bio_counter);
1131 		if (likely(!test_bit(BTRFS_FS_STATE_DEV_REPLACING,
1132 				     &fs_info->fs_state)))
1133 			break;
1134 
1135 		btrfs_bio_counter_dec(fs_info);
1136 		wait_event(fs_info->dev_replace.replace_wait,
1137 			   !test_bit(BTRFS_FS_STATE_DEV_REPLACING,
1138 				     &fs_info->fs_state));
1139 	}
1140 }
1141