1 // SPDX-License-Identifier: GPL-2.0-or-later
5  * Copyright (C) 2000-2004 Neil Brown
7  * RAID-10 support for md.
24 #include "md-bitmap.h"
78 	do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid10 " fmt, ##args); } while (0)
80 #include "raid1-10.c"
85 		write_sequnlock_irq(&(conf)->resync_lock); \
88 #define cmd_after(conf) write_seqlock_irq(&(conf)->resync_lock)
91 	wait_event_cmd((conf)->wait_barrier, cond, cmd_before(conf, cmd), \
98  * for resync bio, r10bio pointer can be retrieved from the per-bio
103 	return get_resync_pages(bio)->raid_bio;  in get_resync_r10bio()
109 	int size = offsetof(struct r10bio, devs[conf->geo.raid_disks]);  in r10bio_pool_alloc()
144 	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||  in r10buf_pool_alloc()
145 	    test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))  in r10buf_pool_alloc()
146 		nalloc = conf->copies; /* resync */  in r10buf_pool_alloc()
151 	if (!conf->have_replacement)  in r10buf_pool_alloc()
162 	for (j = nalloc ; j-- ; ) {  in r10buf_pool_alloc()
166 		bio_init(bio, NULL, bio->bi_inline_vecs, RESYNC_PAGES, 0);  in r10buf_pool_alloc()
167 		r10_bio->devs[j].bio = bio;  in r10buf_pool_alloc()
168 		if (!conf->have_replacement)  in r10buf_pool_alloc()
173 		bio_init(bio, NULL, bio->bi_inline_vecs, RESYNC_PAGES, 0);  in r10buf_pool_alloc()
174 		r10_bio->devs[j].repl_bio = bio;  in r10buf_pool_alloc()
181 		struct bio *rbio = r10_bio->devs[j].repl_bio;  in r10buf_pool_alloc()
188 		bio = r10_bio->devs[j].bio;  in r10buf_pool_alloc()
191 				   &conf->mddev->recovery)) {  in r10buf_pool_alloc()
199 		rp->raid_bio = r10_bio;  in r10buf_pool_alloc()
200 		bio->bi_private = rp;  in r10buf_pool_alloc()
203 			rbio->bi_private = rp_repl;  in r10buf_pool_alloc()
210 	while (--j >= 0)  in r10buf_pool_alloc()
216 		if (r10_bio->devs[j].bio)  in r10buf_pool_alloc()
217 			bio_uninit(r10_bio->devs[j].bio);  in r10buf_pool_alloc()
218 		kfree(r10_bio->devs[j].bio);  in r10buf_pool_alloc()
219 		if (r10_bio->devs[j].repl_bio)  in r10buf_pool_alloc()
220 			bio_uninit(r10_bio->devs[j].repl_bio);  in r10buf_pool_alloc()
221 		kfree(r10_bio->devs[j].repl_bio);  in r10buf_pool_alloc()
236 	for (j = conf->copies; j--; ) {  in r10buf_pool_free()
237 		struct bio *bio = r10bio->devs[j].bio;  in r10buf_pool_free()
246 		bio = r10bio->devs[j].repl_bio;  in r10buf_pool_free()
263 	for (i = 0; i < conf->geo.raid_disks; i++) {  in put_all_bios()
264 		struct bio **bio = & r10_bio->devs[i].bio;  in put_all_bios()
268 		bio = &r10_bio->devs[i].repl_bio;  in put_all_bios()
269 		if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))  in put_all_bios()
277 	struct r10conf *conf = r10_bio->mddev->private;  in free_r10bio()
280 	mempool_free(r10_bio, &conf->r10bio_pool);  in free_r10bio()
285 	struct r10conf *conf = r10_bio->mddev->private;  in put_buf()
287 	mempool_free(r10_bio, &conf->r10buf_pool);  in put_buf()
294 	if (wq_has_sleeper(&conf->wait_barrier))  in wake_up_barrier()
295 		wake_up(&conf->wait_barrier);  in wake_up_barrier()
301 	struct mddev *mddev = r10_bio->mddev;  in reschedule_retry()
302 	struct r10conf *conf = mddev->private;  in reschedule_retry()
304 	spin_lock_irqsave(&conf->device_lock, flags);  in reschedule_retry()
305 	list_add(&r10_bio->retry_list, &conf->retry_list);  in reschedule_retry()
306 	conf->nr_queued ++;  in reschedule_retry()
307 	spin_unlock_irqrestore(&conf->device_lock, flags);  in reschedule_retry()
310 	wake_up(&conf->wait_barrier);  in reschedule_retry()
312 	md_wakeup_thread(mddev->thread);  in reschedule_retry()
322 	struct bio *bio = r10_bio->master_bio;  in raid_end_bio_io()
323 	struct r10conf *conf = r10_bio->mddev->private;  in raid_end_bio_io()
325 	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))  in raid_end_bio_io()
326 		bio->bi_status = BLK_STS_IOERR;  in raid_end_bio_io()
339  * Update disk head position estimator based on IRQ completion info.
343 	struct r10conf *conf = r10_bio->mddev->private;  in update_head_pos()
345 	conf->mirrors[r10_bio->devs[slot].devnum].head_position =  in update_head_pos()
346 		r10_bio->devs[slot].addr + (r10_bio->sectors);  in update_head_pos()
358 	for (slot = 0; slot < conf->geo.raid_disks; slot++) {  in find_bio_disk()
359 		if (r10_bio->devs[slot].bio == bio)  in find_bio_disk()
361 		if (r10_bio->devs[slot].repl_bio == bio) {  in find_bio_disk()
373 	return r10_bio->devs[slot].devnum;  in find_bio_disk()
378 	int uptodate = !bio->bi_status;  in raid10_end_read_request()
379 	struct r10bio *r10_bio = bio->bi_private;  in raid10_end_read_request()
382 	struct r10conf *conf = r10_bio->mddev->private;  in raid10_end_read_request()
384 	slot = r10_bio->read_slot;  in raid10_end_read_request()
385 	rdev = r10_bio->devs[slot].rdev;  in raid10_end_read_request()
398 		 * user-side. So if something waits for IO, then it will  in raid10_end_read_request()
401 		set_bit(R10BIO_Uptodate, &r10_bio->state);  in raid10_end_read_request()
408 		if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state),  in raid10_end_read_request()
409 			     rdev->raid_disk))  in raid10_end_read_request()
414 		rdev_dec_pending(rdev, conf->mddev);  in raid10_end_read_request()
417 		 * oops, read error - keep the refcount on the rdev  in raid10_end_read_request()
420 				   mdname(conf->mddev),  in raid10_end_read_request()
421 				   rdev->bdev,  in raid10_end_read_request()
422 				   (unsigned long long)r10_bio->sector);  in raid10_end_read_request()
423 		set_bit(R10BIO_ReadError, &r10_bio->state);  in raid10_end_read_request()
431 	md_bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,  in close_write()
432 			   r10_bio->sectors,  in close_write()
433 			   !test_bit(R10BIO_Degraded, &r10_bio->state),  in close_write()
435 	md_write_end(r10_bio->mddev);  in close_write()
440 	if (atomic_dec_and_test(&r10_bio->remaining)) {  in one_write_done()
441 		if (test_bit(R10BIO_WriteError, &r10_bio->state))  in one_write_done()
445 			if (test_bit(R10BIO_MadeGood, &r10_bio->state))  in one_write_done()
455 	struct r10bio *r10_bio = bio->bi_private;  in raid10_end_write_request()
458 	struct r10conf *conf = r10_bio->mddev->private;  in raid10_end_write_request()
464 	discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD;  in raid10_end_write_request()
469 		rdev = conf->mirrors[dev].replacement;  in raid10_end_write_request()
473 		rdev = conf->mirrors[dev].rdev;  in raid10_end_write_request()
478 	if (bio->bi_status && !discard_error) {  in raid10_end_write_request()
483 			md_error(rdev->mddev, rdev);  in raid10_end_write_request()
485 			set_bit(WriteErrorSeen,	&rdev->flags);  in raid10_end_write_request()
486 			if (!test_and_set_bit(WantReplacement, &rdev->flags))  in raid10_end_write_request()
488 					&rdev->mddev->recovery);  in raid10_end_write_request()
491 			if (test_bit(FailFast, &rdev->flags) &&  in raid10_end_write_request()
492 			    (bio->bi_opf & MD_FAILFAST)) {  in raid10_end_write_request()
493 				md_error(rdev->mddev, rdev);  in raid10_end_write_request()
500 			if (!test_bit(Faulty, &rdev->flags))  in raid10_end_write_request()
501 				set_bit(R10BIO_WriteError, &r10_bio->state);  in raid10_end_write_request()
504 				set_bit(R10BIO_Degraded, &r10_bio->state);  in raid10_end_write_request()
505 				r10_bio->devs[slot].bio = NULL;  in raid10_end_write_request()
517 		 * user-side. So if something waits for IO, then it will  in raid10_end_write_request()
528 		 * before rdev->recovery_offset, but for simplicity we don't  in raid10_end_write_request()
531 		if (test_bit(In_sync, &rdev->flags) &&  in raid10_end_write_request()
532 		    !test_bit(Faulty, &rdev->flags))  in raid10_end_write_request()
533 			set_bit(R10BIO_Uptodate, &r10_bio->state);  in raid10_end_write_request()
537 				r10_bio->devs[slot].addr,  in raid10_end_write_request()
538 				r10_bio->sectors,  in raid10_end_write_request()
542 				r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;  in raid10_end_write_request()
544 				r10_bio->devs[slot].bio = IO_MADE_GOOD;  in raid10_end_write_request()
546 			set_bit(R10BIO_MadeGood, &r10_bio->state);  in raid10_end_write_request()
557 		rdev_dec_pending(rdev, conf->mddev);  in raid10_end_write_request()
597 	last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;  in __raid10_find_phys()
598 	last_far_set_start *= geo->far_set_size;  in __raid10_find_phys()
600 	last_far_set_size = geo->far_set_size;  in __raid10_find_phys()
601 	last_far_set_size += (geo->raid_disks % geo->far_set_size);  in __raid10_find_phys()
604 	chunk = r10bio->sector >> geo->chunk_shift;  in __raid10_find_phys()
605 	sector = r10bio->sector & geo->chunk_mask;  in __raid10_find_phys()
607 	chunk *= geo->near_copies;  in __raid10_find_phys()
609 	dev = sector_div(stripe, geo->raid_disks);  in __raid10_find_phys()
610 	if (geo->far_offset)  in __raid10_find_phys()
611 		stripe *= geo->far_copies;  in __raid10_find_phys()
613 	sector += stripe << geo->chunk_shift;  in __raid10_find_phys()
616 	for (n = 0; n < geo->near_copies; n++) {  in __raid10_find_phys()
620 		r10bio->devs[slot].devnum = d;  in __raid10_find_phys()
621 		r10bio->devs[slot].addr = s;  in __raid10_find_phys()
624 		for (f = 1; f < geo->far_copies; f++) {  in __raid10_find_phys()
625 			set = d / geo->far_set_size;  in __raid10_find_phys()
626 			d += geo->near_copies;  in __raid10_find_phys()
628 			if ((geo->raid_disks % geo->far_set_size) &&  in __raid10_find_phys()
630 				d -= last_far_set_start;  in __raid10_find_phys()
634 				d %= geo->far_set_size;  in __raid10_find_phys()
635 				d += geo->far_set_size * set;  in __raid10_find_phys()
637 			s += geo->stride;  in __raid10_find_phys()
638 			r10bio->devs[slot].devnum = d;  in __raid10_find_phys()
639 			r10bio->devs[slot].addr = s;  in __raid10_find_phys()
643 		if (dev >= geo->raid_disks) {  in __raid10_find_phys()
645 			sector += (geo->chunk_mask + 1);  in __raid10_find_phys()
652 	struct geom *geo = &conf->geo;  in raid10_find_phys()
654 	if (conf->reshape_progress != MaxSector &&  in raid10_find_phys()
655 	    ((r10bio->sector >= conf->reshape_progress) !=  in raid10_find_phys()
656 	     conf->mddev->reshape_backwards)) {  in raid10_find_phys()
657 		set_bit(R10BIO_Previous, &r10bio->state);  in raid10_find_phys()
658 		geo = &conf->prev;  in raid10_find_phys()
660 		clear_bit(R10BIO_Previous, &r10bio->state);  in raid10_find_phys()
668 	/* Never use conf->prev as this is only called during resync  in raid10_find_virt()
671 	struct geom *geo = &conf->geo;  in raid10_find_virt()
672 	int far_set_start = (dev / geo->far_set_size) * geo->far_set_size;  in raid10_find_virt()
673 	int far_set_size = geo->far_set_size;  in raid10_find_virt()
676 	if (geo->raid_disks % geo->far_set_size) {  in raid10_find_virt()
677 		last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;  in raid10_find_virt()
678 		last_far_set_start *= geo->far_set_size;  in raid10_find_virt()
681 			far_set_size = geo->far_set_size;  in raid10_find_virt()
682 			far_set_size += (geo->raid_disks % geo->far_set_size);  in raid10_find_virt()
687 	offset = sector & geo->chunk_mask;  in raid10_find_virt()
688 	if (geo->far_offset) {  in raid10_find_virt()
689 		int fc;  in raid10_find_virt()  local
690 		chunk = sector >> geo->chunk_shift;  in raid10_find_virt()
691 		fc = sector_div(chunk, geo->far_copies);  in raid10_find_virt()
692 		dev -= fc * geo->near_copies;  in raid10_find_virt()
696 		while (sector >= geo->stride) {  in raid10_find_virt()
697 			sector -= geo->stride;  in raid10_find_virt()
698 			if (dev < (geo->near_copies + far_set_start))  in raid10_find_virt()
699 				dev += far_set_size - geo->near_copies;  in raid10_find_virt()
701 				dev -= geo->near_copies;  in raid10_find_virt()
703 		chunk = sector >> geo->chunk_shift;  in raid10_find_virt()
705 	vchunk = chunk * geo->raid_disks + dev;  in raid10_find_virt()
706 	sector_div(vchunk, geo->near_copies);  in raid10_find_virt()
707 	return (vchunk << geo->chunk_shift) + offset;  in raid10_find_virt()
712  * be done. There is a per-array 'next expected sequential IO' sector
713  * number - if this matches on the next IO then we use the last disk.
714  * There is also a per-disk 'last know head position' sector that is
716  * completion handlers update this position correctly. If there is no
733 	const sector_t this_sector = r10_bio->sector;  in read_balance()
735 	int sectors = r10_bio->sectors;  in read_balance()
743 	struct geom *geo = &conf->geo;  in read_balance()
747 	best_dist_slot = -1;  in read_balance()
754 	clear_bit(R10BIO_FailFast, &r10_bio->state);  in read_balance()
761 	if ((conf->mddev->recovery_cp < MaxSector  in read_balance()
762 	     && (this_sector + sectors >= conf->next_resync)) ||  in read_balance()
763 	    (mddev_is_clustered(conf->mddev) &&  in read_balance()
764 	     md_cluster_ops->area_resyncing(conf->mddev, READ, this_sector,  in read_balance()
768 	for (slot = 0; slot < conf->copies ; slot++) {  in read_balance()
775 		if (r10_bio->devs[slot].bio == IO_BLOCKED)  in read_balance()
777 		disk = r10_bio->devs[slot].devnum;  in read_balance()
778 		rdev = rcu_dereference(conf->mirrors[disk].replacement);  in read_balance()
779 		if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||  in read_balance()
780 		    r10_bio->devs[slot].addr + sectors >  in read_balance()
781 		    rdev->recovery_offset) {  in read_balance()
788 			rdev = rcu_dereference(conf->mirrors[disk].rdev);  in read_balance()
791 		    test_bit(Faulty, &rdev->flags))  in read_balance()
793 		if (!test_bit(In_sync, &rdev->flags) &&  in read_balance()
794 		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)  in read_balance()
797 		dev_sector = r10_bio->devs[slot].addr;  in read_balance()
808 				bad_sectors -= (dev_sector - first_bad);  in read_balance()
815 					first_bad - dev_sector;  in read_balance()
832 		nonrot = bdev_nonrot(rdev->bdev);  in read_balance()
834 		pending = atomic_read(&rdev->nr_pending);  in read_balance()
843 			set_bit(R10BIO_FailFast, &r10_bio->state);  in read_balance()
848 		if (geo->near_copies > 1 && !pending)  in read_balance()
852 		else if (geo->far_copies > 1)  in read_balance()
853 			new_distance = r10_bio->devs[slot].addr;  in read_balance()
855 			new_distance = abs(r10_bio->devs[slot].addr -  in read_balance()
856 					   conf->mirrors[disk].head_position);  in read_balance()
864 	if (slot >= conf->copies) {  in read_balance()
875 		atomic_inc(&rdev->nr_pending);  in read_balance()
876 		r10_bio->read_slot = slot;  in read_balance()
890 	spin_lock_irq(&conf->device_lock);  in flush_pending_writes()
892 	if (conf->pending_bio_list.head) {  in flush_pending_writes()
896 		bio = bio_list_get(&conf->pending_bio_list);  in flush_pending_writes()
897 		spin_unlock_irq(&conf->device_lock);  in flush_pending_writes()
901 		 * current->state might be TASK_UNINTERRUPTIBLE which will  in flush_pending_writes()
905 		 * is a false-positive. Silence the warning by resetting  in flush_pending_writes()
911 		raid1_prepare_flush_writes(conf->mddev->bitmap);  in flush_pending_writes()
912 		wake_up(&conf->wait_barrier);  in flush_pending_writes()
915 			struct bio *next = bio->bi_next;  in flush_pending_writes()
923 		spin_unlock_irq(&conf->device_lock);  in flush_pending_writes()
935  * We choose only to raise the barrier if no-one is waiting for the
950 	write_seqlock_irq(&conf->resync_lock);  in raise_barrier()
952 	if (WARN_ON_ONCE(force && !conf->barrier))  in raise_barrier()
956 	wait_event_barrier(conf, force || !conf->nr_waiting);  in raise_barrier()
959 	WRITE_ONCE(conf->barrier, conf->barrier + 1);  in raise_barrier()
962 	wait_event_barrier(conf, !atomic_read(&conf->nr_pending) &&  in raise_barrier()
963 				 conf->barrier < RESYNC_DEPTH);  in raise_barrier()
965 	write_sequnlock_irq(&conf->resync_lock);  in raise_barrier()
972 	write_seqlock_irqsave(&conf->resync_lock, flags);  in lower_barrier()
973 	WRITE_ONCE(conf->barrier, conf->barrier - 1);  in lower_barrier()
974 	write_sequnlock_irqrestore(&conf->resync_lock, flags);  in lower_barrier()
975 	wake_up(&conf->wait_barrier);  in lower_barrier()
980 	struct bio_list *bio_list = current->bio_list;  in stop_waiting_barrier()
984 	if (!conf->barrier)  in stop_waiting_barrier()
989 	 * rising completely), and the pre-process bio queue isn't empty, then  in stop_waiting_barrier()
993 	if (atomic_read(&conf->nr_pending) && bio_list &&  in stop_waiting_barrier()
998 	thread = rcu_dereference_protected(conf->mddev->thread, true);  in stop_waiting_barrier()
1004 	if (thread->tsk == current) {  in stop_waiting_barrier()
1005 		WARN_ON_ONCE(atomic_read(&conf->nr_pending) == 0);  in stop_waiting_barrier()
1014 	unsigned int seq = read_seqbegin(&conf->resync_lock);  in wait_barrier_nolock()
1016 	if (READ_ONCE(conf->barrier))  in wait_barrier_nolock()
1019 	atomic_inc(&conf->nr_pending);  in wait_barrier_nolock()
1020 	if (!read_seqretry(&conf->resync_lock, seq))  in wait_barrier_nolock()
1023 	if (atomic_dec_and_test(&conf->nr_pending))  in wait_barrier_nolock()
1036 	write_seqlock_irq(&conf->resync_lock);  in wait_barrier()
1037 	if (conf->barrier) {  in wait_barrier()
1042 			conf->nr_waiting++;  in wait_barrier()
1043 			raid10_log(conf->mddev, "wait barrier");  in wait_barrier()
1045 			conf->nr_waiting--;  in wait_barrier()
1047 		if (!conf->nr_waiting)  in wait_barrier()
1048 			wake_up(&conf->wait_barrier);  in wait_barrier()
1052 		atomic_inc(&conf->nr_pending);  in wait_barrier()
1053 	write_sequnlock_irq(&conf->resync_lock);  in wait_barrier()
1059 	if ((atomic_dec_and_test(&conf->nr_pending)) ||  in allow_barrier()
1060 			(conf->array_freeze_pending))  in allow_barrier()
1073 	 * pending IO requests to complete or be queued for re-try.  in freeze_array()
1078 	write_seqlock_irq(&conf->resync_lock);  in freeze_array()
1079 	conf->array_freeze_pending++;  in freeze_array()
1080 	WRITE_ONCE(conf->barrier, conf->barrier + 1);  in freeze_array()
1081 	conf->nr_waiting++;  in freeze_array()
1082 	wait_event_barrier_cmd(conf, atomic_read(&conf->nr_pending) ==  in freeze_array()
1083 			conf->nr_queued + extra, flush_pending_writes(conf));  in freeze_array()
1084 	conf->array_freeze_pending--;  in freeze_array()
1085 	write_sequnlock_irq(&conf->resync_lock);  in freeze_array()
1091 	write_seqlock_irq(&conf->resync_lock);  in unfreeze_array()
1092 	WRITE_ONCE(conf->barrier, conf->barrier - 1);  in unfreeze_array()
1093 	conf->nr_waiting--;  in unfreeze_array()
1094 	wake_up(&conf->wait_barrier);  in unfreeze_array()
1095 	write_sequnlock_irq(&conf->resync_lock);  in unfreeze_array()
1101 	if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||  in choose_data_offset()
1102 	    test_bit(R10BIO_Previous, &r10_bio->state))  in choose_data_offset()
1103 		return rdev->data_offset;  in choose_data_offset()
1105 		return rdev->new_data_offset;  in choose_data_offset()
1111 	struct mddev *mddev = plug->cb.data;  in raid10_unplug()
1112 	struct r10conf *conf = mddev->private;  in raid10_unplug()
1116 		spin_lock_irq(&conf->device_lock);  in raid10_unplug()
1117 		bio_list_merge(&conf->pending_bio_list, &plug->pending);  in raid10_unplug()
1118 		spin_unlock_irq(&conf->device_lock);  in raid10_unplug()
1120 		md_wakeup_thread(mddev->thread);  in raid10_unplug()
1125 	/* we aren't scheduling, so we can do the write-out directly. */  in raid10_unplug()
1126 	bio = bio_list_get(&plug->pending);  in raid10_unplug()
1127 	raid1_prepare_flush_writes(mddev->bitmap);  in raid10_unplug()
1131 		struct bio *next = bio->bi_next;  in raid10_unplug()
1150 	if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) {  in regular_request_wait()
1154 	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&  in regular_request_wait()
1155 	    bio->bi_iter.bi_sector < conf->reshape_progress &&  in regular_request_wait()
1156 	    bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {  in regular_request_wait()
1158 		if (bio->bi_opf & REQ_NOWAIT) {  in regular_request_wait()
1162 		raid10_log(conf->mddev, "wait reshape");  in regular_request_wait()
1163 		wait_event(conf->wait_barrier,  in regular_request_wait()
1164 			   conf->reshape_progress <= bio->bi_iter.bi_sector ||  in regular_request_wait()
1165 			   conf->reshape_progress >= bio->bi_iter.bi_sector +  in regular_request_wait()
1175 	struct r10conf *conf = mddev->private;  in raid10_read_request()
1178 	const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;  in raid10_read_request()
1182 	int slot = r10_bio->read_slot;  in raid10_read_request()
1186 	if (slot >= 0 && r10_bio->devs[slot].rdev) {  in raid10_read_request()
1202 		disk = r10_bio->devs[slot].devnum;  in raid10_read_request()
1203 		err_rdev = rcu_dereference(conf->mirrors[disk].rdev);  in raid10_read_request()
1205 			snprintf(b, sizeof(b), "%pg", err_rdev->bdev);  in raid10_read_request()
1209 			err_rdev = r10_bio->devs[slot].rdev;  in raid10_read_request()
1214 	if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors))  in raid10_read_request()
1221 					    (unsigned long long)r10_bio->sector);  in raid10_read_request()
1229 				   rdev->bdev,  in raid10_read_request()
1230 				   (unsigned long long)r10_bio->sector);  in raid10_read_request()
1233 					      gfp, &conf->bio_split);  in raid10_read_request()
1239 		r10_bio->master_bio = bio;  in raid10_read_request()
1240 		r10_bio->sectors = max_sectors;  in raid10_read_request()
1242 	slot = r10_bio->read_slot;  in raid10_read_request()
1246 		r10_bio->master_bio = bio;  in raid10_read_request()
1248 	read_bio = bio_alloc_clone(rdev->bdev, bio, gfp, &mddev->bio_set);  in raid10_read_request()
1250 	r10_bio->devs[slot].bio = read_bio;  in raid10_read_request()
1251 	r10_bio->devs[slot].rdev = rdev;  in raid10_read_request()
1253 	read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr +  in raid10_read_request()
1255 	read_bio->bi_end_io = raid10_end_read_request;  in raid10_read_request()
1256 	read_bio->bi_opf = op | do_sync;  in raid10_read_request()
1257 	if (test_bit(FailFast, &rdev->flags) &&  in raid10_read_request()
1258 	    test_bit(R10BIO_FailFast, &r10_bio->state))  in raid10_read_request()
1259 	        read_bio->bi_opf |= MD_FAILFAST;  in raid10_read_request()
1260 	read_bio->bi_private = r10_bio;  in raid10_read_request()
1262 	if (mddev->gendisk)  in raid10_read_request()
1263 	        trace_block_bio_remap(read_bio, disk_devt(mddev->gendisk),  in raid10_read_request()
1264 	                              r10_bio->sector);  in raid10_read_request()
1274 	const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;  in raid10_write_one_disk()
1275 	const blk_opf_t do_fua = bio->bi_opf & REQ_FUA;  in raid10_write_one_disk()
1277 	struct r10conf *conf = mddev->private;  in raid10_write_one_disk()
1279 	int devnum = r10_bio->devs[n_copy].devnum;  in raid10_write_one_disk()
1283 		rdev = conf->mirrors[devnum].replacement;  in raid10_write_one_disk()
1287 			rdev = conf->mirrors[devnum].rdev;  in raid10_write_one_disk()
1290 		rdev = conf->mirrors[devnum].rdev;  in raid10_write_one_disk()
1292 	mbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO, &mddev->bio_set);  in raid10_write_one_disk()
1294 		r10_bio->devs[n_copy].repl_bio = mbio;  in raid10_write_one_disk()
1296 		r10_bio->devs[n_copy].bio = mbio;  in raid10_write_one_disk()
1298 	mbio->bi_iter.bi_sector	= (r10_bio->devs[n_copy].addr +  in raid10_write_one_disk()
1300 	mbio->bi_end_io	= raid10_end_write_request;  in raid10_write_one_disk()
1301 	mbio->bi_opf = op | do_sync | do_fua;  in raid10_write_one_disk()
1303 				     &conf->mirrors[devnum].rdev->flags)  in raid10_write_one_disk()
1305 		mbio->bi_opf |= MD_FAILFAST;  in raid10_write_one_disk()
1306 	mbio->bi_private = r10_bio;  in raid10_write_one_disk()
1308 	if (conf->mddev->gendisk)  in raid10_write_one_disk()
1309 		trace_block_bio_remap(mbio, disk_devt(conf->mddev->gendisk),  in raid10_write_one_disk()
1310 				      r10_bio->sector);  in raid10_write_one_disk()
1312 	mbio->bi_bdev = (void *)rdev;  in raid10_write_one_disk()
1314 	atomic_inc(&r10_bio->remaining);  in raid10_write_one_disk()
1316 	if (!raid1_add_bio_to_plug(mddev, mbio, raid10_unplug, conf->copies)) {  in raid10_write_one_disk()
1317 		spin_lock_irqsave(&conf->device_lock, flags);  in raid10_write_one_disk()
1318 		bio_list_add(&conf->pending_bio_list, mbio);  in raid10_write_one_disk()
1319 		spin_unlock_irqrestore(&conf->device_lock, flags);  in raid10_write_one_disk()
1320 		md_wakeup_thread(mddev->thread);  in raid10_write_one_disk()
1329 	rrdev = rcu_dereference(mirror->replacement);  in dereference_rdev_and_rrdev()
1335 	rdev = rcu_dereference(mirror->rdev);  in dereference_rdev_and_rrdev()
1346 	struct r10conf *conf = mddev->private;  in wait_blocked_dev()
1352 	for (i = 0; i < conf->copies; i++) {  in wait_blocked_dev()
1355 		rdev = dereference_rdev_and_rrdev(&conf->mirrors[i], &rrdev);  in wait_blocked_dev()
1356 		if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {  in wait_blocked_dev()
1357 			atomic_inc(&rdev->nr_pending);  in wait_blocked_dev()
1361 		if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {  in wait_blocked_dev()
1362 			atomic_inc(&rrdev->nr_pending);  in wait_blocked_dev()
1367 		if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {  in wait_blocked_dev()
1369 			sector_t dev_sector = r10_bio->devs[i].addr;  in wait_blocked_dev()
1377 			if (!r10_bio->sectors)  in wait_blocked_dev()
1380 			is_bad = is_badblock(rdev, dev_sector, r10_bio->sectors,  in wait_blocked_dev()
1387 				atomic_inc(&rdev->nr_pending);  in wait_blocked_dev()
1388 				set_bit(BlockedBadBlocks, &rdev->flags);  in wait_blocked_dev()
1399 		raid10_log(conf->mddev, "%s wait rdev %d blocked",  in wait_blocked_dev()
1400 				__func__, blocked_rdev->raid_disk);  in wait_blocked_dev()
1410 	struct r10conf *conf = mddev->private;  in raid10_write_request()
1416 	     md_cluster_ops->area_resyncing(mddev, WRITE,  in raid10_write_request()
1417 					    bio->bi_iter.bi_sector,  in raid10_write_request()
1421 		if (bio->bi_opf & REQ_NOWAIT) {  in raid10_write_request()
1426 			prepare_to_wait(&conf->wait_barrier,  in raid10_write_request()
1428 			if (!md_cluster_ops->area_resyncing(mddev, WRITE,  in raid10_write_request()
1429 				 bio->bi_iter.bi_sector, bio_end_sector(bio)))  in raid10_write_request()
1433 		finish_wait(&conf->wait_barrier, &w);  in raid10_write_request()
1436 	sectors = r10_bio->sectors;  in raid10_write_request()
1439 	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&  in raid10_write_request()
1440 	    (mddev->reshape_backwards  in raid10_write_request()
1441 	     ? (bio->bi_iter.bi_sector < conf->reshape_safe &&  in raid10_write_request()
1442 		bio->bi_iter.bi_sector + sectors > conf->reshape_progress)  in raid10_write_request()
1443 	     : (bio->bi_iter.bi_sector + sectors > conf->reshape_safe &&  in raid10_write_request()
1444 		bio->bi_iter.bi_sector < conf->reshape_progress))) {  in raid10_write_request()
1445 		/* Need to update reshape_position in metadata */  in raid10_write_request()
1446 		mddev->reshape_position = conf->reshape_progress;  in raid10_write_request()
1447 		set_mask_bits(&mddev->sb_flags, 0,  in raid10_write_request()
1449 		md_wakeup_thread(mddev->thread);  in raid10_write_request()
1450 		if (bio->bi_opf & REQ_NOWAIT) {  in raid10_write_request()
1455 		raid10_log(conf->mddev, "wait reshape metadata");  in raid10_write_request()
1456 		wait_event(mddev->sb_wait,  in raid10_write_request()
1457 			   !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));  in raid10_write_request()
1459 		conf->reshape_safe = mddev->reshape_position;  in raid10_write_request()
1472 	r10_bio->read_slot = -1; /* make sure repl_bio gets freed */  in raid10_write_request()
1478 	max_sectors = r10_bio->sectors;  in raid10_write_request()
1480 	for (i = 0;  i < conf->copies; i++) {  in raid10_write_request()
1481 		int d = r10_bio->devs[i].devnum;  in raid10_write_request()
1484 		rdev = dereference_rdev_and_rrdev(&conf->mirrors[d], &rrdev);  in raid10_write_request()
1485 		if (rdev && (test_bit(Faulty, &rdev->flags)))  in raid10_write_request()
1487 		if (rrdev && (test_bit(Faulty, &rrdev->flags)))  in raid10_write_request()
1490 		r10_bio->devs[i].bio = NULL;  in raid10_write_request()
1491 		r10_bio->devs[i].repl_bio = NULL;  in raid10_write_request()
1494 			set_bit(R10BIO_Degraded, &r10_bio->state);  in raid10_write_request()
1497 		if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {  in raid10_write_request()
1499 			sector_t dev_sector = r10_bio->devs[i].addr;  in raid10_write_request()
1507 				bad_sectors -= (dev_sector - first_bad);  in raid10_write_request()
1515 				 * so it might be re-added, and we want to  in raid10_write_request()
1518 				 * fact that this chunk is not in-sync is  in raid10_write_request()
1524 				int good_sectors = first_bad - dev_sector;  in raid10_write_request()
1530 			r10_bio->devs[i].bio = bio;  in raid10_write_request()
1531 			atomic_inc(&rdev->nr_pending);  in raid10_write_request()
1534 			r10_bio->devs[i].repl_bio = bio;  in raid10_write_request()
1535 			atomic_inc(&rrdev->nr_pending);  in raid10_write_request()
1540 	if (max_sectors < r10_bio->sectors)  in raid10_write_request()
1541 		r10_bio->sectors = max_sectors;  in raid10_write_request()
1543 	if (r10_bio->sectors < bio_sectors(bio)) {  in raid10_write_request()
1544 		struct bio *split = bio_split(bio, r10_bio->sectors,  in raid10_write_request()
1545 					      GFP_NOIO, &conf->bio_split);  in raid10_write_request()
1551 		r10_bio->master_bio = bio;  in raid10_write_request()
1555 	r10_bio->master_bio = bio;  in raid10_write_request()
1556 	atomic_set(&r10_bio->remaining, 1);  in raid10_write_request()
1557 	md_bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);  in raid10_write_request()
1559 	for (i = 0; i < conf->copies; i++) {  in raid10_write_request()
1560 		if (r10_bio->devs[i].bio)  in raid10_write_request()
1562 		if (r10_bio->devs[i].repl_bio)  in raid10_write_request()
1570 	struct r10conf *conf = mddev->private;  in __make_request()
1573 	r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO);  in __make_request()
1575 	r10_bio->master_bio = bio;  in __make_request()
1576 	r10_bio->sectors = sectors;  in __make_request()
1578 	r10_bio->mddev = mddev;  in __make_request()
1579 	r10_bio->sector = bio->bi_iter.bi_sector;  in __make_request()
1580 	r10_bio->state = 0;  in __make_request()
1581 	r10_bio->read_slot = -1;  in __make_request()
1582 	memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) *  in __make_request()
1583 			conf->geo.raid_disks);  in __make_request()
1593 	struct r10conf *conf = r10bio->mddev->private;  in raid_end_discard_bio()
1596 	while (atomic_dec_and_test(&r10bio->remaining)) {  in raid_end_discard_bio()
1600 		if (!test_bit(R10BIO_Discard, &r10bio->state)) {  in raid_end_discard_bio()
1601 			first_r10bio = (struct r10bio *)r10bio->master_bio;  in raid_end_discard_bio()
1605 			md_write_end(r10bio->mddev);  in raid_end_discard_bio()
1606 			bio_endio(r10bio->master_bio);  in raid_end_discard_bio()
1615 	struct r10bio *r10_bio = bio->bi_private;  in raid10_end_discard_request()
1616 	struct r10conf *conf = r10_bio->mddev->private;  in raid10_end_discard_request()
1624 	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))  in raid10_end_discard_request()
1625 		set_bit(R10BIO_Uptodate, &r10_bio->state);  in raid10_end_discard_request()
1629 		rdev = conf->mirrors[dev].replacement;  in raid10_end_discard_request()
1637 		rdev = conf->mirrors[dev].rdev;  in raid10_end_discard_request()
1641 	rdev_dec_pending(rdev, conf->mddev);  in raid10_end_discard_request()
1652 	struct r10conf *conf = mddev->private;  in raid10_handle_discard()
1653 	struct geom *geo = &conf->geo;  in raid10_handle_discard()
1654 	int far_copies = geo->far_copies;  in raid10_handle_discard()
1671 	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))  in raid10_handle_discard()
1672 		return -EAGAIN;  in raid10_handle_discard()
1674 	if (WARN_ON_ONCE(bio->bi_opf & REQ_NOWAIT)) {  in raid10_handle_discard()
1684 	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))  in raid10_handle_discard()
1687 	if (geo->near_copies)  in raid10_handle_discard()
1688 		stripe_data_disks = geo->raid_disks / geo->near_copies +  in raid10_handle_discard()
1689 					geo->raid_disks % geo->near_copies;  in raid10_handle_discard()
1691 		stripe_data_disks = geo->raid_disks;  in raid10_handle_discard()
1693 	stripe_size = stripe_data_disks << geo->chunk_shift;  in raid10_handle_discard()
1695 	bio_start = bio->bi_iter.bi_sector;  in raid10_handle_discard()
1714 		split_size = stripe_size - remainder;  in raid10_handle_discard()
1715 		split = bio_split(bio, split_size, GFP_NOIO, &conf->bio_split);  in raid10_handle_discard()
1724 		split_size = bio_sectors(bio) - remainder;  in raid10_handle_discard()
1725 		split = bio_split(bio, split_size, GFP_NOIO, &conf->bio_split);  in raid10_handle_discard()
1734 	bio_start = bio->bi_iter.bi_sector;  in raid10_handle_discard()
1742 	chunk = bio_start >> geo->chunk_shift;  in raid10_handle_discard()
1743 	chunk *= geo->near_copies;  in raid10_handle_discard()
1745 	start_disk_index = sector_div(first_stripe_index, geo->raid_disks);  in raid10_handle_discard()
1746 	if (geo->far_offset)  in raid10_handle_discard()
1747 		first_stripe_index *= geo->far_copies;  in raid10_handle_discard()
1748 	start_disk_offset = (bio_start & geo->chunk_mask) +  in raid10_handle_discard()
1749 				(first_stripe_index << geo->chunk_shift);  in raid10_handle_discard()
1751 	chunk = bio_end >> geo->chunk_shift;  in raid10_handle_discard()
1752 	chunk *= geo->near_copies;  in raid10_handle_discard()
1754 	end_disk_index = sector_div(last_stripe_index, geo->raid_disks);  in raid10_handle_discard()
1755 	if (geo->far_offset)  in raid10_handle_discard()
1756 		last_stripe_index *= geo->far_copies;  in raid10_handle_discard()
1757 	end_disk_offset = (bio_end & geo->chunk_mask) +  in raid10_handle_discard()
1758 				(last_stripe_index << geo->chunk_shift);  in raid10_handle_discard()
1761 	r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO);  in raid10_handle_discard()
1762 	r10_bio->mddev = mddev;  in raid10_handle_discard()
1763 	r10_bio->state = 0;  in raid10_handle_discard()
1764 	r10_bio->sectors = 0;  in raid10_handle_discard()
1765 	memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * geo->raid_disks);  in raid10_handle_discard()
1770 	 * Inspired by raid10_sync_request, we can use the first r10bio->master_bio  in raid10_handle_discard()
1771 	 * to record the discard bio. Other r10bio->master_bio record the first  in raid10_handle_discard()
1776 		r10_bio->master_bio = bio;  in raid10_handle_discard()
1777 		set_bit(R10BIO_Discard, &r10_bio->state);  in raid10_handle_discard()
1781 		r10_bio->master_bio = (struct bio *)first_r10bio;  in raid10_handle_discard()
1789 	for (disk = 0; disk < geo->raid_disks; disk++) {  in raid10_handle_discard()
1792 		rdev = dereference_rdev_and_rrdev(&conf->mirrors[disk], &rrdev);  in raid10_handle_discard()
1793 		r10_bio->devs[disk].bio = NULL;  in raid10_handle_discard()
1794 		r10_bio->devs[disk].repl_bio = NULL;  in raid10_handle_discard()
1796 		if (rdev && (test_bit(Faulty, &rdev->flags)))  in raid10_handle_discard()
1798 		if (rrdev && (test_bit(Faulty, &rrdev->flags)))  in raid10_handle_discard()
1804 			r10_bio->devs[disk].bio = bio;  in raid10_handle_discard()
1805 			atomic_inc(&rdev->nr_pending);  in raid10_handle_discard()
1808 			r10_bio->devs[disk].repl_bio = bio;  in raid10_handle_discard()
1809 			atomic_inc(&rrdev->nr_pending);  in raid10_handle_discard()
1814 	atomic_set(&r10_bio->remaining, 1);  in raid10_handle_discard()
1815 	for (disk = 0; disk < geo->raid_disks; disk++) {  in raid10_handle_discard()
1832 			dev_start = (first_stripe_index + 1) * mddev->chunk_sectors;  in raid10_handle_discard()
1834 			dev_start = first_stripe_index * mddev->chunk_sectors;  in raid10_handle_discard()
1839 			dev_end = (last_stripe_index + 1) * mddev->chunk_sectors;  in raid10_handle_discard()
1841 			dev_end = last_stripe_index * mddev->chunk_sectors;  in raid10_handle_discard()
1849 		 * add rdev->nr_pending in the first loop.  in raid10_handle_discard()
1851 		if (r10_bio->devs[disk].bio) {  in raid10_handle_discard()
1852 			struct md_rdev *rdev = conf->mirrors[disk].rdev;  in raid10_handle_discard()
1853 			mbio = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO,  in raid10_handle_discard()
1854 					       &mddev->bio_set);  in raid10_handle_discard()
1855 			mbio->bi_end_io = raid10_end_discard_request;  in raid10_handle_discard()
1856 			mbio->bi_private = r10_bio;  in raid10_handle_discard()
1857 			r10_bio->devs[disk].bio = mbio;  in raid10_handle_discard()
1858 			r10_bio->devs[disk].devnum = disk;  in raid10_handle_discard()
1859 			atomic_inc(&r10_bio->remaining);  in raid10_handle_discard()
1862 					dev_end - dev_start);  in raid10_handle_discard()
1865 		if (r10_bio->devs[disk].repl_bio) {  in raid10_handle_discard()
1866 			struct md_rdev *rrdev = conf->mirrors[disk].replacement;  in raid10_handle_discard()
1867 			rbio = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO,  in raid10_handle_discard()
1868 					       &mddev->bio_set);  in raid10_handle_discard()
1869 			rbio->bi_end_io = raid10_end_discard_request;  in raid10_handle_discard()
1870 			rbio->bi_private = r10_bio;  in raid10_handle_discard()
1871 			r10_bio->devs[disk].repl_bio = rbio;  in raid10_handle_discard()
1872 			r10_bio->devs[disk].devnum = disk;  in raid10_handle_discard()
1873 			atomic_inc(&r10_bio->remaining);  in raid10_handle_discard()
1876 					dev_end - dev_start);  in raid10_handle_discard()
1881 	if (!geo->far_offset && --far_copies) {  in raid10_handle_discard()
1882 		first_stripe_index += geo->stride >> geo->chunk_shift;  in raid10_handle_discard()
1883 		start_disk_offset += geo->stride;  in raid10_handle_discard()
1884 		last_stripe_index += geo->stride >> geo->chunk_shift;  in raid10_handle_discard()
1885 		end_disk_offset += geo->stride;  in raid10_handle_discard()
1886 		atomic_inc(&first_r10bio->remaining);  in raid10_handle_discard()
1897 	return -EAGAIN;  in raid10_handle_discard()
1902 	struct r10conf *conf = mddev->private;  in raid10_make_request()
1903 	sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);  in raid10_make_request()
1907 	if (unlikely(bio->bi_opf & REQ_PREFLUSH)  in raid10_make_request()
1922 	if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +  in raid10_make_request()
1924 		     && (conf->geo.near_copies < conf->geo.raid_disks  in raid10_make_request()
1925 			 || conf->prev.near_copies <  in raid10_make_request()
1926 			 conf->prev.raid_disks)))  in raid10_make_request()
1927 		sectors = chunk_sects -  in raid10_make_request()
1928 			(bio->bi_iter.bi_sector &  in raid10_make_request()
1929 			 (chunk_sects - 1));  in raid10_make_request()
1939 	struct r10conf *conf = mddev->private;  in raid10_status()
1942 	if (conf->geo.near_copies < conf->geo.raid_disks)  in raid10_status()
1943 		seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);  in raid10_status()
1944 	if (conf->geo.near_copies > 1)  in raid10_status()
1945 		seq_printf(seq, " %d near-copies", conf->geo.near_copies);  in raid10_status()
1946 	if (conf->geo.far_copies > 1) {  in raid10_status()
1947 		if (conf->geo.far_offset)  in raid10_status()
1948 			seq_printf(seq, " %d offset-copies", conf->geo.far_copies);  in raid10_status()
1950 			seq_printf(seq, " %d far-copies", conf->geo.far_copies);  in raid10_status()
1951 		if (conf->geo.far_set_size != conf->geo.raid_disks)  in raid10_status()
1952 			seq_printf(seq, " %d devices per set", conf->geo.far_set_size);  in raid10_status()
1954 	seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks,  in raid10_status()
1955 					conf->geo.raid_disks - mddev->degraded);  in raid10_status()
1957 	for (i = 0; i < conf->geo.raid_disks; i++) {  in raid10_status()
1958 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);  in raid10_status()
1959 		seq_printf(seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");  in raid10_status()
1976 		disks = conf->prev.raid_disks;  in _enough()
1977 		ncopies = conf->prev.near_copies;  in _enough()
1979 		disks = conf->geo.raid_disks;  in _enough()
1980 		ncopies = conf->geo.near_copies;  in _enough()
1985 		int n = conf->copies;  in _enough()
1988 		while (n--) {  in _enough()
1991 			    (rdev = rcu_dereference(conf->mirrors[this].rdev)) &&  in _enough()
1992 			    test_bit(In_sync, &rdev->flags))  in _enough()
2010 	 * This is ensured if ->reconfig_mutex or ->device_lock  in enough()
2018  * raid10_error() - RAID10 error handler.
2024  *	- &MD_BROKEN flag is set in &mddev->flags.
2026  *	- recovery is interrupted.
2027  *	- &mddev->degraded is bumped.
2030  * &mddev->fail_last_dev is off.
2034 	struct r10conf *conf = mddev->private;  in raid10_error()
2037 	spin_lock_irqsave(&conf->device_lock, flags);  in raid10_error()
2039 	if (test_bit(In_sync, &rdev->flags) && !enough(conf, rdev->raid_disk)) {  in raid10_error()
2040 		set_bit(MD_BROKEN, &mddev->flags);  in raid10_error()
2042 		if (!mddev->fail_last_dev) {  in raid10_error()
2043 			spin_unlock_irqrestore(&conf->device_lock, flags);  in raid10_error()
2047 	if (test_and_clear_bit(In_sync, &rdev->flags))  in raid10_error()
2048 		mddev->degraded++;  in raid10_error()
2050 	set_bit(MD_RECOVERY_INTR, &mddev->recovery);  in raid10_error()
2051 	set_bit(Blocked, &rdev->flags);  in raid10_error()
2052 	set_bit(Faulty, &rdev->flags);  in raid10_error()
2053 	set_mask_bits(&mddev->sb_flags, 0,  in raid10_error()
2055 	spin_unlock_irqrestore(&conf->device_lock, flags);  in raid10_error()
2058 		mdname(mddev), rdev->bdev,  in raid10_error()
2059 		mdname(mddev), conf->geo.raid_disks - mddev->degraded);  in raid10_error()
2072 	pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,  in print_conf()
2073 		 conf->geo.raid_disks);  in print_conf()
2075 	/* This is only called with ->reconfix_mutex held, so  in print_conf()
2077 	for (i = 0; i < conf->geo.raid_disks; i++) {  in print_conf()
2078 		rdev = conf->mirrors[i].rdev;  in print_conf()
2081 				 i, !test_bit(In_sync, &rdev->flags),  in print_conf()
2082 				 !test_bit(Faulty, &rdev->flags),  in print_conf()
2083 				 rdev->bdev);  in print_conf()
2092 	mempool_exit(&conf->r10buf_pool);  in close_sync()
2098 	struct r10conf *conf = mddev->private;  in raid10_spare_active()
2104 	 * Find all non-in_sync disks within the RAID10 configuration  in raid10_spare_active()
2107 	for (i = 0; i < conf->geo.raid_disks; i++) {  in raid10_spare_active()
2108 		tmp = conf->mirrors + i;  in raid10_spare_active()
2109 		if (tmp->replacement  in raid10_spare_active()
2110 		    && tmp->replacement->recovery_offset == MaxSector  in raid10_spare_active()
2111 		    && !test_bit(Faulty, &tmp->replacement->flags)  in raid10_spare_active()
2112 		    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {  in raid10_spare_active()
2114 			if (!tmp->rdev  in raid10_spare_active()
2115 			    || !test_and_clear_bit(In_sync, &tmp->rdev->flags))  in raid10_spare_active()
2117 			if (tmp->rdev) {  in raid10_spare_active()
2120 				 * and never re-added.  in raid10_spare_active()
2122 				set_bit(Faulty, &tmp->rdev->flags);  in raid10_spare_active()
2124 					tmp->rdev->sysfs_state);  in raid10_spare_active()
2126 			sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);  in raid10_spare_active()
2127 		} else if (tmp->rdev  in raid10_spare_active()
2128 			   && tmp->rdev->recovery_offset == MaxSector  in raid10_spare_active()
2129 			   && !test_bit(Faulty, &tmp->rdev->flags)  in raid10_spare_active()
2130 			   && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {  in raid10_spare_active()
2132 			sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);  in raid10_spare_active()
2135 	spin_lock_irqsave(&conf->device_lock, flags);  in raid10_spare_active()
2136 	mddev->degraded -= count;  in raid10_spare_active()
2137 	spin_unlock_irqrestore(&conf->device_lock, flags);  in raid10_spare_active()
2145 	struct r10conf *conf = mddev->private;  in raid10_add_disk()
2146 	int err = -EEXIST;  in raid10_add_disk()
2147 	int mirror, repl_slot = -1;  in raid10_add_disk()
2149 	int last = conf->geo.raid_disks - 1;  in raid10_add_disk()
2152 	if (mddev->recovery_cp < MaxSector)  in raid10_add_disk()
2153 		/* only hot-add to in-sync arrays, as recovery is  in raid10_add_disk()
2156 		return -EBUSY;  in raid10_add_disk()
2157 	if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1))  in raid10_add_disk()
2158 		return -EINVAL;  in raid10_add_disk()
2161 		return -ENXIO;  in raid10_add_disk()
2163 	if (rdev->raid_disk >= 0)  in raid10_add_disk()
2164 		first = last = rdev->raid_disk;  in raid10_add_disk()
2166 	if (rdev->saved_raid_disk >= first &&  in raid10_add_disk()
2167 	    rdev->saved_raid_disk < conf->geo.raid_disks &&  in raid10_add_disk()
2168 	    conf->mirrors[rdev->saved_raid_disk].rdev == NULL)  in raid10_add_disk()
2169 		mirror = rdev->saved_raid_disk;  in raid10_add_disk()
2173 		p = &conf->mirrors[mirror];  in raid10_add_disk()
2174 		if (p->recovery_disabled == mddev->recovery_disabled)  in raid10_add_disk()
2176 		if (p->rdev) {  in raid10_add_disk()
2177 			if (test_bit(WantReplacement, &p->rdev->flags) &&  in raid10_add_disk()
2178 			    p->replacement == NULL && repl_slot < 0)  in raid10_add_disk()
2183 		if (mddev->gendisk)  in raid10_add_disk()
2184 			disk_stack_limits(mddev->gendisk, rdev->bdev,  in raid10_add_disk()
2185 					  rdev->data_offset << 9);  in raid10_add_disk()
2187 		p->head_position = 0;  in raid10_add_disk()
2188 		p->recovery_disabled = mddev->recovery_disabled - 1;  in raid10_add_disk()
2189 		rdev->raid_disk = mirror;  in raid10_add_disk()
2191 		if (rdev->saved_raid_disk != mirror)  in raid10_add_disk()
2192 			conf->fullsync = 1;  in raid10_add_disk()
2193 		rcu_assign_pointer(p->rdev, rdev);  in raid10_add_disk()
2198 		p = &conf->mirrors[repl_slot];  in raid10_add_disk()
2199 		clear_bit(In_sync, &rdev->flags);  in raid10_add_disk()
2200 		set_bit(Replacement, &rdev->flags);  in raid10_add_disk()
2201 		rdev->raid_disk = repl_slot;  in raid10_add_disk()
2203 		if (mddev->gendisk)  in raid10_add_disk()
2204 			disk_stack_limits(mddev->gendisk, rdev->bdev,  in raid10_add_disk()
2205 					  rdev->data_offset << 9);  in raid10_add_disk()
2206 		conf->fullsync = 1;  in raid10_add_disk()
2207 		rcu_assign_pointer(p->replacement, rdev);  in raid10_add_disk()
2216 	struct r10conf *conf = mddev->private;  in raid10_remove_disk()
2218 	int number = rdev->raid_disk;  in raid10_remove_disk()
2223 	if (unlikely(number >= mddev->raid_disks))  in raid10_remove_disk()
2225 	p = conf->mirrors + number;  in raid10_remove_disk()
2226 	if (rdev == p->rdev)  in raid10_remove_disk()
2227 		rdevp = &p->rdev;  in raid10_remove_disk()
2228 	else if (rdev == p->replacement)  in raid10_remove_disk()
2229 		rdevp = &p->replacement;  in raid10_remove_disk()
2233 	if (test_bit(In_sync, &rdev->flags) ||  in raid10_remove_disk()
2234 	    atomic_read(&rdev->nr_pending)) {  in raid10_remove_disk()
2235 		err = -EBUSY;  in raid10_remove_disk()
2238 	/* Only remove non-faulty devices if recovery  in raid10_remove_disk()
2241 	if (!test_bit(Faulty, &rdev->flags) &&  in raid10_remove_disk()
2242 	    mddev->recovery_disabled != p->recovery_disabled &&  in raid10_remove_disk()
2243 	    (!p->replacement || p->replacement == rdev) &&  in raid10_remove_disk()
2244 	    number < conf->geo.raid_disks &&  in raid10_remove_disk()
2245 	    enough(conf, -1)) {  in raid10_remove_disk()
2246 		err = -EBUSY;  in raid10_remove_disk()
2250 	if (!test_bit(RemoveSynchronized, &rdev->flags)) {  in raid10_remove_disk()
2252 		if (atomic_read(&rdev->nr_pending)) {  in raid10_remove_disk()
2254 			err = -EBUSY;  in raid10_remove_disk()
2259 	if (p->replacement) {  in raid10_remove_disk()
2261 		p->rdev = p->replacement;  in raid10_remove_disk()
2262 		clear_bit(Replacement, &p->replacement->flags);  in raid10_remove_disk()
2264 			   * but will never see neither -- if they are careful.  in raid10_remove_disk()
2266 		p->replacement = NULL;  in raid10_remove_disk()
2269 	clear_bit(WantReplacement, &rdev->flags);  in raid10_remove_disk()
2280 	struct r10conf *conf = r10_bio->mddev->private;  in __end_sync_read()
2282 	if (!bio->bi_status)  in __end_sync_read()
2283 		set_bit(R10BIO_Uptodate, &r10_bio->state);  in __end_sync_read()
2288 		atomic_add(r10_bio->sectors,  in __end_sync_read()
2289 			   &conf->mirrors[d].rdev->corrected_errors);  in __end_sync_read()
2294 	rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);  in __end_sync_read()
2295 	if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||  in __end_sync_read()
2296 	    atomic_dec_and_test(&r10_bio->remaining)) {  in __end_sync_read()
2307 	struct r10conf *conf = r10_bio->mddev->private;  in end_sync_read()
2316 	struct r10bio *r10_bio = bio->bi_private;  in end_reshape_read()
2318 	__end_sync_read(r10_bio, bio, r10_bio->read_slot);  in end_reshape_read()
2323 	struct mddev *mddev = r10_bio->mddev;  in end_sync_request()
2325 	while (atomic_dec_and_test(&r10_bio->remaining)) {  in end_sync_request()
2326 		if (r10_bio->master_bio == NULL) {  in end_sync_request()
2328 			sector_t s = r10_bio->sectors;  in end_sync_request()
2329 			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||  in end_sync_request()
2330 			    test_bit(R10BIO_WriteError, &r10_bio->state))  in end_sync_request()
2337 			struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;  in end_sync_request()
2338 			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||  in end_sync_request()
2339 			    test_bit(R10BIO_WriteError, &r10_bio->state))  in end_sync_request()
2351 	struct mddev *mddev = r10_bio->mddev;  in end_sync_write()
2352 	struct r10conf *conf = mddev->private;  in end_sync_write()
2362 		rdev = conf->mirrors[d].replacement;  in end_sync_write()
2364 		rdev = conf->mirrors[d].rdev;  in end_sync_write()
2366 	if (bio->bi_status) {  in end_sync_write()
2370 			set_bit(WriteErrorSeen, &rdev->flags);  in end_sync_write()
2371 			if (!test_and_set_bit(WantReplacement, &rdev->flags))  in end_sync_write()
2373 					&rdev->mddev->recovery);  in end_sync_write()
2374 			set_bit(R10BIO_WriteError, &r10_bio->state);  in end_sync_write()
2377 			     r10_bio->devs[slot].addr,  in end_sync_write()
2378 			     r10_bio->sectors,  in end_sync_write()
2380 		set_bit(R10BIO_MadeGood, &r10_bio->state);  in end_sync_write()
2400  * We check if all blocks are in-sync and only write to blocks that
2405 	struct r10conf *conf = mddev->private;  in sync_request_write()
2411 	atomic_set(&r10_bio->remaining, 1);  in sync_request_write()
2414 	for (i=0; i<conf->copies; i++)  in sync_request_write()
2415 		if (!r10_bio->devs[i].bio->bi_status)  in sync_request_write()
2418 	if (i == conf->copies)  in sync_request_write()
2422 	fbio = r10_bio->devs[i].bio;  in sync_request_write()
2423 	fbio->bi_iter.bi_size = r10_bio->sectors << 9;  in sync_request_write()
2424 	fbio->bi_iter.bi_idx = 0;  in sync_request_write()
2425 	fpages = get_resync_pages(fbio)->pages;  in sync_request_write()
2427 	vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);  in sync_request_write()
2429 	for (i=0 ; i < conf->copies ; i++) {  in sync_request_write()
2434 		tbio = r10_bio->devs[i].bio;  in sync_request_write()
2436 		if (tbio->bi_end_io != end_sync_read)  in sync_request_write()
2441 		tpages = get_resync_pages(tbio)->pages;  in sync_request_write()
2442 		d = r10_bio->devs[i].devnum;  in sync_request_write()
2443 		rdev = conf->mirrors[d].rdev;  in sync_request_write()
2444 		if (!r10_bio->devs[i].bio->bi_status) {  in sync_request_write()
2449 			int sectors = r10_bio->sectors;  in sync_request_write()
2458 				sectors -= len/512;  in sync_request_write()
2462 			atomic64_add(r10_bio->sectors, &mddev->resync_mismatches);  in sync_request_write()
2463 			if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))  in sync_request_write()
2466 		} else if (test_bit(FailFast, &rdev->flags)) {  in sync_request_write()
2468 			md_error(rdev->mddev, rdev);  in sync_request_write()
2473 		 * First we need to fixup bv_offset, bv_len and  in sync_request_write()
2477 		bio_reset(tbio, conf->mirrors[d].rdev->bdev, REQ_OP_WRITE);  in sync_request_write()
2479 		md_bio_reset_resync_pages(tbio, rp, fbio->bi_iter.bi_size);  in sync_request_write()
2481 		rp->raid_bio = r10_bio;  in sync_request_write()
2482 		tbio->bi_private = rp;  in sync_request_write()
2483 		tbio->bi_iter.bi_sector = r10_bio->devs[i].addr;  in sync_request_write()
2484 		tbio->bi_end_io = end_sync_write;  in sync_request_write()
2488 		atomic_inc(&conf->mirrors[d].rdev->nr_pending);  in sync_request_write()
2489 		atomic_inc(&r10_bio->remaining);  in sync_request_write()
2490 		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio));  in sync_request_write()
2492 		if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))  in sync_request_write()
2493 			tbio->bi_opf |= MD_FAILFAST;  in sync_request_write()
2494 		tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;  in sync_request_write()
2501 	for (i = 0; i < conf->copies; i++) {  in sync_request_write()
2504 		tbio = r10_bio->devs[i].repl_bio;  in sync_request_write()
2505 		if (!tbio || !tbio->bi_end_io)  in sync_request_write()
2507 		if (r10_bio->devs[i].bio->bi_end_io != end_sync_write  in sync_request_write()
2508 		    && r10_bio->devs[i].bio != fbio)  in sync_request_write()
2510 		d = r10_bio->devs[i].devnum;  in sync_request_write()
2511 		atomic_inc(&r10_bio->remaining);  in sync_request_write()
2512 		md_sync_acct(conf->mirrors[d].replacement->bdev,  in sync_request_write()
2518 	if (atomic_dec_and_test(&r10_bio->remaining)) {  in sync_request_write()
2519 		md_done_sync(mddev, r10_bio->sectors, 1);  in sync_request_write()
2527  * We recover all non-is_sync drives by finding the virtual address of
2529  * There is a separate r10_bio for each non-in_sync drive.
2537 	 * We repeat the read in smaller page-sized sections.  in fix_recovery_read_error()
2543 	struct mddev *mddev = r10_bio->mddev;  in fix_recovery_read_error()
2544 	struct r10conf *conf = mddev->private;  in fix_recovery_read_error()
2545 	struct bio *bio = r10_bio->devs[0].bio;  in fix_recovery_read_error()
2547 	int sectors = r10_bio->sectors;  in fix_recovery_read_error()
2549 	int dr = r10_bio->devs[0].devnum;  in fix_recovery_read_error()
2550 	int dw = r10_bio->devs[1].devnum;  in fix_recovery_read_error()
2551 	struct page **pages = get_resync_pages(bio)->pages;  in fix_recovery_read_error()
2562 		rdev = conf->mirrors[dr].rdev;  in fix_recovery_read_error()
2563 		addr = r10_bio->devs[0].addr + sect,  in fix_recovery_read_error()
2570 			rdev = conf->mirrors[dw].rdev;  in fix_recovery_read_error()
2571 			addr = r10_bio->devs[1].addr + sect;  in fix_recovery_read_error()
2578 				set_bit(WriteErrorSeen, &rdev->flags);  in fix_recovery_read_error()
2580 						      &rdev->flags))  in fix_recovery_read_error()
2582 						&rdev->mddev->recovery);  in fix_recovery_read_error()
2586 			/* We don't worry if we cannot set a bad block -  in fix_recovery_read_error()
2592 			if (rdev != conf->mirrors[dw].rdev) {  in fix_recovery_read_error()
2594 				struct md_rdev *rdev2 = conf->mirrors[dw].rdev;  in fix_recovery_read_error()
2595 				addr = r10_bio->devs[1].addr + sect;  in fix_recovery_read_error()
2602 					conf->mirrors[dw].recovery_disabled  in fix_recovery_read_error()
2603 						= mddev->recovery_disabled;  in fix_recovery_read_error()
2605 						&mddev->recovery);  in fix_recovery_read_error()
2611 		sectors -= s;  in fix_recovery_read_error()
2619 	struct r10conf *conf = mddev->private;  in recovery_request_write()
2621 	struct bio *wbio = r10_bio->devs[1].bio;  in recovery_request_write()
2622 	struct bio *wbio2 = r10_bio->devs[1].repl_bio;  in recovery_request_write()
2624 	/* Need to test wbio2->bi_end_io before we call  in recovery_request_write()
2628 	if (wbio2 && !wbio2->bi_end_io)  in recovery_request_write()
2631 	if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {  in recovery_request_write()
2633 		if (wbio->bi_end_io)  in recovery_request_write()
2644 	d = r10_bio->devs[1].devnum;  in recovery_request_write()
2645 	if (wbio->bi_end_io) {  in recovery_request_write()
2646 		atomic_inc(&conf->mirrors[d].rdev->nr_pending);  in recovery_request_write()
2647 		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio));  in recovery_request_write()
2651 		atomic_inc(&conf->mirrors[d].replacement->nr_pending);  in recovery_request_write()
2652 		md_sync_acct(conf->mirrors[d].replacement->bdev,  in recovery_request_write()
2668 	unsigned int read_errors = atomic_read(&rdev->read_errors);  in check_decay_read_errors()
2672 	if (rdev->last_read_error == 0) {  in check_decay_read_errors()
2674 		rdev->last_read_error = cur_time_mon;  in check_decay_read_errors()
2678 	hours_since_last = (long)(cur_time_mon -  in check_decay_read_errors()
2679 			    rdev->last_read_error) / 3600;  in check_decay_read_errors()
2681 	rdev->last_read_error = cur_time_mon;  in check_decay_read_errors()
2689 		atomic_set(&rdev->read_errors, 0);  in check_decay_read_errors()
2691 		atomic_set(&rdev->read_errors, read_errors >> hours_since_last);  in check_decay_read_errors()
2701 	    && (op == REQ_OP_READ || test_bit(WriteErrorSeen, &rdev->flags)))  in r10_sync_page_io()
2702 		return -1;  in r10_sync_page_io()
2707 		set_bit(WriteErrorSeen, &rdev->flags);  in r10_sync_page_io()
2708 		if (!test_and_set_bit(WantReplacement, &rdev->flags))  in r10_sync_page_io()
2710 				&rdev->mddev->recovery);  in r10_sync_page_io()
2712 	/* need to record an error - either for the block or the device */  in r10_sync_page_io()
2714 		md_error(rdev->mddev, rdev);  in r10_sync_page_io()
2728 	int sect = 0; /* Offset from r10_bio->sector */  in fix_read_error()
2729 	int sectors = r10_bio->sectors, slot = r10_bio->read_slot;  in fix_read_error()
2731 	int max_read_errors = atomic_read(&mddev->max_corr_read_errors);  in fix_read_error()
2732 	int d = r10_bio->devs[slot].devnum;  in fix_read_error()
2737 	rdev = conf->mirrors[d].rdev;  in fix_read_error()
2739 	if (test_bit(Faulty, &rdev->flags))  in fix_read_error()
2745 	atomic_inc(&rdev->read_errors);  in fix_read_error()
2746 	if (atomic_read(&rdev->read_errors) > max_read_errors) {  in fix_read_error()
2748 			  mdname(mddev), rdev->bdev,  in fix_read_error()
2749 			  atomic_read(&rdev->read_errors), max_read_errors);  in fix_read_error()
2751 			  mdname(mddev), rdev->bdev);  in fix_read_error()
2753 		r10_bio->devs[slot].bio = IO_BLOCKED;  in fix_read_error()
2771 			d = r10_bio->devs[sl].devnum;  in fix_read_error()
2772 			rdev = rcu_dereference(conf->mirrors[d].rdev);  in fix_read_error()
2774 			    test_bit(In_sync, &rdev->flags) &&  in fix_read_error()
2775 			    !test_bit(Faulty, &rdev->flags) &&  in fix_read_error()
2776 			    is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,  in fix_read_error()
2778 				atomic_inc(&rdev->nr_pending);  in fix_read_error()
2781 						       r10_bio->devs[sl].addr +  in fix_read_error()
2784 						       conf->tmppage,  in fix_read_error()
2792 			if (sl == conf->copies)  in fix_read_error()
2802 			int dn = r10_bio->devs[slot].devnum;  in fix_read_error()
2803 			rdev = conf->mirrors[dn].rdev;  in fix_read_error()
2807 				    r10_bio->devs[slot].addr  in fix_read_error()
2811 				r10_bio->devs[slot].bio  in fix_read_error()
2818 		/* write it back and re-read */  in fix_read_error()
2822 				sl = conf->copies;  in fix_read_error()
2823 			sl--;  in fix_read_error()
2824 			d = r10_bio->devs[sl].devnum;  in fix_read_error()
2825 			rdev = rcu_dereference(conf->mirrors[d].rdev);  in fix_read_error()
2827 			    test_bit(Faulty, &rdev->flags) ||  in fix_read_error()
2828 			    !test_bit(In_sync, &rdev->flags))  in fix_read_error()
2831 			atomic_inc(&rdev->nr_pending);  in fix_read_error()
2834 					     r10_bio->devs[sl].addr +  in fix_read_error()
2836 					     s, conf->tmppage, REQ_OP_WRITE)  in fix_read_error()
2845 					  rdev->bdev);  in fix_read_error()
2848 					  rdev->bdev);  in fix_read_error()
2856 				sl = conf->copies;  in fix_read_error()
2857 			sl--;  in fix_read_error()
2858 			d = r10_bio->devs[sl].devnum;  in fix_read_error()
2859 			rdev = rcu_dereference(conf->mirrors[d].rdev);  in fix_read_error()
2861 			    test_bit(Faulty, &rdev->flags) ||  in fix_read_error()
2862 			    !test_bit(In_sync, &rdev->flags))  in fix_read_error()
2865 			atomic_inc(&rdev->nr_pending);  in fix_read_error()
2868 					     r10_bio->devs[sl].addr +  in fix_read_error()
2870 					     s, conf->tmppage, REQ_OP_READ)) {  in fix_read_error()
2878 				       rdev->bdev);  in fix_read_error()
2881 				       rdev->bdev);  in fix_read_error()
2889 				       rdev->bdev);  in fix_read_error()
2890 				atomic_add(s, &rdev->corrected_errors);  in fix_read_error()
2898 		sectors -= s;  in fix_read_error()
2905 	struct bio *bio = r10_bio->master_bio;  in narrow_write_error()
2906 	struct mddev *mddev = r10_bio->mddev;  in narrow_write_error()
2907 	struct r10conf *conf = mddev->private;  in narrow_write_error()
2908 	struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;  in narrow_write_error()
2923 	int sect_to_write = r10_bio->sectors;  in narrow_write_error()
2926 	if (rdev->badblocks.shift < 0)  in narrow_write_error()
2929 	block_sectors = roundup(1 << rdev->badblocks.shift,  in narrow_write_error()
2930 				bdev_logical_block_size(rdev->bdev) >> 9);  in narrow_write_error()
2931 	sector = r10_bio->sector;  in narrow_write_error()
2932 	sectors = ((r10_bio->sector + block_sectors)  in narrow_write_error()
2933 		   & ~(sector_t)(block_sectors - 1))  in narrow_write_error()
2934 		- sector;  in narrow_write_error()
2942 		wbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO,  in narrow_write_error()
2943 				       &mddev->bio_set);  in narrow_write_error()
2944 		bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors);  in narrow_write_error()
2945 		wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector);  in narrow_write_error()
2946 		wbio->bi_iter.bi_sector = wsector +  in narrow_write_error()
2948 		wbio->bi_opf = REQ_OP_WRITE;  in narrow_write_error()
2957 		sect_to_write -= sectors;  in narrow_write_error()
2966 	int slot = r10_bio->read_slot;  in handle_read_error()
2968 	struct r10conf *conf = mddev->private;  in handle_read_error()
2969 	struct md_rdev *rdev = r10_bio->devs[slot].rdev;  in handle_read_error()
2974 	 * other devices.  When we find one, we re-write  in handle_read_error()
2979 	bio = r10_bio->devs[slot].bio;  in handle_read_error()
2981 	r10_bio->devs[slot].bio = NULL;  in handle_read_error()
2983 	if (mddev->ro)  in handle_read_error()
2984 		r10_bio->devs[slot].bio = IO_BLOCKED;  in handle_read_error()
2985 	else if (!test_bit(FailFast, &rdev->flags)) {  in handle_read_error()
2993 	r10_bio->state = 0;  in handle_read_error()
2994 	raid10_read_request(mddev, r10_bio->master_bio, r10_bio, false);  in handle_read_error()
2996 	 * allow_barrier after re-submit to ensure no sync io  in handle_read_error()
3013 	if (test_bit(R10BIO_IsSync, &r10_bio->state) ||  in handle_write_completed()
3014 	    test_bit(R10BIO_IsRecover, &r10_bio->state)) {  in handle_write_completed()
3015 		for (m = 0; m < conf->copies; m++) {  in handle_write_completed()
3016 			int dev = r10_bio->devs[m].devnum;  in handle_write_completed()
3017 			rdev = conf->mirrors[dev].rdev;  in handle_write_completed()
3018 			if (r10_bio->devs[m].bio == NULL ||  in handle_write_completed()
3019 				r10_bio->devs[m].bio->bi_end_io == NULL)  in handle_write_completed()
3021 			if (!r10_bio->devs[m].bio->bi_status) {  in handle_write_completed()
3024 					r10_bio->devs[m].addr,  in handle_write_completed()
3025 					r10_bio->sectors, 0);  in handle_write_completed()
3029 					    r10_bio->devs[m].addr,  in handle_write_completed()
3030 					    r10_bio->sectors, 0))  in handle_write_completed()
3031 					md_error(conf->mddev, rdev);  in handle_write_completed()
3033 			rdev = conf->mirrors[dev].replacement;  in handle_write_completed()
3034 			if (r10_bio->devs[m].repl_bio == NULL ||  in handle_write_completed()
3035 				r10_bio->devs[m].repl_bio->bi_end_io == NULL)  in handle_write_completed()
3038 			if (!r10_bio->devs[m].repl_bio->bi_status) {  in handle_write_completed()
3041 					r10_bio->devs[m].addr,  in handle_write_completed()
3042 					r10_bio->sectors, 0);  in handle_write_completed()
3046 					    r10_bio->devs[m].addr,  in handle_write_completed()
3047 					    r10_bio->sectors, 0))  in handle_write_completed()
3048 					md_error(conf->mddev, rdev);  in handle_write_completed()
3054 		for (m = 0; m < conf->copies; m++) {  in handle_write_completed()
3055 			int dev = r10_bio->devs[m].devnum;  in handle_write_completed()
3056 			struct bio *bio = r10_bio->devs[m].bio;  in handle_write_completed()
3057 			rdev = conf->mirrors[dev].rdev;  in handle_write_completed()
3061 					r10_bio->devs[m].addr,  in handle_write_completed()
3062 					r10_bio->sectors, 0);  in handle_write_completed()
3063 				rdev_dec_pending(rdev, conf->mddev);  in handle_write_completed()
3064 			} else if (bio != NULL && bio->bi_status) {  in handle_write_completed()
3067 					md_error(conf->mddev, rdev);  in handle_write_completed()
3069 						&r10_bio->state);  in handle_write_completed()
3071 				rdev_dec_pending(rdev, conf->mddev);  in handle_write_completed()
3073 			bio = r10_bio->devs[m].repl_bio;  in handle_write_completed()
3074 			rdev = conf->mirrors[dev].replacement;  in handle_write_completed()
3078 					r10_bio->devs[m].addr,  in handle_write_completed()
3079 					r10_bio->sectors, 0);  in handle_write_completed()
3080 				rdev_dec_pending(rdev, conf->mddev);  in handle_write_completed()
3084 			spin_lock_irq(&conf->device_lock);  in handle_write_completed()
3085 			list_add(&r10_bio->retry_list, &conf->bio_end_io_list);  in handle_write_completed()
3086 			conf->nr_queued++;  in handle_write_completed()
3087 			spin_unlock_irq(&conf->device_lock);  in handle_write_completed()
3092 			wake_up(&conf->wait_barrier);  in handle_write_completed()
3093 			md_wakeup_thread(conf->mddev->thread);  in handle_write_completed()
3096 				     &r10_bio->state))  in handle_write_completed()
3105 	struct mddev *mddev = thread->mddev;  in raid10d()
3108 	struct r10conf *conf = mddev->private;  in raid10d()
3109 	struct list_head *head = &conf->retry_list;  in raid10d()
3114 	if (!list_empty_careful(&conf->bio_end_io_list) &&  in raid10d()
3115 	    !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {  in raid10d()
3117 		spin_lock_irqsave(&conf->device_lock, flags);  in raid10d()
3118 		if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {  in raid10d()
3119 			while (!list_empty(&conf->bio_end_io_list)) {  in raid10d()
3120 				list_move(conf->bio_end_io_list.prev, &tmp);  in raid10d()
3121 				conf->nr_queued--;  in raid10d()
3124 		spin_unlock_irqrestore(&conf->device_lock, flags);  in raid10d()
3128 			list_del(&r10_bio->retry_list);  in raid10d()
3129 			if (mddev->degraded)  in raid10d()
3130 				set_bit(R10BIO_Degraded, &r10_bio->state);  in raid10d()
3133 				     &r10_bio->state))  in raid10d()
3144 		spin_lock_irqsave(&conf->device_lock, flags);  in raid10d()
3146 			spin_unlock_irqrestore(&conf->device_lock, flags);  in raid10d()
3149 		r10_bio = list_entry(head->prev, struct r10bio, retry_list);  in raid10d()
3150 		list_del(head->prev);  in raid10d()
3151 		conf->nr_queued--;  in raid10d()
3152 		spin_unlock_irqrestore(&conf->device_lock, flags);  in raid10d()
3154 		mddev = r10_bio->mddev;  in raid10d()
3155 		conf = mddev->private;  in raid10d()
3156 		if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||  in raid10d()
3157 		    test_bit(R10BIO_WriteError, &r10_bio->state))  in raid10d()
3159 		else if (test_bit(R10BIO_IsReshape, &r10_bio->state))  in raid10d()
3161 		else if (test_bit(R10BIO_IsSync, &r10_bio->state))  in raid10d()
3163 		else if (test_bit(R10BIO_IsRecover, &r10_bio->state))  in raid10d()
3165 		else if (test_bit(R10BIO_ReadError, &r10_bio->state))  in raid10d()
3171 		if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))  in raid10d()
3182 	BUG_ON(mempool_initialized(&conf->r10buf_pool));  in init_resync()
3183 	conf->have_replacement = 0;  in init_resync()
3184 	for (i = 0; i < conf->geo.raid_disks; i++)  in init_resync()
3185 		if (conf->mirrors[i].replacement)  in init_resync()
3186 			conf->have_replacement = 1;  in init_resync()
3187 	ret = mempool_init(&conf->r10buf_pool, buffs,  in init_resync()
3191 	conf->next_resync = 0;  in init_resync()
3197 	struct r10bio *r10bio = mempool_alloc(&conf->r10buf_pool, GFP_NOIO);  in raid10_alloc_init_r10buf()
3203 	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||  in raid10_alloc_init_r10buf()
3204 	    test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))  in raid10_alloc_init_r10buf()
3205 		nalloc = conf->copies; /* resync */  in raid10_alloc_init_r10buf()
3210 		bio = r10bio->devs[i].bio;  in raid10_alloc_init_r10buf()
3211 		rp = bio->bi_private;  in raid10_alloc_init_r10buf()
3213 		bio->bi_private = rp;  in raid10_alloc_init_r10buf()
3214 		bio = r10bio->devs[i].repl_bio;  in raid10_alloc_init_r10buf()
3216 			rp = bio->bi_private;  in raid10_alloc_init_r10buf()
3218 			bio->bi_private = rp;  in raid10_alloc_init_r10buf()
3245 	chunks = conf->geo.raid_disks / conf->geo.near_copies;  in raid10_set_cluster_sync_high()
3246 	if (conf->geo.raid_disks % conf->geo.near_copies == 0)  in raid10_set_cluster_sync_high()
3250 	window_size = (chunks + extra_chunk) * conf->mddev->chunk_sectors;  in raid10_set_cluster_sync_high()
3258 	conf->cluster_sync_high = conf->cluster_sync_low + window_size;  in raid10_set_cluster_sync_high()
3264  * We need to make sure that no normal I/O request - particularly write
3265  * requests - conflict with active sync requests.
3271  * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
3274  * and update if there are differences.  If only one copy is live,
3277  * value for each non-in_sync drive, and over-write.
3280  * given address, one for each out-of-sync device.  We model this by allocating
3281  * a number of r10_bio structures, one for each out-of-sync device.
3287  * This link is counted in ->remaining.  When the r10_bio that points to NULL
3296 	struct r10conf *conf = mddev->private;  in raid10_sync_request()
3305 	sector_t chunk_mask = conf->geo.chunk_mask;  in raid10_sync_request()
3307 	int error_disk = -1;  in raid10_sync_request()
3313 	if (mddev->bitmap == NULL &&  in raid10_sync_request()
3314 	    mddev->recovery_cp == MaxSector &&  in raid10_sync_request()
3315 	    mddev->reshape_position == MaxSector &&  in raid10_sync_request()
3316 	    !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&  in raid10_sync_request()
3317 	    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&  in raid10_sync_request()
3318 	    !test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&  in raid10_sync_request()
3319 	    conf->fullsync == 0) {  in raid10_sync_request()
3321 		return mddev->dev_sectors - sector_nr;  in raid10_sync_request()
3324 	if (!mempool_initialized(&conf->r10buf_pool))  in raid10_sync_request()
3329 	max_sector = mddev->dev_sectors;  in raid10_sync_request()
3330 	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||  in raid10_sync_request()
3331 	    test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))  in raid10_sync_request()
3332 		max_sector = mddev->resync_max_sectors;  in raid10_sync_request()
3334 		conf->cluster_sync_low = 0;  in raid10_sync_request()
3335 		conf->cluster_sync_high = 0;  in raid10_sync_request()
3342 		 * mddev->curr_resync, but for recovery,  in raid10_sync_request()
3346 		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {  in raid10_sync_request()
3352 		if (mddev->curr_resync < max_sector) { /* aborted */  in raid10_sync_request()
3353 			if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))  in raid10_sync_request()
3354 				md_bitmap_end_sync(mddev->bitmap, mddev->curr_resync,  in raid10_sync_request()
3356 			else for (i = 0; i < conf->geo.raid_disks; i++) {  in raid10_sync_request()
3358 					raid10_find_virt(conf, mddev->curr_resync, i);  in raid10_sync_request()
3359 				md_bitmap_end_sync(mddev->bitmap, sect,  in raid10_sync_request()
3364 			if ((!mddev->bitmap || conf->fullsync)  in raid10_sync_request()
3365 			    && conf->have_replacement  in raid10_sync_request()
3366 			    && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {  in raid10_sync_request()
3371 				for (i = 0; i < conf->geo.raid_disks; i++) {  in raid10_sync_request()
3373 						rcu_dereference(conf->mirrors[i].replacement);  in raid10_sync_request()
3375 						rdev->recovery_offset = MaxSector;  in raid10_sync_request()
3379 			conf->fullsync = 0;  in raid10_sync_request()
3381 		md_bitmap_close_sync(mddev->bitmap);  in raid10_sync_request()
3387 	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))  in raid10_sync_request()
3390 	if (chunks_skipped >= conf->geo.raid_disks) {  in raid10_sync_request()
3392 			test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ?  "resync" : "recovery");  in raid10_sync_request()
3394 		    !test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {  in raid10_sync_request()
3399 			conf->mirrors[error_disk].recovery_disabled =  in raid10_sync_request()
3400 						mddev->recovery_disabled;  in raid10_sync_request()
3408 		return (max_sector - sector_nr) + sectors_skipped;  in raid10_sync_request()
3411 	if (max_sector > mddev->resync_max)  in raid10_sync_request()
3412 		max_sector = mddev->resync_max; /* Don't do IO beyond here */  in raid10_sync_request()
3414 	/* make sure whole request will fit in a chunk - if chunks  in raid10_sync_request()
3417 	if (conf->geo.near_copies < conf->geo.raid_disks &&  in raid10_sync_request()
3422 	 * If there is non-resync activity waiting for a turn, then let it  in raid10_sync_request()
3425 	if (conf->nr_waiting)  in raid10_sync_request()
3438 	/* First, we decide what to do and set ->bi_end_io  in raid10_sync_request()
3443 	max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);  in raid10_sync_request()
3444 	if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {  in raid10_sync_request()
3449 		for (i = 0 ; i < conf->geo.raid_disks; i++) {  in raid10_sync_request()
3455 			struct raid10_info *mirror = &conf->mirrors[i];  in raid10_sync_request()
3459 			mrdev = rcu_dereference(mirror->rdev);  in raid10_sync_request()
3460 			mreplace = rcu_dereference(mirror->replacement);  in raid10_sync_request()
3462 			if (mrdev && (test_bit(Faulty, &mrdev->flags) ||  in raid10_sync_request()
3463 			    test_bit(In_sync, &mrdev->flags)))  in raid10_sync_request()
3465 			if (mreplace && test_bit(Faulty, &mreplace->flags))  in raid10_sync_request()
3477 			if (sect >= mddev->resync_max_sectors) {  in raid10_sync_request()
3478 				/* last stripe is not complete - don't  in raid10_sync_request()
3488 			must_sync = md_bitmap_start_sync(mddev->bitmap, sect,  in raid10_sync_request()
3494 			    !conf->fullsync) {  in raid10_sync_request()
3498 				chunks_skipped = -1;  in raid10_sync_request()
3503 				atomic_inc(&mrdev->nr_pending);  in raid10_sync_request()
3505 				atomic_inc(&mreplace->nr_pending);  in raid10_sync_request()
3509 			r10_bio->state = 0;  in raid10_sync_request()
3511 			atomic_set(&r10_bio->remaining, 0);  in raid10_sync_request()
3513 			r10_bio->master_bio = (struct bio*)rb2;  in raid10_sync_request()
3515 				atomic_inc(&rb2->remaining);  in raid10_sync_request()
3516 			r10_bio->mddev = mddev;  in raid10_sync_request()
3517 			set_bit(R10BIO_IsRecover, &r10_bio->state);  in raid10_sync_request()
3518 			r10_bio->sector = sect;  in raid10_sync_request()
3526 			for (j = 0; j < conf->geo.raid_disks; j++) {  in raid10_sync_request()
3528 					conf->mirrors[j].rdev);  in raid10_sync_request()
3529 				if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {  in raid10_sync_request()
3535 			must_sync = md_bitmap_start_sync(mddev->bitmap, sect,  in raid10_sync_request()
3539 			for (j=0; j<conf->copies;j++) {  in raid10_sync_request()
3541 				int d = r10_bio->devs[j].devnum;  in raid10_sync_request()
3544 					rcu_dereference(conf->mirrors[d].rdev);  in raid10_sync_request()
3548 				    !test_bit(In_sync, &rdev->flags))  in raid10_sync_request()
3552 				sector = r10_bio->devs[j].addr;  in raid10_sync_request()
3557 						max_sync = first_bad - sector;  in raid10_sync_request()
3559 						bad_sectors -= (sector  in raid10_sync_request()
3560 								- first_bad);  in raid10_sync_request()
3566 				bio = r10_bio->devs[0].bio;  in raid10_sync_request()
3567 				bio->bi_next = biolist;  in raid10_sync_request()
3569 				bio->bi_end_io = end_sync_read;  in raid10_sync_request()
3570 				bio->bi_opf = REQ_OP_READ;  in raid10_sync_request()
3571 				if (test_bit(FailFast, &rdev->flags))  in raid10_sync_request()
3572 					bio->bi_opf |= MD_FAILFAST;  in raid10_sync_request()
3573 				from_addr = r10_bio->devs[j].addr;  in raid10_sync_request()
3574 				bio->bi_iter.bi_sector = from_addr +  in raid10_sync_request()
3575 					rdev->data_offset;  in raid10_sync_request()
3576 				bio_set_dev(bio, rdev->bdev);  in raid10_sync_request()
3577 				atomic_inc(&rdev->nr_pending);  in raid10_sync_request()
3580 				for (k=0; k<conf->copies; k++)  in raid10_sync_request()
3581 					if (r10_bio->devs[k].devnum == i)  in raid10_sync_request()
3583 				BUG_ON(k == conf->copies);  in raid10_sync_request()
3584 				to_addr = r10_bio->devs[k].addr;  in raid10_sync_request()
3585 				r10_bio->devs[0].devnum = d;  in raid10_sync_request()
3586 				r10_bio->devs[0].addr = from_addr;  in raid10_sync_request()
3587 				r10_bio->devs[1].devnum = i;  in raid10_sync_request()
3588 				r10_bio->devs[1].addr = to_addr;  in raid10_sync_request()
3591 					bio = r10_bio->devs[1].bio;  in raid10_sync_request()
3592 					bio->bi_next = biolist;  in raid10_sync_request()
3594 					bio->bi_end_io = end_sync_write;  in raid10_sync_request()
3595 					bio->bi_opf = REQ_OP_WRITE;  in raid10_sync_request()
3596 					bio->bi_iter.bi_sector = to_addr  in raid10_sync_request()
3597 						+ mrdev->data_offset;  in raid10_sync_request()
3598 					bio_set_dev(bio, mrdev->bdev);  in raid10_sync_request()
3599 					atomic_inc(&r10_bio->remaining);  in raid10_sync_request()
3601 					r10_bio->devs[1].bio->bi_end_io = NULL;  in raid10_sync_request()
3604 				bio = r10_bio->devs[1].repl_bio;  in raid10_sync_request()
3606 					bio->bi_end_io = NULL;  in raid10_sync_request()
3613 				bio->bi_next = biolist;  in raid10_sync_request()
3615 				bio->bi_end_io = end_sync_write;  in raid10_sync_request()
3616 				bio->bi_opf = REQ_OP_WRITE;  in raid10_sync_request()
3617 				bio->bi_iter.bi_sector = to_addr +  in raid10_sync_request()
3618 					mreplace->data_offset;  in raid10_sync_request()
3619 				bio_set_dev(bio, mreplace->bdev);  in raid10_sync_request()
3620 				atomic_inc(&r10_bio->remaining);  in raid10_sync_request()
3624 			if (j == conf->copies) {  in raid10_sync_request()
3632 					for (k = 0; k < conf->copies; k++)  in raid10_sync_request()
3633 						if (r10_bio->devs[k].devnum == i)  in raid10_sync_request()
3636 						      &mrdev->flags)  in raid10_sync_request()
3639 						    r10_bio->devs[k].addr,  in raid10_sync_request()
3645 						    r10_bio->devs[k].addr,  in raid10_sync_request()
3651 							      &mddev->recovery))  in raid10_sync_request()
3654 					mirror->recovery_disabled  in raid10_sync_request()
3655 						= mddev->recovery_disabled;  in raid10_sync_request()
3661 					atomic_dec(&rb2->remaining);  in raid10_sync_request()
3673 			if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) {  in raid10_sync_request()
3679 				for (; j < conf->copies; j++) {  in raid10_sync_request()
3680 					int d = r10_bio->devs[j].devnum;  in raid10_sync_request()
3681 					if (conf->mirrors[d].rdev &&  in raid10_sync_request()
3683 						      &conf->mirrors[d].rdev->flags))  in raid10_sync_request()
3687 					r10_bio->devs[0].bio->bi_opf  in raid10_sync_request()
3694 				r10_bio = (struct r10bio*) rb2->master_bio;  in raid10_sync_request()
3695 				rb2->master_bio = NULL;  in raid10_sync_request()
3705 		 * Since curr_resync_completed could probably not update in  in raid10_sync_request()
3711 		md_bitmap_cond_end_sync(mddev->bitmap, sector_nr,  in raid10_sync_request()
3713 					(sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high));  in raid10_sync_request()
3715 		if (!md_bitmap_start_sync(mddev->bitmap, sector_nr,  in raid10_sync_request()
3716 					  &sync_blocks, mddev->degraded) &&  in raid10_sync_request()
3717 		    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,  in raid10_sync_request()
3718 						 &mddev->recovery)) {  in raid10_sync_request()
3726 		r10_bio->state = 0;  in raid10_sync_request()
3728 		r10_bio->mddev = mddev;  in raid10_sync_request()
3729 		atomic_set(&r10_bio->remaining, 0);  in raid10_sync_request()
3731 		conf->next_resync = sector_nr;  in raid10_sync_request()
3733 		r10_bio->master_bio = NULL;  in raid10_sync_request()
3734 		r10_bio->sector = sector_nr;  in raid10_sync_request()
3735 		set_bit(R10BIO_IsSync, &r10_bio->state);  in raid10_sync_request()
3737 		r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;  in raid10_sync_request()
3739 		for (i = 0; i < conf->copies; i++) {  in raid10_sync_request()
3740 			int d = r10_bio->devs[i].devnum;  in raid10_sync_request()
3745 			if (r10_bio->devs[i].repl_bio)  in raid10_sync_request()
3746 				r10_bio->devs[i].repl_bio->bi_end_io = NULL;  in raid10_sync_request()
3748 			bio = r10_bio->devs[i].bio;  in raid10_sync_request()
3749 			bio->bi_status = BLK_STS_IOERR;  in raid10_sync_request()
3751 			rdev = rcu_dereference(conf->mirrors[d].rdev);  in raid10_sync_request()
3752 			if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {  in raid10_sync_request()
3756 			sector = r10_bio->devs[i].addr;  in raid10_sync_request()
3760 					max_sync = first_bad - sector;  in raid10_sync_request()
3762 					bad_sectors -= (sector - first_bad);  in raid10_sync_request()
3769 			atomic_inc(&rdev->nr_pending);  in raid10_sync_request()
3770 			atomic_inc(&r10_bio->remaining);  in raid10_sync_request()
3771 			bio->bi_next = biolist;  in raid10_sync_request()
3773 			bio->bi_end_io = end_sync_read;  in raid10_sync_request()
3774 			bio->bi_opf = REQ_OP_READ;  in raid10_sync_request()
3775 			if (test_bit(FailFast, &rdev->flags))  in raid10_sync_request()
3776 				bio->bi_opf |= MD_FAILFAST;  in raid10_sync_request()
3777 			bio->bi_iter.bi_sector = sector + rdev->data_offset;  in raid10_sync_request()
3778 			bio_set_dev(bio, rdev->bdev);  in raid10_sync_request()
3781 			rdev = rcu_dereference(conf->mirrors[d].replacement);  in raid10_sync_request()
3782 			if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {  in raid10_sync_request()
3786 			atomic_inc(&rdev->nr_pending);  in raid10_sync_request()
3789 			bio = r10_bio->devs[i].repl_bio;  in raid10_sync_request()
3790 			bio->bi_status = BLK_STS_IOERR;  in raid10_sync_request()
3792 			sector = r10_bio->devs[i].addr;  in raid10_sync_request()
3793 			bio->bi_next = biolist;  in raid10_sync_request()
3795 			bio->bi_end_io = end_sync_write;  in raid10_sync_request()
3796 			bio->bi_opf = REQ_OP_WRITE;  in raid10_sync_request()
3797 			if (test_bit(FailFast, &rdev->flags))  in raid10_sync_request()
3798 				bio->bi_opf |= MD_FAILFAST;  in raid10_sync_request()
3799 			bio->bi_iter.bi_sector = sector + rdev->data_offset;  in raid10_sync_request()
3800 			bio_set_dev(bio, rdev->bdev);  in raid10_sync_request()
3806 			for (i=0; i<conf->copies; i++) {  in raid10_sync_request()
3807 				int d = r10_bio->devs[i].devnum;  in raid10_sync_request()
3808 				if (r10_bio->devs[i].bio->bi_end_io)  in raid10_sync_request()
3809 					rdev_dec_pending(conf->mirrors[d].rdev,  in raid10_sync_request()
3811 				if (r10_bio->devs[i].repl_bio &&  in raid10_sync_request()
3812 				    r10_bio->devs[i].repl_bio->bi_end_io)  in raid10_sync_request()
3814 						conf->mirrors[d].replacement,  in raid10_sync_request()
3830 			len = (max_sector - sector_nr) << 9;  in raid10_sync_request()
3833 		for (bio= biolist ; bio ; bio=bio->bi_next) {  in raid10_sync_request()
3837 				bio->bi_status = BLK_STS_RESOURCE;  in raid10_sync_request()
3845 	r10_bio->sectors = nr_sectors;  in raid10_sync_request()
3848 	    test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {  in raid10_sync_request()
3850 		if (conf->cluster_sync_high < sector_nr + nr_sectors) {  in raid10_sync_request()
3851 			conf->cluster_sync_low = mddev->curr_resync_completed;  in raid10_sync_request()
3854 			md_cluster_ops->resync_info_update(mddev,  in raid10_sync_request()
3855 						conf->cluster_sync_low,  in raid10_sync_request()
3856 						conf->cluster_sync_high);  in raid10_sync_request()
3863 		for (i = 0; i < conf->geo.raid_disks; i++) {  in raid10_sync_request()
3871 			if (conf->cluster_sync_high < sect_va1 + nr_sectors) {  in raid10_sync_request()
3878 					mddev->curr_resync_completed, i);  in raid10_sync_request()
3880 				if (conf->cluster_sync_low == 0 ||  in raid10_sync_request()
3881 				    conf->cluster_sync_low > sect_va2)  in raid10_sync_request()
3882 					conf->cluster_sync_low = sect_va2;  in raid10_sync_request()
3887 			md_cluster_ops->resync_info_update(mddev,  in raid10_sync_request()
3888 						conf->cluster_sync_low,  in raid10_sync_request()
3889 						conf->cluster_sync_high);  in raid10_sync_request()
3895 		biolist = biolist->bi_next;  in raid10_sync_request()
3897 		bio->bi_next = NULL;  in raid10_sync_request()
3899 		r10_bio->sectors = nr_sectors;  in raid10_sync_request()
3901 		if (bio->bi_end_io == end_sync_read) {  in raid10_sync_request()
3903 			bio->bi_status = 0;  in raid10_sync_request()
3916 	/* There is nowhere to write, so all non-sync  in raid10_sync_request()
3923 	sectors_skipped += (max_sector - sector_nr);  in raid10_sync_request()
3933 	struct r10conf *conf = mddev->private;  in raid10_size()
3936 		raid_disks = min(conf->geo.raid_disks,  in raid10_size()
3937 				 conf->prev.raid_disks);  in raid10_size()
3939 		sectors = conf->dev_sectors;  in raid10_size()
3941 	size = sectors >> conf->geo.chunk_shift;  in raid10_size()
3942 	sector_div(size, conf->geo.far_copies);  in raid10_size()
3944 	sector_div(size, conf->geo.near_copies);  in raid10_size()
3946 	return size << conf->geo.chunk_shift;  in raid10_size()
3951 	/* Calculate the number of sectors-per-device that will  in calc_sectors()
3952 	 * actually be used, and set conf->dev_sectors and  in calc_sectors()
3953 	 * conf->stride  in calc_sectors()
3956 	size = size >> conf->geo.chunk_shift;  in calc_sectors()
3957 	sector_div(size, conf->geo.far_copies);  in calc_sectors()
3958 	size = size * conf->geo.raid_disks;  in calc_sectors()
3959 	sector_div(size, conf->geo.near_copies);  in calc_sectors()
3962 	size = size * conf->copies;  in calc_sectors()
3967 	size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);  in calc_sectors()
3969 	conf->dev_sectors = size << conf->geo.chunk_shift;  in calc_sectors()
3971 	if (conf->geo.far_offset)  in calc_sectors()
3972 		conf->geo.stride = 1 << conf->geo.chunk_shift;  in calc_sectors()
3974 		sector_div(size, conf->geo.far_copies);  in calc_sectors()
3975 		conf->geo.stride = size << conf->geo.chunk_shift;  in calc_sectors()
3982 	int nc, fc, fo;  in setup_geo()  local
3986 		layout = mddev->layout;  in setup_geo()
3987 		chunk = mddev->chunk_sectors;  in setup_geo()
3988 		disks = mddev->raid_disks - mddev->delta_disks;  in setup_geo()
3991 		layout = mddev->new_layout;  in setup_geo()
3992 		chunk = mddev->new_chunk_sectors;  in setup_geo()
3993 		disks = mddev->raid_disks;  in setup_geo()
3996 	case geo_start: /* new when starting reshape - raid_disks not  in setup_geo()
3998 		layout = mddev->new_layout;  in setup_geo()
3999 		chunk = mddev->new_chunk_sectors;  in setup_geo()
4000 		disks = mddev->raid_disks + mddev->delta_disks;  in setup_geo()
4004 		return -1;  in setup_geo()
4007 		return -2;  in setup_geo()
4009 	fc = (layout >> 8) & 255;  in setup_geo()
4011 	geo->raid_disks = disks;  in setup_geo()
4012 	geo->near_copies = nc;  in setup_geo()
4013 	geo->far_copies = fc;  in setup_geo()
4014 	geo->far_offset = fo;  in setup_geo()
4017 		geo->far_set_size = disks;  in setup_geo()
4019 	case 1: /* "improved" layout which was buggy.  Hopefully no-one is  in setup_geo()
4021 		geo->far_set_size = disks/fc;  in setup_geo()
4022 		WARN(geo->far_set_size < fc,  in setup_geo()
4023 		     "This RAID10 layout does not provide data safety - please backup and create new array\n");  in setup_geo()
4026 		geo->far_set_size = fc * nc;  in setup_geo()
4029 		return -1;  in setup_geo()
4031 	geo->chunk_mask = chunk - 1;  in setup_geo()
4032 	geo->chunk_shift = ffz(~chunk);  in setup_geo()
4033 	return nc*fc;  in setup_geo()
4041 	mempool_exit(&conf->r10bio_pool);  in raid10_free_conf()
4042 	kfree(conf->mirrors);  in raid10_free_conf()
4043 	kfree(conf->mirrors_old);  in raid10_free_conf()
4044 	kfree(conf->mirrors_new);  in raid10_free_conf()
4045 	safe_put_page(conf->tmppage);  in raid10_free_conf()
4046 	bioset_exit(&conf->bio_split);  in raid10_free_conf()
4053 	int err = -EINVAL;  in setup_conf()
4059 	if (copies == -2) {  in setup_conf()
4065 	if (copies < 2 || copies > mddev->raid_disks) {  in setup_conf()
4067 			mdname(mddev), mddev->new_layout);  in setup_conf()
4071 	err = -ENOMEM;  in setup_conf()
4077 	conf->mirrors = kcalloc(mddev->raid_disks + max(0, -mddev->delta_disks),  in setup_conf()
4080 	if (!conf->mirrors)  in setup_conf()
4083 	conf->tmppage = alloc_page(GFP_KERNEL);  in setup_conf()
4084 	if (!conf->tmppage)  in setup_conf()
4087 	conf->geo = geo;  in setup_conf()
4088 	conf->copies = copies;  in setup_conf()
4089 	err = mempool_init(&conf->r10bio_pool, NR_RAID_BIOS, r10bio_pool_alloc,  in setup_conf()
4094 	err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);  in setup_conf()
4098 	calc_sectors(conf, mddev->dev_sectors);  in setup_conf()
4099 	if (mddev->reshape_position == MaxSector) {  in setup_conf()
4100 		conf->prev = conf->geo;  in setup_conf()
4101 		conf->reshape_progress = MaxSector;  in setup_conf()
4103 		if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) {  in setup_conf()
4104 			err = -EINVAL;  in setup_conf()
4107 		conf->reshape_progress = mddev->reshape_position;  in setup_conf()
4108 		if (conf->prev.far_offset)  in setup_conf()
4109 			conf->prev.stride = 1 << conf->prev.chunk_shift;  in setup_conf()
4112 			conf->prev.stride = conf->dev_sectors;  in setup_conf()
4114 	conf->reshape_safe = conf->reshape_progress;  in setup_conf()
4115 	spin_lock_init(&conf->device_lock);  in setup_conf()
4116 	INIT_LIST_HEAD(&conf->retry_list);  in setup_conf()
4117 	INIT_LIST_HEAD(&conf->bio_end_io_list);  in setup_conf()
4119 	seqlock_init(&conf->resync_lock);  in setup_conf()
4120 	init_waitqueue_head(&conf->wait_barrier);  in setup_conf()
4121 	atomic_set(&conf->nr_pending, 0);  in setup_conf()
4123 	err = -ENOMEM;  in setup_conf()
4124 	rcu_assign_pointer(conf->thread,  in setup_conf()
4126 	if (!conf->thread)  in setup_conf()
4129 	conf->mddev = mddev;  in setup_conf()
4139 	int raid_disks = conf->geo.raid_disks;  in raid10_set_io_opt()
4141 	if (!(conf->geo.raid_disks % conf->geo.near_copies))  in raid10_set_io_opt()
4142 		raid_disks /= conf->geo.near_copies;  in raid10_set_io_opt()
4143 	blk_queue_io_opt(conf->mddev->queue, (conf->mddev->chunk_sectors << 9) *  in raid10_set_io_opt()
4158 		return -ENOMEM;  in raid10_run()
4160 	if (mddev->private == NULL) {  in raid10_run()
4164 		mddev->private = conf;  in raid10_run()
4166 	conf = mddev->private;  in raid10_run()
4170 	rcu_assign_pointer(mddev->thread, conf->thread);  in raid10_run()
4171 	rcu_assign_pointer(conf->thread, NULL);  in raid10_run()
4173 	if (mddev_is_clustered(conf->mddev)) {  in raid10_run()
4174 		int fc, fo;  in raid10_run()  local
4176 		fc = (mddev->layout >> 8) & 255;  in raid10_run()
4177 		fo = mddev->layout & (1<<16);  in raid10_run()
4178 		if (fc > 1 || fo > 0) {  in raid10_run()
4185 	if (mddev->queue) {  in raid10_run()
4186 		blk_queue_max_write_zeroes_sectors(mddev->queue, 0);  in raid10_run()
4187 		blk_queue_io_min(mddev->queue, mddev->chunk_sectors << 9);  in raid10_run()
4194 		disk_idx = rdev->raid_disk;  in raid10_run()
4197 		if (disk_idx >= conf->geo.raid_disks &&  in raid10_run()
4198 		    disk_idx >= conf->prev.raid_disks)  in raid10_run()
4200 		disk = conf->mirrors + disk_idx;  in raid10_run()
4202 		if (test_bit(Replacement, &rdev->flags)) {  in raid10_run()
4203 			if (disk->replacement)  in raid10_run()
4205 			disk->replacement = rdev;  in raid10_run()
4207 			if (disk->rdev)  in raid10_run()
4209 			disk->rdev = rdev;  in raid10_run()
4211 		diff = (rdev->new_data_offset - rdev->data_offset);  in raid10_run()
4212 		if (!mddev->reshape_backwards)  in raid10_run()
4213 			diff = -diff;  in raid10_run()
4219 		if (mddev->gendisk)  in raid10_run()
4220 			disk_stack_limits(mddev->gendisk, rdev->bdev,  in raid10_run()
4221 					  rdev->data_offset << 9);  in raid10_run()
4223 		disk->head_position = 0;  in raid10_run()
4228 	if (!enough(conf, -1)) {  in raid10_run()
4234 	if (conf->reshape_progress != MaxSector) {  in raid10_run()
4236 		if (conf->geo.far_copies != 1 &&  in raid10_run()
4237 		    conf->geo.far_offset == 0)  in raid10_run()
4239 		if (conf->prev.far_copies != 1 &&  in raid10_run()
4240 		    conf->prev.far_offset == 0)  in raid10_run()
4244 	mddev->degraded = 0;  in raid10_run()
4246 	     i < conf->geo.raid_disks  in raid10_run()
4247 		     || i < conf->prev.raid_disks;  in raid10_run()
4250 		disk = conf->mirrors + i;  in raid10_run()
4252 		if (!disk->rdev && disk->replacement) {  in raid10_run()
4253 			/* The replacement is all we have - use it */  in raid10_run()
4254 			disk->rdev = disk->replacement;  in raid10_run()
4255 			disk->replacement = NULL;  in raid10_run()
4256 			clear_bit(Replacement, &disk->rdev->flags);  in raid10_run()
4259 		if (!disk->rdev ||  in raid10_run()
4260 		    !test_bit(In_sync, &disk->rdev->flags)) {  in raid10_run()
4261 			disk->head_position = 0;  in raid10_run()
4262 			mddev->degraded++;  in raid10_run()
4263 			if (disk->rdev &&  in raid10_run()
4264 			    disk->rdev->saved_raid_disk < 0)  in raid10_run()
4265 				conf->fullsync = 1;  in raid10_run()
4268 		if (disk->replacement &&  in raid10_run()
4269 		    !test_bit(In_sync, &disk->replacement->flags) &&  in raid10_run()
4270 		    disk->replacement->saved_raid_disk < 0) {  in raid10_run()
4271 			conf->fullsync = 1;  in raid10_run()
4274 		disk->recovery_disabled = mddev->recovery_disabled - 1;  in raid10_run()
4277 	if (mddev->recovery_cp != MaxSector)  in raid10_run()
4278 		pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n",  in raid10_run()
4281 		mdname(mddev), conf->geo.raid_disks - mddev->degraded,  in raid10_run()
4282 		conf->geo.raid_disks);  in raid10_run()
4286 	mddev->dev_sectors = conf->dev_sectors;  in raid10_run()
4289 	mddev->resync_max_sectors = size;  in raid10_run()
4290 	set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);  in raid10_run()
4295 	if (conf->reshape_progress != MaxSector) {  in raid10_run()
4298 		before_length = ((1 << conf->prev.chunk_shift) *  in raid10_run()
4299 				 conf->prev.far_copies);  in raid10_run()
4300 		after_length = ((1 << conf->geo.chunk_shift) *  in raid10_run()
4301 				conf->geo.far_copies);  in raid10_run()
4308 		conf->offset_diff = min_offset_diff;  in raid10_run()
4310 		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);  in raid10_run()
4311 		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);  in raid10_run()
4312 		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);  in raid10_run()
4313 		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);  in raid10_run()
4314 		rcu_assign_pointer(mddev->sync_thread,  in raid10_run()
4316 		if (!mddev->sync_thread)  in raid10_run()
4323 	md_unregister_thread(mddev, &mddev->thread);  in raid10_run()
4325 	mddev->private = NULL;  in raid10_run()
4327 	return -EIO;  in raid10_run()
4337 	struct r10conf *conf = mddev->private;  in raid10_quiesce()
4359 	struct r10conf *conf = mddev->private;  in raid10_resize()
4362 	if (mddev->reshape_position != MaxSector)  in raid10_resize()
4363 		return -EBUSY;  in raid10_resize()
4365 	if (conf->geo.far_copies > 1 && !conf->geo.far_offset)  in raid10_resize()
4366 		return -EINVAL;  in raid10_resize()
4370 	if (mddev->external_size &&  in raid10_resize()
4371 	    mddev->array_sectors > size)  in raid10_resize()
4372 		return -EINVAL;  in raid10_resize()
4373 	if (mddev->bitmap) {  in raid10_resize()
4374 		int ret = md_bitmap_resize(mddev->bitmap, size, 0, 0);  in raid10_resize()
4379 	if (sectors > mddev->dev_sectors &&  in raid10_resize()
4380 	    mddev->recovery_cp > oldsize) {  in raid10_resize()
4381 		mddev->recovery_cp = oldsize;  in raid10_resize()
4382 		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);  in raid10_resize()
4385 	mddev->dev_sectors = conf->dev_sectors;  in raid10_resize()
4386 	mddev->resync_max_sectors = size;  in raid10_resize()
4395 	if (mddev->degraded > 0) {  in raid10_takeover_raid0()
4398 		return ERR_PTR(-EINVAL);  in raid10_takeover_raid0()
4403 	mddev->new_level = 10;  in raid10_takeover_raid0()
4405 	mddev->new_layout = (1<<8) + 2;  in raid10_takeover_raid0()
4406 	mddev->new_chunk_sectors = mddev->chunk_sectors;  in raid10_takeover_raid0()
4407 	mddev->delta_disks = mddev->raid_disks;  in raid10_takeover_raid0()
4408 	mddev->raid_disks *= 2;  in raid10_takeover_raid0()
4410 	mddev->recovery_cp = MaxSector;  in raid10_takeover_raid0()
4411 	mddev->dev_sectors = size;  in raid10_takeover_raid0()
4416 			if (rdev->raid_disk >= 0) {  in raid10_takeover_raid0()
4417 				rdev->new_raid_disk = rdev->raid_disk * 2;  in raid10_takeover_raid0()
4418 				rdev->sectors = size;  in raid10_takeover_raid0()
4430 	 *  raid0 - providing it has only two drives  in raid10_takeover()
4432 	if (mddev->level == 0) {  in raid10_takeover()
4434 		raid0_conf = mddev->private;  in raid10_takeover()
4435 		if (raid0_conf->nr_strip_zones > 1) {  in raid10_takeover()
4438 			return ERR_PTR(-EINVAL);  in raid10_takeover()
4441 			raid0_conf->strip_zone->zone_end,  in raid10_takeover()
4442 			raid0_conf->strip_zone->nb_dev);  in raid10_takeover()
4444 	return ERR_PTR(-EINVAL);  in raid10_takeover()
4450 	 * - layout (to ->new_layout)  in raid10_check_reshape()
4451 	 * - chunk size (to ->new_chunk_sectors)  in raid10_check_reshape()
4452 	 * - raid_disks (by delta_disks)  in raid10_check_reshape()
4463 	struct r10conf *conf = mddev->private;  in raid10_check_reshape()
4466 	if (conf->geo.far_copies != 1 && !conf->geo.far_offset)  in raid10_check_reshape()
4467 		return -EINVAL;  in raid10_check_reshape()
4469 	if (setup_geo(&geo, mddev, geo_start) != conf->copies)  in raid10_check_reshape()
4471 		return -EINVAL;  in raid10_check_reshape()
4474 		return -EINVAL;  in raid10_check_reshape()
4476 	if (mddev->array_sectors & geo.chunk_mask)  in raid10_check_reshape()
4478 			return -EINVAL;  in raid10_check_reshape()
4480 	if (!enough(conf, -1))  in raid10_check_reshape()
4481 		return -EINVAL;  in raid10_check_reshape()
4483 	kfree(conf->mirrors_new);  in raid10_check_reshape()
4484 	conf->mirrors_new = NULL;  in raid10_check_reshape()
4485 	if (mddev->delta_disks > 0) {  in raid10_check_reshape()
4487 		conf->mirrors_new =  in raid10_check_reshape()
4488 			kcalloc(mddev->raid_disks + mddev->delta_disks,  in raid10_check_reshape()
4491 		if (!conf->mirrors_new)  in raid10_check_reshape()
4492 			return -ENOMEM;  in raid10_check_reshape()
4499  *  - start an array
4500  *  - remove non-faulty devices
4501  *  - add a spare
4502  *  - allow a reshape
4507  * of the two sections, and some non-in_sync devices may
4518 	for (i = 0; i < conf->prev.raid_disks; i++) {  in calc_degraded()
4519 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);  in calc_degraded()
4520 		if (!rdev || test_bit(Faulty, &rdev->flags))  in calc_degraded()
4522 		else if (!test_bit(In_sync, &rdev->flags))  in calc_degraded()
4530 	if (conf->geo.raid_disks == conf->prev.raid_disks)  in calc_degraded()
4534 	for (i = 0; i < conf->geo.raid_disks; i++) {  in calc_degraded()
4535 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);  in calc_degraded()
4536 		if (!rdev || test_bit(Faulty, &rdev->flags))  in calc_degraded()
4538 		else if (!test_bit(In_sync, &rdev->flags)) {  in calc_degraded()
4544 			if (conf->geo.raid_disks <= conf->prev.raid_disks)  in calc_degraded()
4561 	 * array non-degraded.  We also require that the difference  in raid10_start_reshape()
4562 	 * between old and new data_offset - on each device - is  in raid10_start_reshape()
4563 	 * enough that we never risk over-writing.  in raid10_start_reshape()
4570 	struct r10conf *conf = mddev->private;  in raid10_start_reshape()
4575 	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))  in raid10_start_reshape()
4576 		return -EBUSY;  in raid10_start_reshape()
4578 	if (setup_geo(&new, mddev, geo_start) != conf->copies)  in raid10_start_reshape()
4579 		return -EINVAL;  in raid10_start_reshape()
4581 	before_length = ((1 << conf->prev.chunk_shift) *  in raid10_start_reshape()
4582 			 conf->prev.far_copies);  in raid10_start_reshape()
4583 	after_length = ((1 << conf->geo.chunk_shift) *  in raid10_start_reshape()
4584 			conf->geo.far_copies);  in raid10_start_reshape()
4587 		if (!test_bit(In_sync, &rdev->flags)  in raid10_start_reshape()
4588 		    && !test_bit(Faulty, &rdev->flags))  in raid10_start_reshape()
4590 		if (rdev->raid_disk >= 0) {  in raid10_start_reshape()
4591 			long long diff = (rdev->new_data_offset  in raid10_start_reshape()
4592 					  - rdev->data_offset);  in raid10_start_reshape()
4593 			if (!mddev->reshape_backwards)  in raid10_start_reshape()
4594 				diff = -diff;  in raid10_start_reshape()
4604 		return -EINVAL;  in raid10_start_reshape()
4606 	if (spares < mddev->delta_disks)  in raid10_start_reshape()
4607 		return -EINVAL;  in raid10_start_reshape()
4609 	conf->offset_diff = min_offset_diff;  in raid10_start_reshape()
4610 	spin_lock_irq(&conf->device_lock);  in raid10_start_reshape()
4611 	if (conf->mirrors_new) {  in raid10_start_reshape()
4612 		memcpy(conf->mirrors_new, conf->mirrors,  in raid10_start_reshape()
4613 		       sizeof(struct raid10_info)*conf->prev.raid_disks);  in raid10_start_reshape()
4615 		kfree(conf->mirrors_old);  in raid10_start_reshape()
4616 		conf->mirrors_old = conf->mirrors;  in raid10_start_reshape()
4617 		conf->mirrors = conf->mirrors_new;  in raid10_start_reshape()
4618 		conf->mirrors_new = NULL;  in raid10_start_reshape()
4620 	setup_geo(&conf->geo, mddev, geo_start);  in raid10_start_reshape()
4622 	if (mddev->reshape_backwards) {  in raid10_start_reshape()
4624 		if (size < mddev->array_sectors) {  in raid10_start_reshape()
4625 			spin_unlock_irq(&conf->device_lock);  in raid10_start_reshape()
4628 			return -EINVAL;  in raid10_start_reshape()
4630 		mddev->resync_max_sectors = size;  in raid10_start_reshape()
4631 		conf->reshape_progress = size;  in raid10_start_reshape()
4633 		conf->reshape_progress = 0;  in raid10_start_reshape()
4634 	conf->reshape_safe = conf->reshape_progress;  in raid10_start_reshape()
4635 	spin_unlock_irq(&conf->device_lock);  in raid10_start_reshape()
4637 	if (mddev->delta_disks && mddev->bitmap) {  in raid10_start_reshape()
4642 		newsize = raid10_size(mddev, 0, conf->geo.raid_disks);  in raid10_start_reshape()
4645 			ret = md_bitmap_resize(mddev->bitmap, newsize, 0, 0);  in raid10_start_reshape()
4653 			if (rdev->raid_disk > -1 &&  in raid10_start_reshape()
4654 			    !test_bit(Faulty, &rdev->flags))  in raid10_start_reshape()
4655 				sb = page_address(rdev->sb_page);  in raid10_start_reshape()
4663 		if ((sb && (le32_to_cpu(sb->feature_map) &  in raid10_start_reshape()
4667 		ret = md_bitmap_resize(mddev->bitmap, newsize, 0, 0);  in raid10_start_reshape()
4671 		ret = md_cluster_ops->resize_bitmaps(mddev, newsize, oldsize);  in raid10_start_reshape()
4673 			md_bitmap_resize(mddev->bitmap, oldsize, 0, 0);  in raid10_start_reshape()
4678 	if (mddev->delta_disks > 0) {  in raid10_start_reshape()
4680 			if (rdev->raid_disk < 0 &&  in raid10_start_reshape()
4681 			    !test_bit(Faulty, &rdev->flags)) {  in raid10_start_reshape()
4683 					if (rdev->raid_disk >=  in raid10_start_reshape()
4684 					    conf->prev.raid_disks)  in raid10_start_reshape()
4685 						set_bit(In_sync, &rdev->flags);  in raid10_start_reshape()
4687 						rdev->recovery_offset = 0;  in raid10_start_reshape()
4692 			} else if (rdev->raid_disk >= conf->prev.raid_disks  in raid10_start_reshape()
4693 				   && !test_bit(Faulty, &rdev->flags)) {  in raid10_start_reshape()
4695 				set_bit(In_sync, &rdev->flags);  in raid10_start_reshape()
4699 	 * ->degraded is measured against the larger of the  in raid10_start_reshape()
4702 	spin_lock_irq(&conf->device_lock);  in raid10_start_reshape()
4703 	mddev->degraded = calc_degraded(conf);  in raid10_start_reshape()
4704 	spin_unlock_irq(&conf->device_lock);  in raid10_start_reshape()
4705 	mddev->raid_disks = conf->geo.raid_disks;  in raid10_start_reshape()
4706 	mddev->reshape_position = conf->reshape_progress;  in raid10_start_reshape()
4707 	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);  in raid10_start_reshape()
4709 	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);  in raid10_start_reshape()
4710 	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);  in raid10_start_reshape()
4711 	clear_bit(MD_RECOVERY_DONE, &mddev->recovery);  in raid10_start_reshape()
4712 	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);  in raid10_start_reshape()
4713 	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);  in raid10_start_reshape()
4715 	rcu_assign_pointer(mddev->sync_thread,  in raid10_start_reshape()
4717 	if (!mddev->sync_thread) {  in raid10_start_reshape()
4718 		ret = -EAGAIN;  in raid10_start_reshape()
4721 	conf->reshape_checkpoint = jiffies;  in raid10_start_reshape()
4722 	md_wakeup_thread(mddev->sync_thread);  in raid10_start_reshape()
4727 	mddev->recovery = 0;  in raid10_start_reshape()
4728 	spin_lock_irq(&conf->device_lock);  in raid10_start_reshape()
4729 	conf->geo = conf->prev;  in raid10_start_reshape()
4730 	mddev->raid_disks = conf->geo.raid_disks;  in raid10_start_reshape()
4732 		rdev->new_data_offset = rdev->data_offset;  in raid10_start_reshape()
4734 	conf->reshape_progress = MaxSector;  in raid10_start_reshape()
4735 	conf->reshape_safe = MaxSector;  in raid10_start_reshape()
4736 	mddev->reshape_position = MaxSector;  in raid10_start_reshape()
4737 	spin_unlock_irq(&conf->device_lock);  in raid10_start_reshape()
4741 /* Calculate the last device-address that could contain
4742  * any block from the chunk that includes the array-address 's'
4744  * i.e. the address returned will be chunk-aligned and after
4749 	s = (s | geo->chunk_mask) + 1;  in last_dev_address()
4750 	s >>= geo->chunk_shift;  in last_dev_address()
4751 	s *= geo->near_copies;  in last_dev_address()
4752 	s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks);  in last_dev_address()
4753 	s *= geo->far_copies;  in last_dev_address()
4754 	s <<= geo->chunk_shift;  in last_dev_address()
4758 /* Calculate the first device-address that could contain
4759  * any block from the chunk that includes the array-address 's'.
4764 	s >>= geo->chunk_shift;  in first_dev_address()
4765 	s *= geo->near_copies;  in first_dev_address()
4766 	sector_div(s, geo->raid_disks);  in first_dev_address()
4767 	s *= geo->far_copies;  in first_dev_address()
4768 	s <<= geo->chunk_shift;  in first_dev_address()
4779 	 * a chunk, but that is not a problem as mddev->reshape_position  in reshape_request()
4784 	 * we need to flush all reshape requests and update the metadata.  in reshape_request()
4795 	 * we must update the metadata to increase the 'safe' position.  in reshape_request()
4802 	 * (conf->offset_diff - always positive) allows a bit of slack,  in reshape_request()
4807 	 * The means one for each copy for write-out and an extra one for  in reshape_request()
4808 	 * read-in.  in reshape_request()
4809 	 * We store the read-in bio in ->master_bio and the others in  in reshape_request()
4810 	 * ->devs[x].bio and ->devs[x].repl_bio.  in reshape_request()
4812 	struct r10conf *conf = mddev->private;  in reshape_request()
4827 		if (mddev->reshape_backwards &&  in reshape_request()
4828 		    conf->reshape_progress < raid10_size(mddev, 0, 0)) {  in reshape_request()
4830 				     - conf->reshape_progress);  in reshape_request()
4831 		} else if (!mddev->reshape_backwards &&  in reshape_request()
4832 			   conf->reshape_progress > 0)  in reshape_request()
4833 			sector_nr = conf->reshape_progress;  in reshape_request()
4835 			mddev->curr_resync_completed = sector_nr;  in reshape_request()
4836 			sysfs_notify_dirent_safe(mddev->sysfs_completed);  in reshape_request()
4843 	 * as that doesn't work well for ->reshape_backwards.  in reshape_request()
4844 	 * So just use ->reshape_progress.  in reshape_request()
4846 	if (mddev->reshape_backwards) {  in reshape_request()
4850 		next = first_dev_address(conf->reshape_progress - 1,  in reshape_request()
4851 					 &conf->geo);  in reshape_request()
4856 		safe = last_dev_address(conf->reshape_safe - 1,  in reshape_request()
4857 					&conf->prev);  in reshape_request()
4859 		if (next + conf->offset_diff < safe)  in reshape_request()
4862 		last = conf->reshape_progress - 1;  in reshape_request()
4863 		sector_nr = last & ~(sector_t)(conf->geo.chunk_mask  in reshape_request()
4864 					       & conf->prev.chunk_mask);  in reshape_request()
4866 			sector_nr = last + 1 - RESYNC_SECTORS;  in reshape_request()
4871 		next = last_dev_address(conf->reshape_progress, &conf->geo);  in reshape_request()
4876 		safe = first_dev_address(conf->reshape_safe, &conf->prev);  in reshape_request()
4878 		/* Need to update metadata if 'next' might be beyond 'safe'  in reshape_request()
4881 		if (next > safe + conf->offset_diff)  in reshape_request()
4884 		sector_nr = conf->reshape_progress;  in reshape_request()
4885 		last  = sector_nr | (conf->geo.chunk_mask  in reshape_request()
4886 				     & conf->prev.chunk_mask);  in reshape_request()
4889 			last = sector_nr + RESYNC_SECTORS - 1;  in reshape_request()
4893 	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {  in reshape_request()
4894 		/* Need to update reshape_position in metadata */  in reshape_request()
4896 		mddev->reshape_position = conf->reshape_progress;  in reshape_request()
4897 		if (mddev->reshape_backwards)  in reshape_request()
4898 			mddev->curr_resync_completed = raid10_size(mddev, 0, 0)  in reshape_request()
4899 				- conf->reshape_progress;  in reshape_request()
4901 			mddev->curr_resync_completed = conf->reshape_progress;  in reshape_request()
4902 		conf->reshape_checkpoint = jiffies;  in reshape_request()
4903 		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);  in reshape_request()
4904 		md_wakeup_thread(mddev->thread);  in reshape_request()
4905 		wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||  in reshape_request()
4906 			   test_bit(MD_RECOVERY_INTR, &mddev->recovery));  in reshape_request()
4907 		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {  in reshape_request()
4911 		conf->reshape_safe = mddev->reshape_position;  in reshape_request()
4919 	r10_bio->state = 0;  in reshape_request()
4921 	atomic_set(&r10_bio->remaining, 0);  in reshape_request()
4922 	r10_bio->mddev = mddev;  in reshape_request()
4923 	r10_bio->sector = sector_nr;  in reshape_request()
4924 	set_bit(R10BIO_IsReshape, &r10_bio->state);  in reshape_request()
4925 	r10_bio->sectors = last - sector_nr + 1;  in reshape_request()
4927 	BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state));  in reshape_request()
4934 		mempool_free(r10_bio, &conf->r10buf_pool);  in reshape_request()
4935 		set_bit(MD_RECOVERY_INTR, &mddev->recovery);  in reshape_request()
4939 	read_bio = bio_alloc_bioset(rdev->bdev, RESYNC_PAGES, REQ_OP_READ,  in reshape_request()
4940 				    GFP_KERNEL, &mddev->bio_set);  in reshape_request()
4941 	read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr  in reshape_request()
4942 			       + rdev->data_offset);  in reshape_request()
4943 	read_bio->bi_private = r10_bio;  in reshape_request()
4944 	read_bio->bi_end_io = end_reshape_read;  in reshape_request()
4945 	r10_bio->master_bio = read_bio;  in reshape_request()
4946 	r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;  in reshape_request()
4952 	if (mddev_is_clustered(mddev) && conf->cluster_sync_high <= sector_nr) {  in reshape_request()
4956 		conf->cluster_sync_low = sector_nr;  in reshape_request()
4957 		conf->cluster_sync_high = sector_nr + CLUSTER_RESYNC_WINDOW_SECTORS;  in reshape_request()
4958 		sb = page_address(rdev->sb_page);  in reshape_request()
4960 			sb_reshape_pos = le64_to_cpu(sb->reshape_position);  in reshape_request()
4964 			 * update cluster_sync_low until it has finished reshape.  in reshape_request()
4966 			if (sb_reshape_pos < conf->cluster_sync_low)  in reshape_request()
4967 				conf->cluster_sync_low = sb_reshape_pos;  in reshape_request()
4970 		md_cluster_ops->resync_info_update(mddev, conf->cluster_sync_low,  in reshape_request()
4971 							  conf->cluster_sync_high);  in reshape_request()
4975 	__raid10_find_phys(&conf->geo, r10_bio);  in reshape_request()
4978 	read_bio->bi_next = NULL;  in reshape_request()
4981 	for (s = 0; s < conf->copies*2; s++) {  in reshape_request()
4983 		int d = r10_bio->devs[s/2].devnum;  in reshape_request()
4986 			rdev2 = rcu_dereference(conf->mirrors[d].replacement);  in reshape_request()
4987 			b = r10_bio->devs[s/2].repl_bio;  in reshape_request()
4989 			rdev2 = rcu_dereference(conf->mirrors[d].rdev);  in reshape_request()
4990 			b = r10_bio->devs[s/2].bio;  in reshape_request()
4992 		if (!rdev2 || test_bit(Faulty, &rdev2->flags))  in reshape_request()
4995 		bio_set_dev(b, rdev2->bdev);  in reshape_request()
4996 		b->bi_iter.bi_sector = r10_bio->devs[s/2].addr +  in reshape_request()
4997 			rdev2->new_data_offset;  in reshape_request()
4998 		b->bi_end_io = end_reshape_write;  in reshape_request()
4999 		b->bi_opf = REQ_OP_WRITE;  in reshape_request()
5000 		b->bi_next = blist;  in reshape_request()
5007 	pages = get_resync_pages(r10_bio->devs[0].bio)->pages;  in reshape_request()
5010 		int len = (max_sectors - s) << 9;  in reshape_request()
5013 		for (bio = blist; bio ; bio = bio->bi_next) {  in reshape_request()
5015 				bio->bi_status = BLK_STS_RESOURCE;  in reshape_request()
5024 	r10_bio->sectors = nr_sectors;  in reshape_request()
5027 	md_sync_acct_bio(read_bio, r10_bio->sectors);  in reshape_request()
5028 	atomic_inc(&r10_bio->remaining);  in reshape_request()
5029 	read_bio->bi_next = NULL;  in reshape_request()
5038 	 * update reshape_progress  in reshape_request()
5040 	if (mddev->reshape_backwards)  in reshape_request()
5041 		conf->reshape_progress -= sectors_done;  in reshape_request()
5043 		conf->reshape_progress += sectors_done;  in reshape_request()
5055 	 * If we got a read error then we do sync 1-page reads from  in reshape_request_write()
5056 	 * elsewhere until we find the data - or give up.  in reshape_request_write()
5058 	struct r10conf *conf = mddev->private;  in reshape_request_write()
5061 	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))  in reshape_request_write()
5064 			md_done_sync(mddev, r10_bio->sectors, 0);  in reshape_request_write()
5071 	atomic_set(&r10_bio->remaining, 1);  in reshape_request_write()
5072 	for (s = 0; s < conf->copies*2; s++) {  in reshape_request_write()
5074 		int d = r10_bio->devs[s/2].devnum;  in reshape_request_write()
5078 			rdev = rcu_dereference(conf->mirrors[d].replacement);  in reshape_request_write()
5079 			b = r10_bio->devs[s/2].repl_bio;  in reshape_request_write()
5081 			rdev = rcu_dereference(conf->mirrors[d].rdev);  in reshape_request_write()
5082 			b = r10_bio->devs[s/2].bio;  in reshape_request_write()
5084 		if (!rdev || test_bit(Faulty, &rdev->flags)) {  in reshape_request_write()
5088 		atomic_inc(&rdev->nr_pending);  in reshape_request_write()
5090 		md_sync_acct_bio(b, r10_bio->sectors);  in reshape_request_write()
5091 		atomic_inc(&r10_bio->remaining);  in reshape_request_write()
5092 		b->bi_next = NULL;  in reshape_request_write()
5100 	if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))  in end_reshape()
5103 	spin_lock_irq(&conf->device_lock);  in end_reshape()
5104 	conf->prev = conf->geo;  in end_reshape()
5105 	md_finish_reshape(conf->mddev);  in end_reshape()
5107 	conf->reshape_progress = MaxSector;  in end_reshape()
5108 	conf->reshape_safe = MaxSector;  in end_reshape()
5109 	spin_unlock_irq(&conf->device_lock);  in end_reshape()
5111 	if (conf->mddev->queue)  in end_reshape()
5113 	conf->fullsync = 0;  in end_reshape()
5118 	struct r10conf *conf = mddev->private;  in raid10_update_reshape_pos()
5121 	md_cluster_ops->resync_info_get(mddev, &lo, &hi);  in raid10_update_reshape_pos()
5122 	if (((mddev->reshape_position <= hi) && (mddev->reshape_position >= lo))  in raid10_update_reshape_pos()
5123 	    || mddev->reshape_position == MaxSector)  in raid10_update_reshape_pos()
5124 		conf->reshape_progress = mddev->reshape_position;  in raid10_update_reshape_pos()
5133 	int sectors = r10_bio->sectors;  in handle_reshape_read_error()
5134 	struct r10conf *conf = mddev->private;  in handle_reshape_read_error()
5140 	r10b = kmalloc(struct_size(r10b, devs, conf->copies), GFP_NOIO);  in handle_reshape_read_error()
5142 		set_bit(MD_RECOVERY_INTR, &mddev->recovery);  in handle_reshape_read_error()
5143 		return -ENOMEM;  in handle_reshape_read_error()
5147 	pages = get_resync_pages(r10_bio->devs[0].bio)->pages;  in handle_reshape_read_error()
5149 	r10b->sector = r10_bio->sector;  in handle_reshape_read_error()
5150 	__raid10_find_phys(&conf->prev, r10b);  in handle_reshape_read_error()
5162 			int d = r10b->devs[slot].devnum;  in handle_reshape_read_error()
5163 			struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);  in handle_reshape_read_error()
5166 			    test_bit(Faulty, &rdev->flags) ||  in handle_reshape_read_error()
5167 			    !test_bit(In_sync, &rdev->flags))  in handle_reshape_read_error()
5170 			addr = r10b->devs[slot].addr + idx * PAGE_SIZE;  in handle_reshape_read_error()
5171 			atomic_inc(&rdev->nr_pending);  in handle_reshape_read_error()
5184 			if (slot >= conf->copies)  in handle_reshape_read_error()
5193 				&mddev->recovery);  in handle_reshape_read_error()
5195 			return -EIO;  in handle_reshape_read_error()
5197 		sectors -= s;  in handle_reshape_read_error()
5207 	struct mddev *mddev = r10_bio->mddev;  in end_reshape_write()
5208 	struct r10conf *conf = mddev->private;  in end_reshape_write()
5216 		rdev = conf->mirrors[d].replacement;  in end_reshape_write()
5219 		rdev = conf->mirrors[d].rdev;  in end_reshape_write()
5222 	if (bio->bi_status) {  in end_reshape_write()
5233 	if (!atomic_dec_and_test(&r10_bio->remaining))  in end_reshape_request()
5235 	md_done_sync(r10_bio->mddev, r10_bio->sectors, 1);  in end_reshape_request()
5236 	bio_put(r10_bio->master_bio);  in end_reshape_request()
5242 	struct r10conf *conf = mddev->private;  in raid10_finish_reshape()
5244 	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))  in raid10_finish_reshape()
5247 	if (mddev->delta_disks > 0) {  in raid10_finish_reshape()
5248 		if (mddev->recovery_cp > mddev->resync_max_sectors) {  in raid10_finish_reshape()
5249 			mddev->recovery_cp = mddev->resync_max_sectors;  in raid10_finish_reshape()
5250 			set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);  in raid10_finish_reshape()
5252 		mddev->resync_max_sectors = mddev->array_sectors;  in raid10_finish_reshape()
5256 		for (d = conf->geo.raid_disks ;  in raid10_finish_reshape()
5257 		     d < conf->geo.raid_disks - mddev->delta_disks;  in raid10_finish_reshape()
5259 			struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);  in raid10_finish_reshape()
5261 				clear_bit(In_sync, &rdev->flags);  in raid10_finish_reshape()
5262 			rdev = rcu_dereference(conf->mirrors[d].replacement);  in raid10_finish_reshape()
5264 				clear_bit(In_sync, &rdev->flags);  in raid10_finish_reshape()
5268 	mddev->layout = mddev->new_layout;  in raid10_finish_reshape()
5269 	mddev->chunk_sectors = 1 << conf->geo.chunk_shift;  in raid10_finish_reshape()
5270 	mddev->reshape_position = MaxSector;  in raid10_finish_reshape()
5271 	mddev->delta_disks = 0;  in raid10_finish_reshape()
5272 	mddev->reshape_backwards = 0;  in raid10_finish_reshape()
5313 MODULE_ALIAS("md-personality-9"); /* RAID10 */
5314 MODULE_ALIAS("md-raid10");
5315 MODULE_ALIAS("md-level-10");