26  * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
119 static inline int raid6_d0(struct stripe_head *sh)  in raid6_d0()  argument
121 	if (sh->ddf_layout)  in raid6_d0()
125 	if (sh->qd_idx == sh->disks - 1)  in raid6_d0()
128 		return sh->qd_idx + 1;  in raid6_d0()
141 static int raid6_idx_to_slot(int idx, struct stripe_head *sh,  in raid6_idx_to_slot()  argument
146 	if (sh->ddf_layout)  in raid6_idx_to_slot()
148 	if (idx == sh->pd_idx)  in raid6_idx_to_slot()
150 	if (idx == sh->qd_idx)  in raid6_idx_to_slot()
152 	if (!sh->ddf_layout)  in raid6_idx_to_slot()
159 static int stripe_operations_active(struct stripe_head *sh)  in stripe_operations_active()  argument
161 	return sh->check_state || sh->reconstruct_state ||  in stripe_operations_active()
162 	       test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||  in stripe_operations_active()
163 	       test_bit(STRIPE_COMPUTE_RUN, &sh->state);  in stripe_operations_active()
166 static bool stripe_is_lowprio(struct stripe_head *sh)  in stripe_is_lowprio()  argument
168 	return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) ||  in stripe_is_lowprio()
169 		test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) &&  in stripe_is_lowprio()
170 	       !test_bit(STRIPE_R5C_CACHING, &sh->state);  in stripe_is_lowprio()
173 static void raid5_wakeup_stripe_thread(struct stripe_head *sh)  in raid5_wakeup_stripe_thread()  argument
174 	__must_hold(&sh->raid_conf->device_lock)  in raid5_wakeup_stripe_thread()
176 	struct r5conf *conf = sh->raid_conf;  in raid5_wakeup_stripe_thread()
179 	int i, cpu = sh->cpu;  in raid5_wakeup_stripe_thread()
183 		sh->cpu = cpu;  in raid5_wakeup_stripe_thread()
186 	if (list_empty(&sh->lru)) {  in raid5_wakeup_stripe_thread()
189 		if (stripe_is_lowprio(sh))  in raid5_wakeup_stripe_thread()
190 			list_add_tail(&sh->lru, &group->loprio_list);  in raid5_wakeup_stripe_thread()
192 			list_add_tail(&sh->lru, &group->handle_list);  in raid5_wakeup_stripe_thread()
194 		sh->group = group;  in raid5_wakeup_stripe_thread()
202 	group = conf->worker_groups + cpu_to_group(sh->cpu);  in raid5_wakeup_stripe_thread()
206 	queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);  in raid5_wakeup_stripe_thread()
213 			queue_work_on(sh->cpu, raid5_wq,  in raid5_wakeup_stripe_thread()
220 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,  in do_release_stripe()  argument
227 	BUG_ON(!list_empty(&sh->lru));  in do_release_stripe()
231 		for (i = sh->disks; i--; )  in do_release_stripe()
232 			if (test_bit(R5_InJournal, &sh->dev[i].flags))  in do_release_stripe()
241 	if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) ||  in do_release_stripe()
243 	     !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) {  in do_release_stripe()
244 		if (test_bit(STRIPE_R5C_CACHING, &sh->state))  in do_release_stripe()
245 			r5c_make_stripe_write_out(sh);  in do_release_stripe()
246 		set_bit(STRIPE_HANDLE, &sh->state);  in do_release_stripe()
249 	if (test_bit(STRIPE_HANDLE, &sh->state)) {  in do_release_stripe()
250 		if (test_bit(STRIPE_DELAYED, &sh->state) &&  in do_release_stripe()
251 		    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in do_release_stripe()
252 			list_add_tail(&sh->lru, &conf->delayed_list);  in do_release_stripe()
253 		else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&  in do_release_stripe()
254 			   sh->bm_seq - conf->seq_write > 0)  in do_release_stripe()
255 			list_add_tail(&sh->lru, &conf->bitmap_list);  in do_release_stripe()
257 			clear_bit(STRIPE_DELAYED, &sh->state);  in do_release_stripe()
258 			clear_bit(STRIPE_BIT_DELAY, &sh->state);  in do_release_stripe()
260 				if (stripe_is_lowprio(sh))  in do_release_stripe()
261 					list_add_tail(&sh->lru,  in do_release_stripe()
264 					list_add_tail(&sh->lru,  in do_release_stripe()
267 				raid5_wakeup_stripe_thread(sh);  in do_release_stripe()
273 		BUG_ON(stripe_operations_active(sh));  in do_release_stripe()
274 		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in do_release_stripe()
279 		if (!test_bit(STRIPE_EXPANDING, &sh->state)) {  in do_release_stripe()
281 				list_add_tail(&sh->lru, temp_inactive_list);  in do_release_stripe()
283 				WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags));  in do_release_stripe()
285 					list_add_tail(&sh->lru, temp_inactive_list);  in do_release_stripe()
288 					if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state))  in do_release_stripe()
290 					if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))  in do_release_stripe()
292 					list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);  in do_release_stripe()
300 					list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);  in do_release_stripe()
306 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,  in __release_stripe()  argument
310 	if (atomic_dec_and_test(&sh->count))  in __release_stripe()
311 		do_release_stripe(conf, sh, temp_inactive_list);  in __release_stripe()
367 	struct stripe_head *sh, *t;  in release_stripe_list()  local
373 	llist_for_each_entry_safe(sh, t, head, release_list) {  in release_stripe_list()
376 		/* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */  in release_stripe_list()
378 		clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);  in release_stripe_list()
384 		hash = sh->hash_lock_index;  in release_stripe_list()
385 		__release_stripe(conf, sh, &temp_inactive_list[hash]);  in release_stripe_list()
392 void raid5_release_stripe(struct stripe_head *sh)  in raid5_release_stripe()  argument
394 	struct r5conf *conf = sh->raid_conf;  in raid5_release_stripe()
402 	if (atomic_add_unless(&sh->count, -1, 1))  in raid5_release_stripe()
406 		test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))  in raid5_release_stripe()
408 	wakeup = llist_add(&sh->release_list, &conf->released_stripes);  in raid5_release_stripe()
414 	if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) {  in raid5_release_stripe()
416 		hash = sh->hash_lock_index;  in raid5_release_stripe()
417 		do_release_stripe(conf, sh, &list);  in raid5_release_stripe()
423 static inline void remove_hash(struct stripe_head *sh)  in remove_hash()  argument
426 		(unsigned long long)sh->sector);  in remove_hash()
428 	hlist_del_init(&sh->hash);  in remove_hash()
431 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)  in insert_hash()  argument
433 	struct hlist_head *hp = stripe_hash(conf, sh->sector);  in insert_hash()
436 		(unsigned long long)sh->sector);  in insert_hash()
438 	hlist_add_head(&sh->hash, hp);  in insert_hash()
444 	struct stripe_head *sh = NULL;  in get_free_stripe()  local
450 	sh = list_entry(first, struct stripe_head, lru);  in get_free_stripe()
452 	remove_hash(sh);  in get_free_stripe()
454 	BUG_ON(hash != sh->hash_lock_index);  in get_free_stripe()
458 	return sh;  in get_free_stripe()
462 static void free_stripe_pages(struct stripe_head *sh)  in free_stripe_pages()  argument
468 	if (!sh->pages)  in free_stripe_pages()
471 	for (i = 0; i < sh->nr_pages; i++) {  in free_stripe_pages()
472 		p = sh->pages[i];  in free_stripe_pages()
475 		sh->pages[i] = NULL;  in free_stripe_pages()
479 static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp)  in alloc_stripe_pages()  argument
484 	for (i = 0; i < sh->nr_pages; i++) {  in alloc_stripe_pages()
486 		if (sh->pages[i])  in alloc_stripe_pages()
491 			free_stripe_pages(sh);  in alloc_stripe_pages()
494 		sh->pages[i] = p;  in alloc_stripe_pages()
500 init_stripe_shared_pages(struct stripe_head *sh, struct r5conf *conf, int disks)  in init_stripe_shared_pages()  argument
504 	if (sh->pages)  in init_stripe_shared_pages()
507 	/* Each of the sh->dev[i] need one conf->stripe_size */  in init_stripe_shared_pages()
511 	sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);  in init_stripe_shared_pages()
512 	if (!sh->pages)  in init_stripe_shared_pages()
514 	sh->nr_pages = nr_pages;  in init_stripe_shared_pages()
515 	sh->stripes_per_page = cnt;  in init_stripe_shared_pages()
520 static void shrink_buffers(struct stripe_head *sh)  in shrink_buffers()  argument
523 	int num = sh->raid_conf->pool_size;  in shrink_buffers()
529 		WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);  in shrink_buffers()
530 		p = sh->dev[i].page;  in shrink_buffers()
533 		sh->dev[i].page = NULL;  in shrink_buffers()
538 		sh->dev[i].page = NULL;  in shrink_buffers()
539 	free_stripe_pages(sh); /* Free pages */  in shrink_buffers()
543 static int grow_buffers(struct stripe_head *sh, gfp_t gfp)  in grow_buffers()  argument
546 	int num = sh->raid_conf->pool_size;  in grow_buffers()
555 		sh->dev[i].page = page;  in grow_buffers()
556 		sh->dev[i].orig_page = page;  in grow_buffers()
557 		sh->dev[i].offset = 0;  in grow_buffers()
560 	if (alloc_stripe_pages(sh, gfp))  in grow_buffers()
564 		sh->dev[i].page = raid5_get_dev_page(sh, i);  in grow_buffers()
565 		sh->dev[i].orig_page = sh->dev[i].page;  in grow_buffers()
566 		sh->dev[i].offset = raid5_get_page_offset(sh, i);  in grow_buffers()
573 			    struct stripe_head *sh);
575 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)  in init_stripe()  argument
577 	struct r5conf *conf = sh->raid_conf;  in init_stripe()
580 	BUG_ON(atomic_read(&sh->count) != 0);  in init_stripe()
581 	BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));  in init_stripe()
582 	BUG_ON(stripe_operations_active(sh));  in init_stripe()
583 	BUG_ON(sh->batch_head);  in init_stripe()
589 	sh->generation = conf->generation - previous;  in init_stripe()
590 	sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;  in init_stripe()
591 	sh->sector = sector;  in init_stripe()
592 	stripe_set_idx(sector, conf, previous, sh);  in init_stripe()
593 	sh->state = 0;  in init_stripe()
595 	for (i = sh->disks; i--; ) {  in init_stripe()
596 		struct r5dev *dev = &sh->dev[i];  in init_stripe()
601 			       (unsigned long long)sh->sector, i, dev->toread,  in init_stripe()
607 		dev->sector = raid5_compute_blocknr(sh, i, previous);  in init_stripe()
611 	sh->overwrite_disks = 0;  in init_stripe()
612 	insert_hash(conf, sh);  in init_stripe()
613 	sh->cpu = smp_processor_id();  in init_stripe()
614 	set_bit(STRIPE_BATCH_READY, &sh->state);  in init_stripe()
620 	struct stripe_head *sh;  in __find_stripe()  local
623 	hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)  in __find_stripe()
624 		if (sh->sector == sector && sh->generation == generation)  in __find_stripe()
625 			return sh;  in __find_stripe()
634 	struct stripe_head *sh;  in find_get_stripe()  local
636 	sh = __find_stripe(conf, sector, generation);  in find_get_stripe()
637 	if (!sh)  in find_get_stripe()
640 	if (atomic_inc_not_zero(&sh->count))  in find_get_stripe()
641 		return sh;  in find_get_stripe()
645 	 * be on a list (sh->lru). Must remove the stripe from the list that  in find_get_stripe()
650 	if (!atomic_read(&sh->count)) {  in find_get_stripe()
651 		if (!test_bit(STRIPE_HANDLE, &sh->state))  in find_get_stripe()
653 		BUG_ON(list_empty(&sh->lru) &&  in find_get_stripe()
654 		       !test_bit(STRIPE_EXPANDING, &sh->state));  in find_get_stripe()
658 		list_del_init(&sh->lru);  in find_get_stripe()
662 		if (sh->group) {  in find_get_stripe()
663 			sh->group->stripes_cnt--;  in find_get_stripe()
664 			sh->group = NULL;  in find_get_stripe()
667 	atomic_inc(&sh->count);  in find_get_stripe()
670 	return sh;  in find_get_stripe()
807 	struct stripe_head *sh;  in raid5_get_active_stripe()  local
834 		sh = find_get_stripe(conf, sector, conf->generation - previous,  in raid5_get_active_stripe()
836 		if (sh)  in raid5_get_active_stripe()
840 			sh = get_free_stripe(conf, hash);  in raid5_get_active_stripe()
841 			if (sh) {  in raid5_get_active_stripe()
843 				init_stripe(sh, sector, previous);  in raid5_get_active_stripe()
844 				atomic_inc(&sh->count);  in raid5_get_active_stripe()
871 	return sh;  in raid5_get_active_stripe()
874 static bool is_full_stripe_write(struct stripe_head *sh)  in is_full_stripe_write()  argument
876 	BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));  in is_full_stripe_write()
877 	return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);  in is_full_stripe_write()
902 static bool stripe_can_batch(struct stripe_head *sh)  in stripe_can_batch()  argument
904 	struct r5conf *conf = sh->raid_conf;  in stripe_can_batch()
908 	return test_bit(STRIPE_BATCH_READY, &sh->state) &&  in stripe_can_batch()
909 		!test_bit(STRIPE_BITMAP_PENDING, &sh->state) &&  in stripe_can_batch()
910 		is_full_stripe_write(sh);  in stripe_can_batch()
915 		struct stripe_head *sh, struct stripe_head *last_sh)  in stripe_add_to_batch_list()  argument
923 	tmp_sec = sh->sector;  in stripe_add_to_batch_list()
926 	head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);  in stripe_add_to_batch_list()
943 	lock_two_stripes(head, sh);  in stripe_add_to_batch_list()
945 	if (!stripe_can_batch(head) || !stripe_can_batch(sh))  in stripe_add_to_batch_list()
948 	if (sh->batch_head)  in stripe_add_to_batch_list()
952 	while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)  in stripe_add_to_batch_list()
954 	if (head->dev[dd_idx].towrite->bi_opf != sh->dev[dd_idx].towrite->bi_opf ||  in stripe_add_to_batch_list()
955 	    bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite))  in stripe_add_to_batch_list()
972 		sh->batch_head = head->batch_head;  in stripe_add_to_batch_list()
978 		list_add(&sh->batch_list, &head->batch_list);  in stripe_add_to_batch_list()
982 		sh->batch_head = head->batch_head;  in stripe_add_to_batch_list()
984 		list_add_tail(&sh->batch_list, &head->batch_list);  in stripe_add_to_batch_list()
988 	if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in stripe_add_to_batch_list()
993 	if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) {  in stripe_add_to_batch_list()
994 		int seq = sh->bm_seq;  in stripe_add_to_batch_list()
995 		if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) &&  in stripe_add_to_batch_list()
996 		    sh->batch_head->bm_seq > seq)  in stripe_add_to_batch_list()
997 			seq = sh->batch_head->bm_seq;  in stripe_add_to_batch_list()
998 		set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state);  in stripe_add_to_batch_list()
999 		sh->batch_head->bm_seq = seq;  in stripe_add_to_batch_list()
1002 	atomic_inc(&sh->count);  in stripe_add_to_batch_list()
1004 	unlock_two_stripes(head, sh);  in stripe_add_to_batch_list()
1012 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)  in use_new_offset()  argument
1022 	if (sh->generation == conf->generation - 1)  in use_new_offset()
1139 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)  in ops_run_io()  argument
1141 	struct r5conf *conf = sh->raid_conf;  in ops_run_io()
1142 	int i, disks = sh->disks;  in ops_run_io()
1143 	struct stripe_head *head_sh = sh;  in ops_run_io()
1150 	if (log_stripe(sh, s) == 0)  in ops_run_io()
1162 		sh = head_sh;  in ops_run_io()
1163 		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {  in ops_run_io()
1165 			if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))  in ops_run_io()
1167 			if (test_bit(R5_Discard, &sh->dev[i].flags))  in ops_run_io()
1169 		} else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))  in ops_run_io()
1172 					    &sh->dev[i].flags)) {  in ops_run_io()
1177 		if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))  in ops_run_io()
1181 		dev = &sh->dev[i];  in ops_run_io()
1223 			int bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),  in ops_run_io()
1257 			set_bit(STRIPE_IO_STARTED, &sh->state);  in ops_run_io()
1263 			bi->bi_private = sh;  in ops_run_io()
1266 				__func__, (unsigned long long)sh->sector,  in ops_run_io()
1268 			atomic_inc(&sh->count);  in ops_run_io()
1269 			if (sh != head_sh)  in ops_run_io()
1271 			if (use_new_offset(conf, sh))  in ops_run_io()
1272 				bi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1275 				bi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1280 			if (test_bit(R5_SkipCopy, &sh->dev[i].flags))  in ops_run_io()
1281 				WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));  in ops_run_io()
1284 			    test_bit(R5_InJournal, &sh->dev[i].flags))  in ops_run_io()
1290 				sh->dev[i].vec.bv_page = sh->dev[i].orig_page;  in ops_run_io()
1292 				sh->dev[i].vec.bv_page = sh->dev[i].page;  in ops_run_io()
1295 			bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;  in ops_run_io()
1304 				set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);  in ops_run_io()
1309 						sh->dev[i].sector);  in ops_run_io()
1320 			set_bit(STRIPE_IO_STARTED, &sh->state);  in ops_run_io()
1325 			rbi->bi_private = sh;  in ops_run_io()
1329 				__func__, (unsigned long long)sh->sector,  in ops_run_io()
1331 			atomic_inc(&sh->count);  in ops_run_io()
1332 			if (sh != head_sh)  in ops_run_io()
1334 			if (use_new_offset(conf, sh))  in ops_run_io()
1335 				rbi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1338 				rbi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1340 			if (test_bit(R5_SkipCopy, &sh->dev[i].flags))  in ops_run_io()
1341 				WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));  in ops_run_io()
1342 			sh->dev[i].rvec.bv_page = sh->dev[i].page;  in ops_run_io()
1345 			rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset;  in ops_run_io()
1356 						sh->dev[i].sector);  in ops_run_io()
1364 				set_bit(STRIPE_DEGRADED, &sh->state);  in ops_run_io()
1366 				bi->bi_opf, i, (unsigned long long)sh->sector);  in ops_run_io()
1367 			clear_bit(R5_LOCKED, &sh->dev[i].flags);  in ops_run_io()
1368 			set_bit(STRIPE_HANDLE, &sh->state);  in ops_run_io()
1373 		sh = list_first_entry(&sh->batch_list, struct stripe_head,  in ops_run_io()
1375 		if (sh != head_sh)  in ops_run_io()
1386 	struct stripe_head *sh, int no_skipcopy)  in async_copy_data()  argument
1394 	struct r5conf *conf = sh->raid_conf;  in async_copy_data()
1450 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_biofill()  local
1452 	struct r5conf *conf = sh->raid_conf;  in ops_complete_biofill()
1455 		(unsigned long long)sh->sector);  in ops_complete_biofill()
1458 	for (i = sh->disks; i--; ) {  in ops_complete_biofill()
1459 		struct r5dev *dev = &sh->dev[i];  in ops_complete_biofill()
1480 	clear_bit(STRIPE_BIOFILL_RUN, &sh->state);  in ops_complete_biofill()
1482 	set_bit(STRIPE_HANDLE, &sh->state);  in ops_complete_biofill()
1483 	raid5_release_stripe(sh);  in ops_complete_biofill()
1486 static void ops_run_biofill(struct stripe_head *sh)  in ops_run_biofill()  argument
1491 	struct r5conf *conf = sh->raid_conf;  in ops_run_biofill()
1493 	BUG_ON(sh->batch_head);  in ops_run_biofill()
1495 		(unsigned long long)sh->sector);  in ops_run_biofill()
1497 	for (i = sh->disks; i--; ) {  in ops_run_biofill()
1498 		struct r5dev *dev = &sh->dev[i];  in ops_run_biofill()
1501 			spin_lock_irq(&sh->stripe_lock);  in ops_run_biofill()
1504 			spin_unlock_irq(&sh->stripe_lock);  in ops_run_biofill()
1509 						     dev->sector, tx, sh, 0);  in ops_run_biofill()
1515 	atomic_inc(&sh->count);  in ops_run_biofill()
1516 	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);  in ops_run_biofill()
1520 static void mark_target_uptodate(struct stripe_head *sh, int target)  in mark_target_uptodate()  argument
1527 	tgt = &sh->dev[target];  in mark_target_uptodate()
1535 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_compute()  local
1538 		(unsigned long long)sh->sector);  in ops_complete_compute()
1541 	mark_target_uptodate(sh, sh->ops.target);  in ops_complete_compute()
1542 	mark_target_uptodate(sh, sh->ops.target2);  in ops_complete_compute()
1544 	clear_bit(STRIPE_COMPUTE_RUN, &sh->state);  in ops_complete_compute()
1545 	if (sh->check_state == check_state_compute_run)  in ops_complete_compute()
1546 		sh->check_state = check_state_compute_result;  in ops_complete_compute()
1547 	set_bit(STRIPE_HANDLE, &sh->state);  in ops_complete_compute()
1548 	raid5_release_stripe(sh);  in ops_complete_compute()
1558 static addr_conv_t *to_addr_conv(struct stripe_head *sh,  in to_addr_conv()  argument
1561 	return (void *) (to_addr_page(percpu, i) + sh->disks + 2);  in to_addr_conv()
1568 to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu)  in to_addr_offs()  argument
1570 	return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);  in to_addr_offs()
1574 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_compute5()  argument
1576 	int disks = sh->disks;  in ops_run_compute5()
1578 	unsigned int *off_srcs = to_addr_offs(sh, percpu);  in ops_run_compute5()
1579 	int target = sh->ops.target;  in ops_run_compute5()
1580 	struct r5dev *tgt = &sh->dev[target];  in ops_run_compute5()
1588 	BUG_ON(sh->batch_head);  in ops_run_compute5()
1591 		__func__, (unsigned long long)sh->sector, target);  in ops_run_compute5()
1596 			off_srcs[count] = sh->dev[i].offset;  in ops_run_compute5()
1597 			xor_srcs[count++] = sh->dev[i].page;  in ops_run_compute5()
1601 	atomic_inc(&sh->count);  in ops_run_compute5()
1604 			  ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));  in ops_run_compute5()
1607 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_compute5()
1610 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_compute5()
1616  * @srcs - (struct page *) array of size sh->disks
1618  * @sh - stripe_head to parse
1627 				struct stripe_head *sh,  in set_syndrome_sources()  argument
1630 	int disks = sh->disks;  in set_syndrome_sources()
1631 	int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);  in set_syndrome_sources()
1632 	int d0_idx = raid6_d0(sh);  in set_syndrome_sources()
1642 		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);  in set_syndrome_sources()
1643 		struct r5dev *dev = &sh->dev[i];  in set_syndrome_sources()
1645 		if (i == sh->qd_idx || i == sh->pd_idx ||  in set_syndrome_sources()
1654 				srcs[slot] = sh->dev[i].orig_page;  in set_syndrome_sources()
1656 				srcs[slot] = sh->dev[i].page;  in set_syndrome_sources()
1662 			offs[slot] = sh->dev[i].offset;  in set_syndrome_sources()
1671 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_compute6_1()  argument
1673 	int disks = sh->disks;  in ops_run_compute6_1()
1675 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_compute6_1()
1677 	int qd_idx = sh->qd_idx;  in ops_run_compute6_1()
1686 	BUG_ON(sh->batch_head);  in ops_run_compute6_1()
1687 	if (sh->ops.target < 0)  in ops_run_compute6_1()
1688 		target = sh->ops.target2;  in ops_run_compute6_1()
1689 	else if (sh->ops.target2 < 0)  in ops_run_compute6_1()
1690 		target = sh->ops.target;  in ops_run_compute6_1()
1696 		__func__, (unsigned long long)sh->sector, target);  in ops_run_compute6_1()
1698 	tgt = &sh->dev[target];  in ops_run_compute6_1()
1703 	atomic_inc(&sh->count);  in ops_run_compute6_1()
1706 		count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);  in ops_run_compute6_1()
1710 				  ops_complete_compute, sh,  in ops_run_compute6_1()
1711 				  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_1()
1713 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_compute6_1()
1720 			offs[count] = sh->dev[i].offset;  in ops_run_compute6_1()
1721 			blocks[count++] = sh->dev[i].page;  in ops_run_compute6_1()
1725 				  NULL, ops_complete_compute, sh,  in ops_run_compute6_1()
1726 				  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_1()
1728 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_compute6_1()
1735 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_compute6_2()  argument
1737 	int i, count, disks = sh->disks;  in ops_run_compute6_2()
1738 	int syndrome_disks = sh->ddf_layout ? disks : disks-2;  in ops_run_compute6_2()
1739 	int d0_idx = raid6_d0(sh);  in ops_run_compute6_2()
1741 	int target = sh->ops.target;  in ops_run_compute6_2()
1742 	int target2 = sh->ops.target2;  in ops_run_compute6_2()
1743 	struct r5dev *tgt = &sh->dev[target];  in ops_run_compute6_2()
1744 	struct r5dev *tgt2 = &sh->dev[target2];  in ops_run_compute6_2()
1747 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_compute6_2()
1750 	BUG_ON(sh->batch_head);  in ops_run_compute6_2()
1752 		 __func__, (unsigned long long)sh->sector, target, target2);  in ops_run_compute6_2()
1767 		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);  in ops_run_compute6_2()
1769 		offs[slot] = sh->dev[i].offset;  in ops_run_compute6_2()
1770 		blocks[slot] = sh->dev[i].page;  in ops_run_compute6_2()
1783 		 __func__, (unsigned long long)sh->sector, faila, failb);  in ops_run_compute6_2()
1785 	atomic_inc(&sh->count);  in ops_run_compute6_2()
1792 					  ops_complete_compute, sh,  in ops_run_compute6_2()
1793 					  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1795 						  RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1801 			int qd_idx = sh->qd_idx;  in ops_run_compute6_2()
1813 				offs[count] = sh->dev[i].offset;  in ops_run_compute6_2()
1814 				blocks[count++] = sh->dev[i].page;  in ops_run_compute6_2()
1816 			dest = sh->dev[data_target].page;  in ops_run_compute6_2()
1817 			dest_off = sh->dev[data_target].offset;  in ops_run_compute6_2()
1821 					  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1823 				       RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1826 			count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);  in ops_run_compute6_2()
1828 					  ops_complete_compute, sh,  in ops_run_compute6_2()
1829 					  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1831 						  RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1836 				  ops_complete_compute, sh,  in ops_run_compute6_2()
1837 				  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1841 						RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1847 						RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1856 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_prexor()  local
1859 		(unsigned long long)sh->sector);  in ops_complete_prexor()
1861 	if (r5c_is_writeback(sh->raid_conf->log))  in ops_complete_prexor()
1866 		r5c_release_extra_page(sh);  in ops_complete_prexor()
1870 ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_prexor5()  argument
1873 	int disks = sh->disks;  in ops_run_prexor5()
1875 	unsigned int *off_srcs = to_addr_offs(sh, percpu);  in ops_run_prexor5()
1876 	int count = 0, pd_idx = sh->pd_idx, i;  in ops_run_prexor5()
1880 	unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;  in ops_run_prexor5()
1881 	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;  in ops_run_prexor5()
1883 	BUG_ON(sh->batch_head);  in ops_run_prexor5()
1885 		(unsigned long long)sh->sector);  in ops_run_prexor5()
1888 		struct r5dev *dev = &sh->dev[i];  in ops_run_prexor5()
1904 			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));  in ops_run_prexor5()
1906 			RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_prexor5()
1912 ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_prexor6()  argument
1916 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_prexor6()
1921 		(unsigned long long)sh->sector);  in ops_run_prexor6()
1923 	count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);  in ops_run_prexor6()
1926 			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));  in ops_run_prexor6()
1928 			RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_prexor6()
1934 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)  in ops_run_biodrain()  argument
1936 	struct r5conf *conf = sh->raid_conf;  in ops_run_biodrain()
1937 	int disks = sh->disks;  in ops_run_biodrain()
1939 	struct stripe_head *head_sh = sh;  in ops_run_biodrain()
1942 		(unsigned long long)sh->sector);  in ops_run_biodrain()
1948 		sh = head_sh;  in ops_run_biodrain()
1953 			dev = &sh->dev[i];  in ops_run_biodrain()
1959 			spin_lock_irq(&sh->stripe_lock);  in ops_run_biodrain()
1962 			sh->overwrite_disks = 0;  in ops_run_biodrain()
1965 			spin_unlock_irq(&sh->stripe_lock);  in ops_run_biodrain()
1979 							     dev->sector, tx, sh,  in ops_run_biodrain()
1992 				sh = list_first_entry(&sh->batch_list,  in ops_run_biodrain()
1995 				if (sh == head_sh)  in ops_run_biodrain()
2007 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_reconstruct()  local
2008 	int disks = sh->disks;  in ops_complete_reconstruct()
2009 	int pd_idx = sh->pd_idx;  in ops_complete_reconstruct()
2010 	int qd_idx = sh->qd_idx;  in ops_complete_reconstruct()
2015 		(unsigned long long)sh->sector);  in ops_complete_reconstruct()
2018 		fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);  in ops_complete_reconstruct()
2019 		sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);  in ops_complete_reconstruct()
2020 		discard |= test_bit(R5_Discard, &sh->dev[i].flags);  in ops_complete_reconstruct()
2024 		struct r5dev *dev = &sh->dev[i];  in ops_complete_reconstruct()
2029 				if (test_bit(STRIPE_EXPAND_READY, &sh->state))  in ops_complete_reconstruct()
2039 	if (sh->reconstruct_state == reconstruct_state_drain_run)  in ops_complete_reconstruct()
2040 		sh->reconstruct_state = reconstruct_state_drain_result;  in ops_complete_reconstruct()
2041 	else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)  in ops_complete_reconstruct()
2042 		sh->reconstruct_state = reconstruct_state_prexor_drain_result;  in ops_complete_reconstruct()
2044 		BUG_ON(sh->reconstruct_state != reconstruct_state_run);  in ops_complete_reconstruct()
2045 		sh->reconstruct_state = reconstruct_state_result;  in ops_complete_reconstruct()
2048 	set_bit(STRIPE_HANDLE, &sh->state);  in ops_complete_reconstruct()
2049 	raid5_release_stripe(sh);  in ops_complete_reconstruct()
2053 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_reconstruct5()  argument
2056 	int disks = sh->disks;  in ops_run_reconstruct5()
2060 	int count, pd_idx = sh->pd_idx, i;  in ops_run_reconstruct5()
2066 	struct stripe_head *head_sh = sh;  in ops_run_reconstruct5()
2070 		(unsigned long long)sh->sector);  in ops_run_reconstruct5()
2072 	for (i = 0; i < sh->disks; i++) {  in ops_run_reconstruct5()
2075 		if (!test_bit(R5_Discard, &sh->dev[i].flags))  in ops_run_reconstruct5()
2078 	if (i >= sh->disks) {  in ops_run_reconstruct5()
2079 		atomic_inc(&sh->count);  in ops_run_reconstruct5()
2080 		set_bit(R5_Discard, &sh->dev[pd_idx].flags);  in ops_run_reconstruct5()
2081 		ops_complete_reconstruct(sh);  in ops_run_reconstruct5()
2087 	off_srcs = to_addr_offs(sh, percpu);  in ops_run_reconstruct5()
2093 		off_dest = off_srcs[count] = sh->dev[pd_idx].offset;  in ops_run_reconstruct5()
2094 		xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;  in ops_run_reconstruct5()
2096 			struct r5dev *dev = &sh->dev[i];  in ops_run_reconstruct5()
2104 		xor_dest = sh->dev[pd_idx].page;  in ops_run_reconstruct5()
2105 		off_dest = sh->dev[pd_idx].offset;  in ops_run_reconstruct5()
2107 			struct r5dev *dev = &sh->dev[i];  in ops_run_reconstruct5()
2121 		list_first_entry(&sh->batch_list,  in ops_run_reconstruct5()
2129 				  to_addr_conv(sh, percpu, j));  in ops_run_reconstruct5()
2133 				  to_addr_conv(sh, percpu, j));  in ops_run_reconstruct5()
2138 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_reconstruct5()
2141 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_reconstruct5()
2144 		sh = list_first_entry(&sh->batch_list, struct stripe_head,  in ops_run_reconstruct5()
2151 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_reconstruct6()  argument
2158 	struct stripe_head *head_sh = sh;  in ops_run_reconstruct6()
2163 	pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);  in ops_run_reconstruct6()
2165 	for (i = 0; i < sh->disks; i++) {  in ops_run_reconstruct6()
2166 		if (sh->pd_idx == i || sh->qd_idx == i)  in ops_run_reconstruct6()
2168 		if (!test_bit(R5_Discard, &sh->dev[i].flags))  in ops_run_reconstruct6()
2171 	if (i >= sh->disks) {  in ops_run_reconstruct6()
2172 		atomic_inc(&sh->count);  in ops_run_reconstruct6()
2173 		set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);  in ops_run_reconstruct6()
2174 		set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);  in ops_run_reconstruct6()
2175 		ops_complete_reconstruct(sh);  in ops_run_reconstruct6()
2181 	offs = to_addr_offs(sh, percpu);  in ops_run_reconstruct6()
2183 	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {  in ops_run_reconstruct6()
2191 	count = set_syndrome_sources(blocks, offs, sh, synflags);  in ops_run_reconstruct6()
2193 		list_first_entry(&sh->batch_list,  in ops_run_reconstruct6()
2199 				  head_sh, to_addr_conv(sh, percpu, j));  in ops_run_reconstruct6()
2202 				  to_addr_conv(sh, percpu, j));  in ops_run_reconstruct6()
2204 			RAID5_STRIPE_SIZE(sh->raid_conf),  &submit);  in ops_run_reconstruct6()
2207 		sh = list_first_entry(&sh->batch_list, struct stripe_head,  in ops_run_reconstruct6()
2215 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_check()  local
2218 		(unsigned long long)sh->sector);  in ops_complete_check()
2220 	sh->check_state = check_state_check_result;  in ops_complete_check()
2221 	set_bit(STRIPE_HANDLE, &sh->state);  in ops_complete_check()
2222 	raid5_release_stripe(sh);  in ops_complete_check()
2225 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_check_p()  argument
2227 	int disks = sh->disks;  in ops_run_check_p()
2228 	int pd_idx = sh->pd_idx;  in ops_run_check_p()
2229 	int qd_idx = sh->qd_idx;  in ops_run_check_p()
2233 	unsigned int *off_srcs = to_addr_offs(sh, percpu);  in ops_run_check_p()
2240 		(unsigned long long)sh->sector);  in ops_run_check_p()
2242 	BUG_ON(sh->batch_head);  in ops_run_check_p()
2244 	xor_dest = sh->dev[pd_idx].page;  in ops_run_check_p()
2245 	off_dest = sh->dev[pd_idx].offset;  in ops_run_check_p()
2251 		off_srcs[count] = sh->dev[i].offset;  in ops_run_check_p()
2252 		xor_srcs[count++] = sh->dev[i].page;  in ops_run_check_p()
2256 			  to_addr_conv(sh, percpu, 0));  in ops_run_check_p()
2258 			   RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_check_p()
2259 			   &sh->ops.zero_sum_result, &submit);  in ops_run_check_p()
2261 	atomic_inc(&sh->count);  in ops_run_check_p()
2262 	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);  in ops_run_check_p()
2266 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)  in ops_run_check_pq()  argument
2269 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_check_pq()
2274 		(unsigned long long)sh->sector, checkp);  in ops_run_check_pq()
2276 	BUG_ON(sh->batch_head);  in ops_run_check_pq()
2277 	count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);  in ops_run_check_pq()
2281 	atomic_inc(&sh->count);  in ops_run_check_pq()
2283 			  sh, to_addr_conv(sh, percpu, 0));  in ops_run_check_pq()
2285 			   RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_check_pq()
2286 			   &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);  in ops_run_check_pq()
2289 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)  in raid_run_ops()  argument
2291 	int overlap_clear = 0, i, disks = sh->disks;  in raid_run_ops()
2293 	struct r5conf *conf = sh->raid_conf;  in raid_run_ops()
2300 		ops_run_biofill(sh);  in raid_run_ops()
2306 			tx = ops_run_compute5(sh, percpu);  in raid_run_ops()
2308 			if (sh->ops.target2 < 0 || sh->ops.target < 0)  in raid_run_ops()
2309 				tx = ops_run_compute6_1(sh, percpu);  in raid_run_ops()
2311 				tx = ops_run_compute6_2(sh, percpu);  in raid_run_ops()
2320 			tx = ops_run_prexor5(sh, percpu, tx);  in raid_run_ops()
2322 			tx = ops_run_prexor6(sh, percpu, tx);  in raid_run_ops()
2326 		tx = ops_run_partial_parity(sh, percpu, tx);  in raid_run_ops()
2329 		tx = ops_run_biodrain(sh, tx);  in raid_run_ops()
2335 			ops_run_reconstruct5(sh, percpu, tx);  in raid_run_ops()
2337 			ops_run_reconstruct6(sh, percpu, tx);  in raid_run_ops()
2341 		if (sh->check_state == check_state_run)  in raid_run_ops()
2342 			ops_run_check_p(sh, percpu);  in raid_run_ops()
2343 		else if (sh->check_state == check_state_run_q)  in raid_run_ops()
2344 			ops_run_check_pq(sh, percpu, 0);  in raid_run_ops()
2345 		else if (sh->check_state == check_state_run_pq)  in raid_run_ops()
2346 			ops_run_check_pq(sh, percpu, 1);  in raid_run_ops()
2351 	if (overlap_clear && !sh->batch_head) {  in raid_run_ops()
2353 			struct r5dev *dev = &sh->dev[i];  in raid_run_ops()
2355 				wake_up(&sh->raid_conf->wait_for_overlap);  in raid_run_ops()
2361 static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh)  in free_stripe()  argument
2364 	kfree(sh->pages);  in free_stripe()
2366 	if (sh->ppl_page)  in free_stripe()
2367 		__free_page(sh->ppl_page);  in free_stripe()
2368 	kmem_cache_free(sc, sh);  in free_stripe()
2374 	struct stripe_head *sh;  in alloc_stripe()  local
2376 	sh = kmem_cache_zalloc(sc, gfp);  in alloc_stripe()
2377 	if (sh) {  in alloc_stripe()
2378 		spin_lock_init(&sh->stripe_lock);  in alloc_stripe()
2379 		spin_lock_init(&sh->batch_lock);  in alloc_stripe()
2380 		INIT_LIST_HEAD(&sh->batch_list);  in alloc_stripe()
2381 		INIT_LIST_HEAD(&sh->lru);  in alloc_stripe()
2382 		INIT_LIST_HEAD(&sh->r5c);  in alloc_stripe()
2383 		INIT_LIST_HEAD(&sh->log_list);  in alloc_stripe()
2384 		atomic_set(&sh->count, 1);  in alloc_stripe()
2385 		sh->raid_conf = conf;  in alloc_stripe()
2386 		sh->log_start = MaxSector;  in alloc_stripe()
2389 			sh->ppl_page = alloc_page(gfp);  in alloc_stripe()
2390 			if (!sh->ppl_page) {  in alloc_stripe()
2391 				free_stripe(sc, sh);  in alloc_stripe()
2396 		if (init_stripe_shared_pages(sh, conf, disks)) {  in alloc_stripe()
2397 			free_stripe(sc, sh);  in alloc_stripe()
2402 	return sh;  in alloc_stripe()
2406 	struct stripe_head *sh;  in grow_one_stripe()  local
2408 	sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf);  in grow_one_stripe()
2409 	if (!sh)  in grow_one_stripe()
2412 	if (grow_buffers(sh, gfp)) {  in grow_one_stripe()
2413 		shrink_buffers(sh);  in grow_one_stripe()
2414 		free_stripe(conf->slab_cache, sh);  in grow_one_stripe()
2417 	sh->hash_lock_index =  in grow_one_stripe()
2422 	raid5_release_stripe(sh);  in grow_one_stripe()
2708 	struct stripe_head *sh;  in drop_one_stripe()  local
2712 	sh = get_free_stripe(conf, hash);  in drop_one_stripe()
2714 	if (!sh)  in drop_one_stripe()
2716 	BUG_ON(atomic_read(&sh->count));  in drop_one_stripe()
2717 	shrink_buffers(sh);  in drop_one_stripe()
2718 	free_stripe(conf->slab_cache, sh);  in drop_one_stripe()
2758 	struct stripe_head *sh = bi->bi_private;  in raid5_end_read_request()  local
2759 	struct r5conf *conf = sh->raid_conf;  in raid5_end_read_request()
2760 	int disks = sh->disks, i;  in raid5_end_read_request()
2765 		if (bi == &sh->dev[i].req)  in raid5_end_read_request()
2769 		(unsigned long long)sh->sector, i, atomic_read(&sh->count),  in raid5_end_read_request()
2775 	if (test_bit(R5_ReadRepl, &sh->dev[i].flags))  in raid5_end_read_request()
2785 	if (use_new_offset(conf, sh))  in raid5_end_read_request()
2786 		s = sh->sector + rdev->new_data_offset;  in raid5_end_read_request()
2788 		s = sh->sector + rdev->data_offset;  in raid5_end_read_request()
2790 		set_bit(R5_UPTODATE, &sh->dev[i].flags);  in raid5_end_read_request()
2791 		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {  in raid5_end_read_request()
2802 			clear_bit(R5_ReadError, &sh->dev[i].flags);  in raid5_end_read_request()
2803 			clear_bit(R5_ReWrite, &sh->dev[i].flags);  in raid5_end_read_request()
2804 		} else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))  in raid5_end_read_request()
2805 			clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);  in raid5_end_read_request()
2807 		if (test_bit(R5_InJournal, &sh->dev[i].flags))  in raid5_end_read_request()
2812 			set_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);  in raid5_end_read_request()
2820 		clear_bit(R5_UPTODATE, &sh->dev[i].flags);  in raid5_end_read_request()
2823 		if (test_bit(R5_ReadRepl, &sh->dev[i].flags))  in raid5_end_read_request()
2836 		} else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {  in raid5_end_read_request()
2857 		    && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))  in raid5_end_read_request()
2860 			if (sh->qd_idx >= 0 && sh->pd_idx == i)  in raid5_end_read_request()
2861 				set_bit(R5_ReadError, &sh->dev[i].flags);  in raid5_end_read_request()
2862 			else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {  in raid5_end_read_request()
2863 				set_bit(R5_ReadError, &sh->dev[i].flags);  in raid5_end_read_request()
2864 				clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);  in raid5_end_read_request()
2866 				set_bit(R5_ReadNoMerge, &sh->dev[i].flags);  in raid5_end_read_request()
2868 			clear_bit(R5_ReadError, &sh->dev[i].flags);  in raid5_end_read_request()
2869 			clear_bit(R5_ReWrite, &sh->dev[i].flags);  in raid5_end_read_request()
2873 				      rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))  in raid5_end_read_request()
2879 	clear_bit(R5_LOCKED, &sh->dev[i].flags);  in raid5_end_read_request()
2880 	set_bit(STRIPE_HANDLE, &sh->state);  in raid5_end_read_request()
2881 	raid5_release_stripe(sh);  in raid5_end_read_request()
2886 	struct stripe_head *sh = bi->bi_private;  in raid5_end_write_request()  local
2887 	struct r5conf *conf = sh->raid_conf;  in raid5_end_write_request()
2888 	int disks = sh->disks, i;  in raid5_end_write_request()
2895 		if (bi == &sh->dev[i].req) {  in raid5_end_write_request()
2899 		if (bi == &sh->dev[i].rreq) {  in raid5_end_write_request()
2913 		(unsigned long long)sh->sector, i, atomic_read(&sh->count),  in raid5_end_write_request()
2923 		else if (is_badblock(rdev, sh->sector,  in raid5_end_write_request()
2926 			set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);  in raid5_end_write_request()
2929 			set_bit(STRIPE_DEGRADED, &sh->state);  in raid5_end_write_request()
2931 			set_bit(R5_WriteError, &sh->dev[i].flags);  in raid5_end_write_request()
2935 		} else if (is_badblock(rdev, sh->sector,  in raid5_end_write_request()
2938 			set_bit(R5_MadeGood, &sh->dev[i].flags);  in raid5_end_write_request()
2939 			if (test_bit(R5_ReadError, &sh->dev[i].flags))  in raid5_end_write_request()
2944 				set_bit(R5_ReWrite, &sh->dev[i].flags);  in raid5_end_write_request()
2949 	if (sh->batch_head && bi->bi_status && !replacement)  in raid5_end_write_request()
2950 		set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);  in raid5_end_write_request()
2953 	if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))  in raid5_end_write_request()
2954 		clear_bit(R5_LOCKED, &sh->dev[i].flags);  in raid5_end_write_request()
2955 	set_bit(STRIPE_HANDLE, &sh->state);  in raid5_end_write_request()
2957 	if (sh->batch_head && sh != sh->batch_head)  in raid5_end_write_request()
2958 		raid5_release_stripe(sh->batch_head);  in raid5_end_write_request()
2959 	raid5_release_stripe(sh);  in raid5_end_write_request()
3002 			      struct stripe_head *sh)  in raid5_compute_sector()  argument
3190 	if (sh) {  in raid5_compute_sector()
3191 		sh->pd_idx = pd_idx;  in raid5_compute_sector()
3192 		sh->qd_idx = qd_idx;  in raid5_compute_sector()
3193 		sh->ddf_layout = ddf_layout;  in raid5_compute_sector()
3202 sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous)  in raid5_compute_blocknr()  argument
3204 	struct r5conf *conf = sh->raid_conf;  in raid5_compute_blocknr()
3205 	int raid_disks = sh->disks;  in raid5_compute_blocknr()
3207 	sector_t new_sector = sh->sector, check;  in raid5_compute_blocknr()
3222 	if (i == sh->pd_idx)  in raid5_compute_blocknr()
3230 			if (i > sh->pd_idx)  in raid5_compute_blocknr()
3235 			if (i < sh->pd_idx)  in raid5_compute_blocknr()
3237 			i -= (sh->pd_idx + 1);  in raid5_compute_blocknr()
3249 		if (i == sh->qd_idx)  in raid5_compute_blocknr()
3256 			if (sh->pd_idx == raid_disks-1)  in raid5_compute_blocknr()
3258 			else if (i > sh->pd_idx)  in raid5_compute_blocknr()
3263 			if (sh->pd_idx == raid_disks-1)  in raid5_compute_blocknr()
3267 				if (i < sh->pd_idx)  in raid5_compute_blocknr()
3269 				i -= (sh->pd_idx + 2);  in raid5_compute_blocknr()
3279 			if (sh->pd_idx == 0)  in raid5_compute_blocknr()
3283 				if (i < sh->pd_idx)  in raid5_compute_blocknr()
3285 				i -= (sh->pd_idx + 1);  in raid5_compute_blocknr()
3290 			if (i > sh->pd_idx)  in raid5_compute_blocknr()
3295 			if (i < sh->pd_idx)  in raid5_compute_blocknr()
3297 			i -= (sh->pd_idx + 1);  in raid5_compute_blocknr()
3313 	if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx  in raid5_compute_blocknr()
3314 		|| sh2.qd_idx != sh->qd_idx) {  in raid5_compute_blocknr()
3379 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,  in schedule_reconstruction()  argument
3382 	int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;  in schedule_reconstruction()
3383 	struct r5conf *conf = sh->raid_conf;  in schedule_reconstruction()
3393 		r5c_release_extra_page(sh);  in schedule_reconstruction()
3396 			struct r5dev *dev = &sh->dev[i];  in schedule_reconstruction()
3417 			sh->reconstruct_state = reconstruct_state_drain_run;  in schedule_reconstruction()
3420 			sh->reconstruct_state = reconstruct_state_run;  in schedule_reconstruction()
3425 			if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))  in schedule_reconstruction()
3428 		BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||  in schedule_reconstruction()
3429 			test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));  in schedule_reconstruction()
3431 			(!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||  in schedule_reconstruction()
3432 			   test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));  in schedule_reconstruction()
3435 			struct r5dev *dev = &sh->dev[i];  in schedule_reconstruction()
3454 		sh->reconstruct_state = reconstruct_state_prexor_drain_run;  in schedule_reconstruction()
3463 	set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);  in schedule_reconstruction()
3464 	clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);  in schedule_reconstruction()
3468 		int qd_idx = sh->qd_idx;  in schedule_reconstruction()
3469 		struct r5dev *dev = &sh->dev[qd_idx];  in schedule_reconstruction()
3476 	if (raid5_has_ppl(sh->raid_conf) && sh->ppl_page &&  in schedule_reconstruction()
3478 	    !test_bit(STRIPE_FULL_WRITE, &sh->state) &&  in schedule_reconstruction()
3479 	    test_bit(R5_Insync, &sh->dev[pd_idx].flags))  in schedule_reconstruction()
3483 		__func__, (unsigned long long)sh->sector,  in schedule_reconstruction()
3487 static bool stripe_bio_overlaps(struct stripe_head *sh, struct bio *bi,  in stripe_bio_overlaps()  argument
3490 	struct r5conf *conf = sh->raid_conf;  in stripe_bio_overlaps()
3494 		 bi->bi_iter.bi_sector, sh->sector);  in stripe_bio_overlaps()
3497 	if (sh->batch_head)  in stripe_bio_overlaps()
3501 		bip = &sh->dev[dd_idx].towrite;  in stripe_bio_overlaps()
3503 		bip = &sh->dev[dd_idx].toread;  in stripe_bio_overlaps()
3528 		for (i = 0; i < sh->disks; i++) {  in stripe_bio_overlaps()
3529 			if (i != sh->pd_idx &&  in stripe_bio_overlaps()
3530 			    (i == dd_idx || sh->dev[i].towrite)) {  in stripe_bio_overlaps()
3531 				sector = sh->dev[i].sector;  in stripe_bio_overlaps()
3547 static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi,  in __add_stripe_bio()  argument
3550 	struct r5conf *conf = sh->raid_conf;  in __add_stripe_bio()
3555 		bip = &sh->dev[dd_idx].towrite;  in __add_stripe_bio()
3559 		bip = &sh->dev[dd_idx].toread;  in __add_stripe_bio()
3566 		clear_bit(STRIPE_BATCH_READY, &sh->state);  in __add_stripe_bio()
3577 		sector_t sector = sh->dev[dd_idx].sector;  in __add_stripe_bio()
3578 		for (bi=sh->dev[dd_idx].towrite;  in __add_stripe_bio()
3579 		     sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&  in __add_stripe_bio()
3581 		     bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {  in __add_stripe_bio()
3585 		if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))  in __add_stripe_bio()
3586 			if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))  in __add_stripe_bio()
3587 				sh->overwrite_disks++;  in __add_stripe_bio()
3591 		 (*bip)->bi_iter.bi_sector, sh->sector, dd_idx,  in __add_stripe_bio()
3592 		 sh->dev[dd_idx].sector);  in __add_stripe_bio()
3607 		set_bit(STRIPE_BITMAP_PENDING, &sh->state);  in __add_stripe_bio()
3608 		spin_unlock_irq(&sh->stripe_lock);  in __add_stripe_bio()
3609 		md_bitmap_startwrite(conf->mddev->bitmap, sh->sector,  in __add_stripe_bio()
3611 		spin_lock_irq(&sh->stripe_lock);  in __add_stripe_bio()
3612 		clear_bit(STRIPE_BITMAP_PENDING, &sh->state);  in __add_stripe_bio()
3613 		if (!sh->batch_head) {  in __add_stripe_bio()
3614 			sh->bm_seq = conf->seq_flush+1;  in __add_stripe_bio()
3615 			set_bit(STRIPE_BIT_DELAY, &sh->state);  in __add_stripe_bio()
3625 static bool add_stripe_bio(struct stripe_head *sh, struct bio *bi,  in add_stripe_bio()  argument
3628 	spin_lock_irq(&sh->stripe_lock);  in add_stripe_bio()
3630 	if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) {  in add_stripe_bio()
3631 		set_bit(R5_Overlap, &sh->dev[dd_idx].flags);  in add_stripe_bio()
3632 		spin_unlock_irq(&sh->stripe_lock);  in add_stripe_bio()
3636 	__add_stripe_bio(sh, bi, dd_idx, forwrite, previous);  in add_stripe_bio()
3637 	spin_unlock_irq(&sh->stripe_lock);  in add_stripe_bio()
3644 			    struct stripe_head *sh)  in stripe_set_idx()  argument
3656 			     &dd_idx, sh);  in stripe_set_idx()
3660 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,  in handle_failed_stripe()  argument
3664 	BUG_ON(sh->batch_head);  in handle_failed_stripe()
3669 		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {  in handle_failed_stripe()
3682 					    sh->sector,  in handle_failed_stripe()
3688 		spin_lock_irq(&sh->stripe_lock);  in handle_failed_stripe()
3690 		bi = sh->dev[i].towrite;  in handle_failed_stripe()
3691 		sh->dev[i].towrite = NULL;  in handle_failed_stripe()
3692 		sh->overwrite_disks = 0;  in handle_failed_stripe()
3693 		spin_unlock_irq(&sh->stripe_lock);  in handle_failed_stripe()
3697 		log_stripe_write_finished(sh);  in handle_failed_stripe()
3699 		if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))  in handle_failed_stripe()
3703 			sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_failed_stripe()
3704 			struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3711 			md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,  in handle_failed_stripe()
3715 		bi = sh->dev[i].written;  in handle_failed_stripe()
3716 		sh->dev[i].written = NULL;  in handle_failed_stripe()
3717 		if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {  in handle_failed_stripe()
3718 			WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));  in handle_failed_stripe()
3719 			sh->dev[i].page = sh->dev[i].orig_page;  in handle_failed_stripe()
3724 		       sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_failed_stripe()
3725 			struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3735 		if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&  in handle_failed_stripe()
3737 		    (!test_bit(R5_Insync, &sh->dev[i].flags) ||  in handle_failed_stripe()
3738 		      test_bit(R5_ReadError, &sh->dev[i].flags))) {  in handle_failed_stripe()
3739 			spin_lock_irq(&sh->stripe_lock);  in handle_failed_stripe()
3740 			bi = sh->dev[i].toread;  in handle_failed_stripe()
3741 			sh->dev[i].toread = NULL;  in handle_failed_stripe()
3742 			spin_unlock_irq(&sh->stripe_lock);  in handle_failed_stripe()
3743 			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))  in handle_failed_stripe()
3748 			       sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_failed_stripe()
3750 					r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3757 			md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,  in handle_failed_stripe()
3762 		clear_bit(R5_LOCKED, &sh->dev[i].flags);  in handle_failed_stripe()
3767 	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))  in handle_failed_stripe()
3773 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,  in handle_failed_sync()  argument
3779 	BUG_ON(sh->batch_head);  in handle_failed_sync()
3780 	clear_bit(STRIPE_SYNCING, &sh->state);  in handle_failed_sync()
3781 	if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))  in handle_failed_sync()
3802 			    && !rdev_set_badblocks(rdev, sh->sector,  in handle_failed_sync()
3809 			    && !rdev_set_badblocks(rdev, sh->sector,  in handle_failed_sync()
3821 static int want_replace(struct stripe_head *sh, int disk_idx)  in want_replace()  argument
3827 	rdev = rcu_dereference(sh->raid_conf->disks[disk_idx].replacement);  in want_replace()
3831 	    && (rdev->recovery_offset <= sh->sector  in want_replace()
3832 		|| rdev->mddev->recovery_cp <= sh->sector))  in want_replace()
3838 static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,  in need_this_block()  argument
3841 	struct r5dev *dev = &sh->dev[disk_idx];  in need_this_block()
3842 	struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],  in need_this_block()
3843 				  &sh->dev[s->failed_num[1]] };  in need_this_block()
3845 	bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW);  in need_this_block()
3861 	    (s->replacing && want_replace(sh, disk_idx)))  in need_this_block()
3886 	    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in need_this_block()
3907 		     s->failed_num[i] == sh->pd_idx ||  in need_this_block()
3908 		     s->failed_num[i] == sh->qd_idx) &&  in need_this_block()
3925 	    sh->sector < sh->raid_conf->mddev->recovery_cp)  in need_this_block()
3929 		if (s->failed_num[i] != sh->pd_idx &&  in need_this_block()
3930 		    s->failed_num[i] != sh->qd_idx &&  in need_this_block()
3945 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,  in fetch_block()  argument
3948 	struct r5dev *dev = &sh->dev[disk_idx];  in fetch_block()
3951 	if (need_this_block(sh, s, disk_idx, disks)) {  in fetch_block()
3957 		BUG_ON(sh->batch_head);  in fetch_block()
3969 		    ((sh->qd_idx >= 0 && sh->pd_idx == disk_idx) ||  in fetch_block()
3976 			       (unsigned long long)sh->sector, disk_idx);  in fetch_block()
3977 			set_bit(STRIPE_COMPUTE_RUN, &sh->state);  in fetch_block()
3980 			sh->ops.target = disk_idx;  in fetch_block()
3981 			sh->ops.target2 = -1; /* no 2nd target */  in fetch_block()
4000 				      &sh->dev[other].flags))  in fetch_block()
4005 			       (unsigned long long)sh->sector,  in fetch_block()
4007 			set_bit(STRIPE_COMPUTE_RUN, &sh->state);  in fetch_block()
4009 			set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);  in fetch_block()
4010 			set_bit(R5_Wantcompute, &sh->dev[other].flags);  in fetch_block()
4011 			sh->ops.target = disk_idx;  in fetch_block()
4012 			sh->ops.target2 = other;  in fetch_block()
4031 static void handle_stripe_fill(struct stripe_head *sh,  in handle_stripe_fill()  argument
4041 	if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&  in handle_stripe_fill()
4042 	    !sh->reconstruct_state) {  in handle_stripe_fill()
4052 			if (test_bit(STRIPE_R5C_CACHING, &sh->state))  in handle_stripe_fill()
4053 				r5c_make_stripe_write_out(sh);  in handle_stripe_fill()
4058 			if (fetch_block(sh, s, i, disks))  in handle_stripe_fill()
4062 	set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe_fill()
4073 	struct stripe_head *sh, int disks)  in handle_stripe_clean_event()  argument
4078 	struct stripe_head *head_sh = sh;  in handle_stripe_clean_event()
4082 		if (sh->dev[i].written) {  in handle_stripe_clean_event()
4083 			dev = &sh->dev[i];  in handle_stripe_clean_event()
4109 				md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,  in handle_stripe_clean_event()
4111 						   !test_bit(STRIPE_DEGRADED, &sh->state),  in handle_stripe_clean_event()
4114 					sh = list_first_entry(&sh->batch_list,  in handle_stripe_clean_event()
4117 					if (sh != head_sh) {  in handle_stripe_clean_event()
4118 						dev = &sh->dev[i];  in handle_stripe_clean_event()
4122 				sh = head_sh;  in handle_stripe_clean_event()
4123 				dev = &sh->dev[i];  in handle_stripe_clean_event()
4128 	log_stripe_write_finished(sh);  in handle_stripe_clean_event()
4131 	    test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {  in handle_stripe_clean_event()
4133 		clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);  in handle_stripe_clean_event()
4134 		clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);  in handle_stripe_clean_event()
4135 		if (sh->qd_idx >= 0) {  in handle_stripe_clean_event()
4136 			clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);  in handle_stripe_clean_event()
4137 			clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);  in handle_stripe_clean_event()
4140 		clear_bit(STRIPE_DISCARD, &sh->state);  in handle_stripe_clean_event()
4147 		hash = sh->hash_lock_index;  in handle_stripe_clean_event()
4149 		remove_hash(sh);  in handle_stripe_clean_event()
4152 			sh = list_first_entry(&sh->batch_list,  in handle_stripe_clean_event()
4154 			if (sh != head_sh)  in handle_stripe_clean_event()
4157 		sh = head_sh;  in handle_stripe_clean_event()
4159 		if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))  in handle_stripe_clean_event()
4160 			set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe_clean_event()
4164 	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))  in handle_stripe_clean_event()
4188 				  struct stripe_head *sh,  in handle_stripe_dirtying()  argument
4203 	    (recovery_cp < MaxSector && sh->sector >= recovery_cp &&  in handle_stripe_dirtying()
4209 		pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n",  in handle_stripe_dirtying()
4211 			 (unsigned long long)sh->sector);  in handle_stripe_dirtying()
4214 		struct r5dev *dev = &sh->dev[i];  in handle_stripe_dirtying()
4216 		     i == sh->pd_idx || i == sh->qd_idx ||  in handle_stripe_dirtying()
4228 		    i != sh->pd_idx && i != sh->qd_idx &&  in handle_stripe_dirtying()
4240 		 (unsigned long long)sh->sector, sh->state, rmw, rcw);  in handle_stripe_dirtying()
4241 	set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe_dirtying()
4247 					  (unsigned long long)sh->sector, rmw);  in handle_stripe_dirtying()
4249 			struct r5dev *dev = &sh->dev[i];  in handle_stripe_dirtying()
4252 			    !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) {  in handle_stripe_dirtying()
4267 					r5c_use_extra_page(sh);  in handle_stripe_dirtying()
4272 				set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe_dirtying()
4279 			struct r5dev *dev = &sh->dev[i];  in handle_stripe_dirtying()
4281 			     i == sh->pd_idx || i == sh->qd_idx ||  in handle_stripe_dirtying()
4288 					     &sh->state)) {  in handle_stripe_dirtying()
4295 					set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe_dirtying()
4304 			struct r5dev *dev = &sh->dev[i];  in handle_stripe_dirtying()
4306 			    i != sh->pd_idx && i != sh->qd_idx &&  in handle_stripe_dirtying()
4313 					     &sh->state)) {  in handle_stripe_dirtying()
4321 					set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe_dirtying()
4326 					  (unsigned long long)sh->sector,  in handle_stripe_dirtying()
4327 					  rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));  in handle_stripe_dirtying()
4331 	    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in handle_stripe_dirtying()
4332 		set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe_dirtying()
4344 	if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&  in handle_stripe_dirtying()
4346 	     !test_bit(STRIPE_BIT_DELAY, &sh->state)))  in handle_stripe_dirtying()
4347 		schedule_reconstruction(sh, s, rcw == 0, 0);  in handle_stripe_dirtying()
4351 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,  in handle_parity_checks5()  argument
4356 	BUG_ON(sh->batch_head);  in handle_parity_checks5()
4357 	set_bit(STRIPE_HANDLE, &sh->state);  in handle_parity_checks5()
4359 	switch (sh->check_state) {  in handle_parity_checks5()
4364 			sh->check_state = check_state_run;  in handle_parity_checks5()
4366 			clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);  in handle_parity_checks5()
4370 		dev = &sh->dev[s->failed_num[0]];  in handle_parity_checks5()
4373 		sh->check_state = check_state_idle;  in handle_parity_checks5()
4375 			dev = &sh->dev[sh->pd_idx];  in handle_parity_checks5()
4378 		if (test_bit(STRIPE_INSYNC, &sh->state))  in handle_parity_checks5()
4389 		clear_bit(STRIPE_DEGRADED, &sh->state);  in handle_parity_checks5()
4390 		set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks5()
4395 		sh->check_state = check_state_idle;  in handle_parity_checks5()
4407 		if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)  in handle_parity_checks5()
4411 			set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks5()
4416 				set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks5()
4419 						    (unsigned long long) sh->sector,  in handle_parity_checks5()
4420 						    (unsigned long long) sh->sector +  in handle_parity_checks5()
4423 				sh->check_state = check_state_compute_run;  in handle_parity_checks5()
4424 				set_bit(STRIPE_COMPUTE_RUN, &sh->state);  in handle_parity_checks5()
4427 					&sh->dev[sh->pd_idx].flags);  in handle_parity_checks5()
4428 				sh->ops.target = sh->pd_idx;  in handle_parity_checks5()
4429 				sh->ops.target2 = -1;  in handle_parity_checks5()
4438 		       __func__, sh->check_state,  in handle_parity_checks5()
4439 		       (unsigned long long) sh->sector);  in handle_parity_checks5()
4444 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,  in handle_parity_checks6()  argument
4448 	int pd_idx = sh->pd_idx;  in handle_parity_checks6()
4449 	int qd_idx = sh->qd_idx;  in handle_parity_checks6()
4452 	BUG_ON(sh->batch_head);  in handle_parity_checks6()
4453 	set_bit(STRIPE_HANDLE, &sh->state);  in handle_parity_checks6()
4463 	switch (sh->check_state) {  in handle_parity_checks6()
4471 			sh->check_state = check_state_run;  in handle_parity_checks6()
4477 			if (sh->check_state == check_state_run)  in handle_parity_checks6()
4478 				sh->check_state = check_state_run_pq;  in handle_parity_checks6()
4480 				sh->check_state = check_state_run_q;  in handle_parity_checks6()
4484 		sh->ops.zero_sum_result = 0;  in handle_parity_checks6()
4486 		if (sh->check_state == check_state_run) {  in handle_parity_checks6()
4488 			clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);  in handle_parity_checks6()
4491 		if (sh->check_state >= check_state_run &&  in handle_parity_checks6()
4492 		    sh->check_state <= check_state_run_pq) {  in handle_parity_checks6()
4504 		sh->check_state = check_state_idle;  in handle_parity_checks6()
4507 		if (test_bit(STRIPE_INSYNC, &sh->state))  in handle_parity_checks6()
4515 			dev = &sh->dev[s->failed_num[1]];  in handle_parity_checks6()
4521 			dev = &sh->dev[s->failed_num[0]];  in handle_parity_checks6()
4526 		if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {  in handle_parity_checks6()
4527 			dev = &sh->dev[pd_idx];  in handle_parity_checks6()
4532 		if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {  in handle_parity_checks6()
4533 			dev = &sh->dev[qd_idx];  in handle_parity_checks6()
4541 			      dev - (struct r5dev *) &sh->dev)) {  in handle_parity_checks6()
4546 		clear_bit(STRIPE_DEGRADED, &sh->state);  in handle_parity_checks6()
4548 		set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks6()
4555 		sh->check_state = check_state_idle;  in handle_parity_checks6()
4561 		if (sh->ops.zero_sum_result == 0) {  in handle_parity_checks6()
4564 				set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks6()
4570 				sh->check_state = check_state_compute_result;  in handle_parity_checks6()
4581 				set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks6()
4584 						    (unsigned long long) sh->sector,  in handle_parity_checks6()
4585 						    (unsigned long long) sh->sector +  in handle_parity_checks6()
4588 				int *target = &sh->ops.target;  in handle_parity_checks6()
4590 				sh->ops.target = -1;  in handle_parity_checks6()
4591 				sh->ops.target2 = -1;  in handle_parity_checks6()
4592 				sh->check_state = check_state_compute_run;  in handle_parity_checks6()
4593 				set_bit(STRIPE_COMPUTE_RUN, &sh->state);  in handle_parity_checks6()
4595 				if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {  in handle_parity_checks6()
4597 						&sh->dev[pd_idx].flags);  in handle_parity_checks6()
4599 					target = &sh->ops.target2;  in handle_parity_checks6()
4602 				if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {  in handle_parity_checks6()
4604 						&sh->dev[qd_idx].flags);  in handle_parity_checks6()
4615 			__func__, sh->check_state,  in handle_parity_checks6()
4616 			(unsigned long long) sh->sector);  in handle_parity_checks6()
4621 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)  in handle_stripe_expansion()  argument
4629 	BUG_ON(sh->batch_head);  in handle_stripe_expansion()
4630 	clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);  in handle_stripe_expansion()
4631 	for (i = 0; i < sh->disks; i++)  in handle_stripe_expansion()
4632 		if (i != sh->pd_idx && i != sh->qd_idx) {  in handle_stripe_expansion()
4637 			sector_t bn = raid5_compute_blocknr(sh, i, 1);  in handle_stripe_expansion()
4658 					  sh->dev[i].page, sh2->dev[dd_idx].offset,  in handle_stripe_expansion()
4659 					  sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),  in handle_stripe_expansion()
4694 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)  in analyse_stripe()  argument
4696 	struct r5conf *conf = sh->raid_conf;  in analyse_stripe()
4697 	int disks = sh->disks;  in analyse_stripe()
4704 	s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;  in analyse_stripe()
4705 	s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;  in analyse_stripe()
4718 		dev = &sh->dev[i];  in analyse_stripe()
4729 		    !test_bit(STRIPE_BIOFILL_RUN, &sh->state))  in analyse_stripe()
4758 		    rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&  in analyse_stripe()
4759 		    !is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),  in analyse_stripe()
4773 			is_bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),  in analyse_stripe()
4800 		else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)  in analyse_stripe()
4870 	if (test_bit(STRIPE_SYNCING, &sh->state)) {  in analyse_stripe()
4880 		    sh->sector >= conf->mddev->recovery_cp ||  in analyse_stripe()
4893 static int clear_batch_ready(struct stripe_head *sh)  in clear_batch_ready()  argument
4896 	if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))  in clear_batch_ready()
4897 		return (sh->batch_head && sh->batch_head != sh);  in clear_batch_ready()
4898 	spin_lock(&sh->stripe_lock);  in clear_batch_ready()
4899 	if (!sh->batch_head) {  in clear_batch_ready()
4900 		spin_unlock(&sh->stripe_lock);  in clear_batch_ready()
4908 	if (sh->batch_head != sh) {  in clear_batch_ready()
4909 		spin_unlock(&sh->stripe_lock);  in clear_batch_ready()
4912 	spin_lock(&sh->batch_lock);  in clear_batch_ready()
4913 	list_for_each_entry(tmp, &sh->batch_list, batch_list)  in clear_batch_ready()
4915 	spin_unlock(&sh->batch_lock);  in clear_batch_ready()
4916 	spin_unlock(&sh->stripe_lock);  in clear_batch_ready()
4928 	struct stripe_head *sh, *next;  in break_stripe_batch_list()  local
4932 	list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) {  in break_stripe_batch_list()
4934 		list_del_init(&sh->batch_list);  in break_stripe_batch_list()
4936 		WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) |  in break_stripe_batch_list()
4948 			"stripe state: %lx\n", sh->state);  in break_stripe_batch_list()
4953 		set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS |  in break_stripe_batch_list()
4959 		sh->check_state = head_sh->check_state;  in break_stripe_batch_list()
4960 		sh->reconstruct_state = head_sh->reconstruct_state;  in break_stripe_batch_list()
4961 		spin_lock_irq(&sh->stripe_lock);  in break_stripe_batch_list()
4962 		sh->batch_head = NULL;  in break_stripe_batch_list()
4963 		spin_unlock_irq(&sh->stripe_lock);  in break_stripe_batch_list()
4964 		for (i = 0; i < sh->disks; i++) {  in break_stripe_batch_list()
4965 			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))  in break_stripe_batch_list()
4967 			sh->dev[i].flags = head_sh->dev[i].flags &  in break_stripe_batch_list()
4971 		    sh->state & handle_flags)  in break_stripe_batch_list()
4972 			set_bit(STRIPE_HANDLE, &sh->state);  in break_stripe_batch_list()
4973 		raid5_release_stripe(sh);  in break_stripe_batch_list()
4988 static void handle_stripe(struct stripe_head *sh)  in handle_stripe()  argument
4991 	struct r5conf *conf = sh->raid_conf;  in handle_stripe()
4994 	int disks = sh->disks;  in handle_stripe()
4997 	clear_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
5005 	if (clear_batch_ready(sh))  in handle_stripe()
5008 	if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {  in handle_stripe()
5011 		set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
5015 	if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))  in handle_stripe()
5016 		break_stripe_batch_list(sh, 0);  in handle_stripe()
5018 	if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {  in handle_stripe()
5019 		spin_lock(&sh->stripe_lock);  in handle_stripe()
5024 		if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&  in handle_stripe()
5025 		    !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) &&  in handle_stripe()
5026 		    !test_bit(STRIPE_DISCARD, &sh->state) &&  in handle_stripe()
5027 		    test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {  in handle_stripe()
5028 			set_bit(STRIPE_SYNCING, &sh->state);  in handle_stripe()
5029 			clear_bit(STRIPE_INSYNC, &sh->state);  in handle_stripe()
5030 			clear_bit(STRIPE_REPLACED, &sh->state);  in handle_stripe()
5032 		spin_unlock(&sh->stripe_lock);  in handle_stripe()
5034 	clear_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe()
5038 	       (unsigned long long)sh->sector, sh->state,  in handle_stripe()
5039 	       atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,  in handle_stripe()
5040 	       sh->check_state, sh->reconstruct_state);  in handle_stripe()
5042 	analyse_stripe(sh, &s);  in handle_stripe()
5044 	if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))  in handle_stripe()
5049 		set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
5056 			set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
5064 	if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {  in handle_stripe()
5066 		set_bit(STRIPE_BIOFILL_RUN, &sh->state);  in handle_stripe()
5082 		sh->check_state = 0;  in handle_stripe()
5083 		sh->reconstruct_state = 0;  in handle_stripe()
5084 		break_stripe_batch_list(sh, 0);  in handle_stripe()
5086 			handle_failed_stripe(conf, sh, &s, disks);  in handle_stripe()
5088 			handle_failed_sync(conf, sh, &s);  in handle_stripe()
5095 	if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)  in handle_stripe()
5097 	if (sh->reconstruct_state == reconstruct_state_drain_result ||  in handle_stripe()
5098 	    sh->reconstruct_state == reconstruct_state_prexor_drain_result) {  in handle_stripe()
5099 		sh->reconstruct_state = reconstruct_state_idle;  in handle_stripe()
5104 		BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&  in handle_stripe()
5105 		       !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));  in handle_stripe()
5106 		BUG_ON(sh->qd_idx >= 0 &&  in handle_stripe()
5107 		       !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&  in handle_stripe()
5108 		       !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));  in handle_stripe()
5110 			struct r5dev *dev = &sh->dev[i];  in handle_stripe()
5112 				(i == sh->pd_idx || i == sh->qd_idx ||  in handle_stripe()
5122 				    ((i == sh->pd_idx || i == sh->qd_idx)  &&  in handle_stripe()
5124 					set_bit(STRIPE_INSYNC, &sh->state);  in handle_stripe()
5127 		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in handle_stripe()
5135 	pdev = &sh->dev[sh->pd_idx];  in handle_stripe()
5136 	s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)  in handle_stripe()
5137 		|| (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);  in handle_stripe()
5138 	qdev = &sh->dev[sh->qd_idx];  in handle_stripe()
5139 	s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)  in handle_stripe()
5140 		|| (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)  in handle_stripe()
5152 		handle_stripe_clean_event(conf, sh, disks);  in handle_stripe()
5155 		r5c_handle_cached_data_endio(conf, sh, disks);  in handle_stripe()
5156 	log_stripe_write_finished(sh);  in handle_stripe()
5167 		handle_stripe_fill(sh, &s, disks);  in handle_stripe()
5174 	r5c_finish_stripe_write_out(conf, sh, &s);  in handle_stripe()
5185 	if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) {  in handle_stripe()
5188 				handle_stripe_dirtying(conf, sh, &s, disks);  in handle_stripe()
5194 				ret = r5c_try_caching_write(conf, sh, &s,  in handle_stripe()
5205 			    (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&  in handle_stripe()
5207 				ret = handle_stripe_dirtying(conf, sh, &s,  in handle_stripe()
5220 	if (sh->check_state ||  in handle_stripe()
5222 	     !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&  in handle_stripe()
5223 	     !test_bit(STRIPE_INSYNC, &sh->state))) {  in handle_stripe()
5225 			handle_parity_checks6(conf, sh, &s, disks);  in handle_stripe()
5227 			handle_parity_checks5(conf, sh, &s, disks);  in handle_stripe()
5231 	    && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)  in handle_stripe()
5232 	    && !test_bit(STRIPE_REPLACED, &sh->state)) {  in handle_stripe()
5235 			if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {  in handle_stripe()
5236 				WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));  in handle_stripe()
5237 				set_bit(R5_WantReplace, &sh->dev[i].flags);  in handle_stripe()
5238 				set_bit(R5_LOCKED, &sh->dev[i].flags);  in handle_stripe()
5242 			set_bit(STRIPE_INSYNC, &sh->state);  in handle_stripe()
5243 		set_bit(STRIPE_REPLACED, &sh->state);  in handle_stripe()
5246 	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&  in handle_stripe()
5247 	    test_bit(STRIPE_INSYNC, &sh->state)) {  in handle_stripe()
5249 		clear_bit(STRIPE_SYNCING, &sh->state);  in handle_stripe()
5250 		if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))  in handle_stripe()
5259 			struct r5dev *dev = &sh->dev[s.failed_num[i]];  in handle_stripe()
5276 	if (sh->reconstruct_state == reconstruct_state_result) {  in handle_stripe()
5278 			= raid5_get_active_stripe(conf, NULL, sh->sector,  in handle_stripe()
5282 			/* sh cannot be written until sh_src has been read.  in handle_stripe()
5283 			 * so arrange for sh to be delayed a little  in handle_stripe()
5285 			set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe()
5286 			set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
5296 		sh->reconstruct_state = reconstruct_state_idle;  in handle_stripe()
5297 		clear_bit(STRIPE_EXPANDING, &sh->state);  in handle_stripe()
5299 			set_bit(R5_Wantwrite, &sh->dev[i].flags);  in handle_stripe()
5300 			set_bit(R5_LOCKED, &sh->dev[i].flags);  in handle_stripe()
5305 	if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&  in handle_stripe()
5306 	    !sh->reconstruct_state) {  in handle_stripe()
5308 		sh->disks = conf->raid_disks;  in handle_stripe()
5309 		stripe_set_idx(sh->sector, conf, 0, sh);  in handle_stripe()
5310 		schedule_reconstruction(sh, &s, 1, 1);  in handle_stripe()
5311 	} else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {  in handle_stripe()
5312 		clear_bit(STRIPE_EXPAND_READY, &sh->state);  in handle_stripe()
5319 	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state))  in handle_stripe()
5320 		handle_stripe_expansion(conf, sh);  in handle_stripe()
5340 			struct r5dev *dev = &sh->dev[i];  in handle_stripe()
5344 				if (!rdev_set_badblocks(rdev, sh->sector,  in handle_stripe()
5351 				rdev_clear_badblocks(rdev, sh->sector,  in handle_stripe()
5360 				rdev_clear_badblocks(rdev, sh->sector,  in handle_stripe()
5367 		raid_run_ops(sh, s.ops_request);  in handle_stripe()
5369 	ops_run_io(sh, &s);  in handle_stripe()
5382 	clear_bit_unlock(STRIPE_ACTIVE, &sh->state);  in handle_stripe()
5391 			struct stripe_head *sh;  in raid5_activate_delayed()  local
5392 			sh = list_entry(l, struct stripe_head, lru);  in raid5_activate_delayed()
5394 			clear_bit(STRIPE_DELAYED, &sh->state);  in raid5_activate_delayed()
5395 			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in raid5_activate_delayed()
5397 			list_add_tail(&sh->lru, &conf->hold_list);  in raid5_activate_delayed()
5398 			raid5_wakeup_stripe_thread(sh);  in raid5_activate_delayed()
5411 		struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);  in activate_bit_delay()  local
5413 		list_del_init(&sh->lru);  in activate_bit_delay()
5414 		atomic_inc(&sh->count);  in activate_bit_delay()
5415 		hash = sh->hash_lock_index;  in activate_bit_delay()
5416 		__release_stripe(conf, sh, &temp_inactive_list[hash]);  in activate_bit_delay()
5619 	struct stripe_head *sh, *tmp;  in __get_priority_stripe()  local
5629 	sh = NULL;  in __get_priority_stripe()
5655 		sh = list_entry(handle_list->next, typeof(*sh), lru);  in __get_priority_stripe()
5659 		else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {  in __get_priority_stripe()
5679 				sh = tmp;  in __get_priority_stripe()
5684 		if (sh) {  in __get_priority_stripe()
5692 	if (!sh) {  in __get_priority_stripe()
5702 		sh->group = NULL;  in __get_priority_stripe()
5704 	list_del_init(&sh->lru);  in __get_priority_stripe()
5705 	BUG_ON(atomic_inc_return(&sh->count) != 1);  in __get_priority_stripe()
5706 	return sh;  in __get_priority_stripe()
5719 	struct stripe_head *sh;  in raid5_unplug()  local
5728 			sh = list_first_entry(&cb->list, struct stripe_head, lru);  in raid5_unplug()
5729 			list_del_init(&sh->lru);  in raid5_unplug()
5736 			clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);  in raid5_unplug()
5741 			hash = sh->hash_lock_index;  in raid5_unplug()
5742 			__release_stripe(conf, sh, &cb->temp_inactive_list[hash]);  in raid5_unplug()
5755 				struct stripe_head *sh)  in release_stripe_plug()  argument
5763 		raid5_release_stripe(sh);  in release_stripe_plug()
5776 	if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))  in release_stripe_plug()
5777 		list_add_tail(&sh->lru, &cb->list);  in release_stripe_plug()
5779 		raid5_release_stripe(sh);  in release_stripe_plug()
5786 	struct stripe_head *sh;  in make_discard_request()  local
5816 		sh = raid5_get_active_stripe(conf, NULL, logical_sector, 0);  in make_discard_request()
5819 		set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);  in make_discard_request()
5820 		if (test_bit(STRIPE_SYNCING, &sh->state)) {  in make_discard_request()
5821 			raid5_release_stripe(sh);  in make_discard_request()
5825 		clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);  in make_discard_request()
5826 		spin_lock_irq(&sh->stripe_lock);  in make_discard_request()
5828 			if (d == sh->pd_idx || d == sh->qd_idx)  in make_discard_request()
5830 			if (sh->dev[d].towrite || sh->dev[d].toread) {  in make_discard_request()
5831 				set_bit(R5_Overlap, &sh->dev[d].flags);  in make_discard_request()
5832 				spin_unlock_irq(&sh->stripe_lock);  in make_discard_request()
5833 				raid5_release_stripe(sh);  in make_discard_request()
5838 		set_bit(STRIPE_DISCARD, &sh->state);  in make_discard_request()
5840 		sh->overwrite_disks = 0;  in make_discard_request()
5842 			if (d == sh->pd_idx || d == sh->qd_idx)  in make_discard_request()
5844 			sh->dev[d].towrite = bi;  in make_discard_request()
5845 			set_bit(R5_OVERWRITE, &sh->dev[d].flags);  in make_discard_request()
5848 			sh->overwrite_disks++;  in make_discard_request()
5850 		spin_unlock_irq(&sh->stripe_lock);  in make_discard_request()
5856 						     sh->sector,  in make_discard_request()
5859 			sh->bm_seq = conf->seq_flush + 1;  in make_discard_request()
5860 			set_bit(STRIPE_BIT_DELAY, &sh->state);  in make_discard_request()
5863 		set_bit(STRIPE_HANDLE, &sh->state);  in make_discard_request()
5864 		clear_bit(STRIPE_DELAYED, &sh->state);  in make_discard_request()
5865 		if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in make_discard_request()
5867 		release_stripe_plug(mddev, sh);  in make_discard_request()
5888 				    struct stripe_head *sh)  in stripe_ahead_of_reshape()  argument
5894 	for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {  in stripe_ahead_of_reshape()
5895 		if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)  in stripe_ahead_of_reshape()
5898 		min_sector = min(min_sector, sh->dev[dd_idx].sector);  in stripe_ahead_of_reshape()
5899 		max_sector = max(max_sector, sh->dev[dd_idx].sector);  in stripe_ahead_of_reshape()
5915 		struct stripe_request_ctx *ctx, struct stripe_head *sh,  in add_all_stripe_bios()  argument
5921 	spin_lock_irq(&sh->stripe_lock);  in add_all_stripe_bios()
5923 	for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {  in add_all_stripe_bios()
5924 		struct r5dev *dev = &sh->dev[dd_idx];  in add_all_stripe_bios()
5926 		if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)  in add_all_stripe_bios()
5933 		if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) {  in add_all_stripe_bios()
5943 	for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {  in add_all_stripe_bios()
5944 		struct r5dev *dev = &sh->dev[dd_idx];  in add_all_stripe_bios()
5946 		if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)  in add_all_stripe_bios()
5953 		__add_stripe_bio(sh, bi, dd_idx, forwrite, previous);  in add_all_stripe_bios()
5959 	spin_unlock_irq(&sh->stripe_lock);  in add_all_stripe_bios()
5982 	struct stripe_head *sh;  in make_stripe_request()  local
6023 	sh = raid5_get_active_stripe(conf, ctx, new_sector, flags);  in make_stripe_request()
6024 	if (unlikely(!sh)) {  in make_stripe_request()
6031 	    stripe_ahead_of_reshape(mddev, conf, sh)) {  in make_stripe_request()
6036 		 * 'sh', we know that if that happens,  in make_stripe_request()
6050 	if (test_bit(STRIPE_EXPANDING, &sh->state) ||  in make_stripe_request()
6051 	    !add_all_stripe_bios(conf, ctx, sh, bi, rw, previous)) {  in make_stripe_request()
6061 	if (stripe_can_batch(sh)) {  in make_stripe_request()
6062 		stripe_add_to_batch_list(conf, sh, ctx->batch_last);  in make_stripe_request()
6065 		atomic_inc(&sh->count);  in make_stripe_request()
6066 		ctx->batch_last = sh;  in make_stripe_request()
6070 		set_bit(STRIPE_R5C_PREFLUSH, &sh->state);  in make_stripe_request()
6075 	set_bit(STRIPE_HANDLE, &sh->state);  in make_stripe_request()
6076 	clear_bit(STRIPE_DELAYED, &sh->state);  in make_stripe_request()
6077 	if ((!sh->batch_head || sh == sh->batch_head) &&  in make_stripe_request()
6079 	    !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in make_stripe_request()
6082 	release_stripe_plug(mddev, sh);  in make_stripe_request()
6086 	raid5_release_stripe(sh);  in make_stripe_request()
6109 	struct stripe_head sh;  in raid5_bio_lowest_chunk_sector()  local
6115 	sector = raid5_compute_sector(conf, r_sector, 0, &dd_idx, &sh);  in raid5_bio_lowest_chunk_sector()
6124 	while (dd_idx == sh.pd_idx || dd_idx == sh.qd_idx)  in raid5_bio_lowest_chunk_sector()
6273 	struct stripe_head *sh;  in reshape_request()  local
6424 		sh = raid5_get_active_stripe(conf, NULL, stripe_addr+i,  in reshape_request()
6426 		set_bit(STRIPE_EXPANDING, &sh->state);  in reshape_request()
6431 		for (j=sh->disks; j--;) {  in reshape_request()
6433 			if (j == sh->pd_idx)  in reshape_request()
6436 			    j == sh->qd_idx)  in reshape_request()
6438 			s = raid5_compute_blocknr(sh, j, 0);  in reshape_request()
6443 			memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));  in reshape_request()
6444 			set_bit(R5_Expanded, &sh->dev[j].flags);  in reshape_request()
6445 			set_bit(R5_UPTODATE, &sh->dev[j].flags);  in reshape_request()
6448 			set_bit(STRIPE_EXPAND_READY, &sh->state);  in reshape_request()
6449 			set_bit(STRIPE_HANDLE, &sh->state);  in reshape_request()
6451 		list_add(&sh->lru, &stripes);  in reshape_request()
6474 		sh = raid5_get_active_stripe(conf, NULL, first_sector,  in reshape_request()
6476 		set_bit(STRIPE_EXPAND_SOURCE, &sh->state);  in reshape_request()
6477 		set_bit(STRIPE_HANDLE, &sh->state);  in reshape_request()
6478 		raid5_release_stripe(sh);  in reshape_request()
6485 		sh = list_entry(stripes.next, struct stripe_head, lru);  in reshape_request()
6486 		list_del_init(&sh->lru);  in reshape_request()
6487 		raid5_release_stripe(sh);  in reshape_request()
6536 	struct stripe_head *sh;  in raid5_sync_request()  local
6595 	sh = raid5_get_active_stripe(conf, NULL, sector_nr,  in raid5_sync_request()
6597 	if (sh == NULL) {  in raid5_sync_request()
6598 		sh = raid5_get_active_stripe(conf, NULL, sector_nr, 0);  in raid5_sync_request()
6619 	set_bit(STRIPE_SYNC_REQUESTED, &sh->state);  in raid5_sync_request()
6620 	set_bit(STRIPE_HANDLE, &sh->state);  in raid5_sync_request()
6622 	raid5_release_stripe(sh);  in raid5_sync_request()
6640 	struct stripe_head *sh;  in retry_aligned_read()  local
6661 		sh = raid5_get_active_stripe(conf, NULL, sector,  in retry_aligned_read()
6663 		if (!sh) {  in retry_aligned_read()
6670 		if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {  in retry_aligned_read()
6671 			raid5_release_stripe(sh);  in retry_aligned_read()
6677 		set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);  in retry_aligned_read()
6678 		handle_stripe(sh);  in retry_aligned_read()
6679 		raid5_release_stripe(sh);  in retry_aligned_read()
6695 	struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;  in handle_active_stripes()  local
6700 			(sh = __get_priority_stripe(conf, group)) != NULL)  in handle_active_stripes()
6701 		batch[batch_size++] = sh;  in handle_active_stripes()