Lines Matching +full:scrubber +full:- +full:done
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2012 Fusion-io All rights reserved.
17 #include "disk-io.h"
20 #include "async-thread.h"
170 /* allocated with real_stripes-many pointers for finish_*() calls */
173 /* allocated with stripe_npages-many bits for finish_*() calls */
193 btrfs_init_work(&rbio->work, work_func, NULL, NULL); in start_async_work()
194 btrfs_queue_work(rbio->fs_info->rmw_workers, &rbio->work); in start_async_work()
210 if (info->stripe_hash_table) in btrfs_alloc_stripe_hash_table()
222 return -ENOMEM; in btrfs_alloc_stripe_hash_table()
224 spin_lock_init(&table->cache_lock); in btrfs_alloc_stripe_hash_table()
225 INIT_LIST_HEAD(&table->stripe_cache); in btrfs_alloc_stripe_hash_table()
227 h = table->table; in btrfs_alloc_stripe_hash_table()
231 INIT_LIST_HEAD(&cur->hash_list); in btrfs_alloc_stripe_hash_table()
232 spin_lock_init(&cur->lock); in btrfs_alloc_stripe_hash_table()
235 x = cmpxchg(&info->stripe_hash_table, NULL, table); in btrfs_alloc_stripe_hash_table()
247 * once the caching is done, we set the cache ready
261 for (i = 0; i < rbio->nr_pages; i++) { in cache_rbio_pages()
262 if (!rbio->bio_pages[i]) in cache_rbio_pages()
265 s = kmap(rbio->bio_pages[i]); in cache_rbio_pages()
266 d = kmap(rbio->stripe_pages[i]); in cache_rbio_pages()
270 kunmap(rbio->bio_pages[i]); in cache_rbio_pages()
271 kunmap(rbio->stripe_pages[i]); in cache_rbio_pages()
272 SetPageUptodate(rbio->stripe_pages[i]); in cache_rbio_pages()
274 set_bit(RBIO_CACHE_READY_BIT, &rbio->flags); in cache_rbio_pages()
282 u64 num = rbio->bbio->raid_map[0]; in rbio_bucket()
305 if (!test_bit(RBIO_CACHE_READY_BIT, &src->flags)) in steal_rbio()
308 for (i = 0; i < dest->nr_pages; i++) { in steal_rbio()
309 s = src->stripe_pages[i]; in steal_rbio()
314 d = dest->stripe_pages[i]; in steal_rbio()
318 dest->stripe_pages[i] = s; in steal_rbio()
319 src->stripe_pages[i] = NULL; in steal_rbio()
328 * must be called with dest->rbio_list_lock held
333 bio_list_merge(&dest->bio_list, &victim->bio_list); in merge_rbio()
334 dest->bio_list_bytes += victim->bio_list_bytes; in merge_rbio()
336 bitmap_or(dest->dbitmap, victim->dbitmap, dest->dbitmap, in merge_rbio()
337 dest->stripe_npages); in merge_rbio()
338 dest->generic_bio_cnt += victim->generic_bio_cnt; in merge_rbio()
339 bio_list_init(&victim->bio_list); in merge_rbio()
356 if (!test_bit(RBIO_CACHE_BIT, &rbio->flags)) in __remove_rbio_from_cache()
359 table = rbio->fs_info->stripe_hash_table; in __remove_rbio_from_cache()
360 h = table->table + bucket; in __remove_rbio_from_cache()
365 spin_lock(&h->lock); in __remove_rbio_from_cache()
371 spin_lock(&rbio->bio_list_lock); in __remove_rbio_from_cache()
373 if (test_and_clear_bit(RBIO_CACHE_BIT, &rbio->flags)) { in __remove_rbio_from_cache()
374 list_del_init(&rbio->stripe_cache); in __remove_rbio_from_cache()
375 table->cache_size -= 1; in __remove_rbio_from_cache()
387 if (bio_list_empty(&rbio->bio_list)) { in __remove_rbio_from_cache()
388 if (!list_empty(&rbio->hash_list)) { in __remove_rbio_from_cache()
389 list_del_init(&rbio->hash_list); in __remove_rbio_from_cache()
390 refcount_dec(&rbio->refs); in __remove_rbio_from_cache()
391 BUG_ON(!list_empty(&rbio->plug_list)); in __remove_rbio_from_cache()
396 spin_unlock(&rbio->bio_list_lock); in __remove_rbio_from_cache()
397 spin_unlock(&h->lock); in __remove_rbio_from_cache()
411 if (!test_bit(RBIO_CACHE_BIT, &rbio->flags)) in remove_rbio_from_cache()
414 table = rbio->fs_info->stripe_hash_table; in remove_rbio_from_cache()
416 spin_lock_irqsave(&table->cache_lock, flags); in remove_rbio_from_cache()
418 spin_unlock_irqrestore(&table->cache_lock, flags); in remove_rbio_from_cache()
430 table = info->stripe_hash_table; in btrfs_clear_rbio_cache()
432 spin_lock_irqsave(&table->cache_lock, flags); in btrfs_clear_rbio_cache()
433 while (!list_empty(&table->stripe_cache)) { in btrfs_clear_rbio_cache()
434 rbio = list_entry(table->stripe_cache.next, in btrfs_clear_rbio_cache()
439 spin_unlock_irqrestore(&table->cache_lock, flags); in btrfs_clear_rbio_cache()
448 if (!info->stripe_hash_table) in btrfs_free_stripe_hash_table()
451 kvfree(info->stripe_hash_table); in btrfs_free_stripe_hash_table()
452 info->stripe_hash_table = NULL; in btrfs_free_stripe_hash_table()
471 if (!test_bit(RBIO_CACHE_READY_BIT, &rbio->flags)) in cache_rbio()
474 table = rbio->fs_info->stripe_hash_table; in cache_rbio()
476 spin_lock_irqsave(&table->cache_lock, flags); in cache_rbio()
477 spin_lock(&rbio->bio_list_lock); in cache_rbio()
480 if (!test_and_set_bit(RBIO_CACHE_BIT, &rbio->flags)) in cache_rbio()
481 refcount_inc(&rbio->refs); in cache_rbio()
483 if (!list_empty(&rbio->stripe_cache)){ in cache_rbio()
484 list_move(&rbio->stripe_cache, &table->stripe_cache); in cache_rbio()
486 list_add(&rbio->stripe_cache, &table->stripe_cache); in cache_rbio()
487 table->cache_size += 1; in cache_rbio()
490 spin_unlock(&rbio->bio_list_lock); in cache_rbio()
492 if (table->cache_size > RBIO_CACHE_SIZE) { in cache_rbio()
495 found = list_entry(table->stripe_cache.prev, in cache_rbio()
503 spin_unlock_irqrestore(&table->cache_lock, flags); in cache_rbio()
521 src_cnt -= xor_src_cnt; in run_xor()
533 unsigned long size = rbio->bio_list_bytes; in rbio_is_full()
536 spin_lock_irqsave(&rbio->bio_list_lock, flags); in rbio_is_full()
537 if (size != rbio->nr_data * rbio->stripe_len) in rbio_is_full()
539 BUG_ON(size > rbio->nr_data * rbio->stripe_len); in rbio_is_full()
540 spin_unlock_irqrestore(&rbio->bio_list_lock, flags); in rbio_is_full()
558 if (test_bit(RBIO_RMW_LOCKED_BIT, &last->flags) || in rbio_can_merge()
559 test_bit(RBIO_RMW_LOCKED_BIT, &cur->flags)) in rbio_can_merge()
569 if (test_bit(RBIO_CACHE_BIT, &last->flags) || in rbio_can_merge()
570 test_bit(RBIO_CACHE_BIT, &cur->flags)) in rbio_can_merge()
573 if (last->bbio->raid_map[0] != in rbio_can_merge()
574 cur->bbio->raid_map[0]) in rbio_can_merge()
578 if (last->operation != cur->operation) in rbio_can_merge()
588 if (last->operation == BTRFS_RBIO_PARITY_SCRUB) in rbio_can_merge()
591 if (last->operation == BTRFS_RBIO_REBUILD_MISSING) in rbio_can_merge()
594 if (last->operation == BTRFS_RBIO_READ_REBUILD) { in rbio_can_merge()
595 int fa = last->faila; in rbio_can_merge()
596 int fb = last->failb; in rbio_can_merge()
597 int cur_fa = cur->faila; in rbio_can_merge()
598 int cur_fb = cur->failb; in rbio_can_merge()
600 if (last->faila >= last->failb) { in rbio_can_merge()
601 fa = last->failb; in rbio_can_merge()
602 fb = last->faila; in rbio_can_merge()
605 if (cur->faila >= cur->failb) { in rbio_can_merge()
606 cur_fa = cur->failb; in rbio_can_merge()
607 cur_fb = cur->faila; in rbio_can_merge()
619 return stripe * rbio->stripe_npages + index; in rbio_stripe_page_index()
629 return rbio->stripe_pages[rbio_stripe_page_index(rbio, stripe, index)]; in rbio_stripe_page()
637 return rbio_stripe_page(rbio, rbio->nr_data, index); in rbio_pstripe_page()
646 if (rbio->nr_data + 1 == rbio->real_stripes) in rbio_qstripe_page()
648 return rbio_stripe_page(rbio, rbio->nr_data + 1, index); in rbio_qstripe_page()
683 h = rbio->fs_info->stripe_hash_table->table + rbio_bucket(rbio); in lock_stripe_add()
685 spin_lock_irqsave(&h->lock, flags); in lock_stripe_add()
686 list_for_each_entry(cur, &h->hash_list, hash_list) { in lock_stripe_add()
687 if (cur->bbio->raid_map[0] != rbio->bbio->raid_map[0]) in lock_stripe_add()
690 spin_lock(&cur->bio_list_lock); in lock_stripe_add()
693 if (bio_list_empty(&cur->bio_list) && in lock_stripe_add()
694 list_empty(&cur->plug_list) && in lock_stripe_add()
695 test_bit(RBIO_CACHE_BIT, &cur->flags) && in lock_stripe_add()
696 !test_bit(RBIO_RMW_LOCKED_BIT, &cur->flags)) { in lock_stripe_add()
697 list_del_init(&cur->hash_list); in lock_stripe_add()
698 refcount_dec(&cur->refs); in lock_stripe_add()
702 spin_unlock(&cur->bio_list_lock); in lock_stripe_add()
710 spin_unlock(&cur->bio_list_lock); in lock_stripe_add()
722 list_for_each_entry(pending, &cur->plug_list, plug_list) { in lock_stripe_add()
725 spin_unlock(&cur->bio_list_lock); in lock_stripe_add()
736 list_add_tail(&rbio->plug_list, &cur->plug_list); in lock_stripe_add()
737 spin_unlock(&cur->bio_list_lock); in lock_stripe_add()
742 refcount_inc(&rbio->refs); in lock_stripe_add()
743 list_add(&rbio->hash_list, &h->hash_list); in lock_stripe_add()
745 spin_unlock_irqrestore(&h->lock, flags); in lock_stripe_add()
765 h = rbio->fs_info->stripe_hash_table->table + bucket; in unlock_stripe()
767 if (list_empty(&rbio->plug_list)) in unlock_stripe()
770 spin_lock_irqsave(&h->lock, flags); in unlock_stripe()
771 spin_lock(&rbio->bio_list_lock); in unlock_stripe()
773 if (!list_empty(&rbio->hash_list)) { in unlock_stripe()
779 if (list_empty(&rbio->plug_list) && in unlock_stripe()
780 test_bit(RBIO_CACHE_BIT, &rbio->flags)) { in unlock_stripe()
782 clear_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags); in unlock_stripe()
783 BUG_ON(!bio_list_empty(&rbio->bio_list)); in unlock_stripe()
784 goto done; in unlock_stripe()
787 list_del_init(&rbio->hash_list); in unlock_stripe()
788 refcount_dec(&rbio->refs); in unlock_stripe()
795 if (!list_empty(&rbio->plug_list)) { in unlock_stripe()
797 struct list_head *head = rbio->plug_list.next; in unlock_stripe()
802 list_del_init(&rbio->plug_list); in unlock_stripe()
804 list_add(&next->hash_list, &h->hash_list); in unlock_stripe()
805 refcount_inc(&next->refs); in unlock_stripe()
806 spin_unlock(&rbio->bio_list_lock); in unlock_stripe()
807 spin_unlock_irqrestore(&h->lock, flags); in unlock_stripe()
809 if (next->operation == BTRFS_RBIO_READ_REBUILD) in unlock_stripe()
811 else if (next->operation == BTRFS_RBIO_REBUILD_MISSING) { in unlock_stripe()
814 } else if (next->operation == BTRFS_RBIO_WRITE) { in unlock_stripe()
817 } else if (next->operation == BTRFS_RBIO_PARITY_SCRUB) { in unlock_stripe()
825 done: in unlock_stripe()
826 spin_unlock(&rbio->bio_list_lock); in unlock_stripe()
827 spin_unlock_irqrestore(&h->lock, flags); in unlock_stripe()
838 if (!refcount_dec_and_test(&rbio->refs)) in __free_raid_bio()
841 WARN_ON(!list_empty(&rbio->stripe_cache)); in __free_raid_bio()
842 WARN_ON(!list_empty(&rbio->hash_list)); in __free_raid_bio()
843 WARN_ON(!bio_list_empty(&rbio->bio_list)); in __free_raid_bio()
845 for (i = 0; i < rbio->nr_pages; i++) { in __free_raid_bio()
846 if (rbio->stripe_pages[i]) { in __free_raid_bio()
847 __free_page(rbio->stripe_pages[i]); in __free_raid_bio()
848 rbio->stripe_pages[i] = NULL; in __free_raid_bio()
852 btrfs_put_bbio(rbio->bbio); in __free_raid_bio()
861 next = cur->bi_next; in rbio_endio_bio_list()
862 cur->bi_next = NULL; in rbio_endio_bio_list()
863 cur->bi_status = err; in rbio_endio_bio_list()
875 struct bio *cur = bio_list_get(&rbio->bio_list); in rbio_orig_end_io()
878 if (rbio->generic_bio_cnt) in rbio_orig_end_io()
879 btrfs_bio_counter_sub(rbio->fs_info, rbio->generic_bio_cnt); in rbio_orig_end_io()
885 bitmap_clear(rbio->dbitmap, 0, rbio->stripe_npages); in rbio_orig_end_io()
888 * At this moment, rbio->bio_list is empty, however since rbio does not in rbio_orig_end_io()
890 * hash list, rbio may be merged with others so that rbio->bio_list in rbio_orig_end_io()
891 * becomes non-empty. in rbio_orig_end_io()
892 * Once unlock_stripe() is done, rbio->bio_list will not be updated any in rbio_orig_end_io()
896 extra = bio_list_get(&rbio->bio_list); in rbio_orig_end_io()
910 struct btrfs_raid_bio *rbio = bio->bi_private; in raid_write_end_io()
911 blk_status_t err = bio->bi_status; in raid_write_end_io()
919 if (!atomic_dec_and_test(&rbio->stripes_pending)) in raid_write_end_io()
925 max_errors = (rbio->operation == BTRFS_RBIO_PARITY_SCRUB) ? in raid_write_end_io()
926 0 : rbio->bbio->max_errors; in raid_write_end_io()
927 if (atomic_read(&rbio->error) > max_errors) in raid_write_end_io()
955 chunk_page = index * (rbio->stripe_len >> PAGE_SHIFT) + pagenr; in page_in_rbio()
957 spin_lock_irq(&rbio->bio_list_lock); in page_in_rbio()
958 p = rbio->bio_pages[chunk_page]; in page_in_rbio()
959 spin_unlock_irq(&rbio->bio_list_lock); in page_in_rbio()
964 return rbio->stripe_pages[chunk_page]; in page_in_rbio()
978 * this does not allocate any pages for rbio->pages.
986 int real_stripes = bbio->num_stripes - bbio->num_tgtdevs; in alloc_rbio()
992 sizeof(*rbio->stripe_pages) * num_pages + in alloc_rbio()
993 sizeof(*rbio->bio_pages) * num_pages + in alloc_rbio()
994 sizeof(*rbio->finish_pointers) * real_stripes + in alloc_rbio()
995 sizeof(*rbio->dbitmap) * BITS_TO_LONGS(stripe_npages) + in alloc_rbio()
996 sizeof(*rbio->finish_pbitmap) * in alloc_rbio()
1000 return ERR_PTR(-ENOMEM); in alloc_rbio()
1002 bio_list_init(&rbio->bio_list); in alloc_rbio()
1003 INIT_LIST_HEAD(&rbio->plug_list); in alloc_rbio()
1004 spin_lock_init(&rbio->bio_list_lock); in alloc_rbio()
1005 INIT_LIST_HEAD(&rbio->stripe_cache); in alloc_rbio()
1006 INIT_LIST_HEAD(&rbio->hash_list); in alloc_rbio()
1007 rbio->bbio = bbio; in alloc_rbio()
1008 rbio->fs_info = fs_info; in alloc_rbio()
1009 rbio->stripe_len = stripe_len; in alloc_rbio()
1010 rbio->nr_pages = num_pages; in alloc_rbio()
1011 rbio->real_stripes = real_stripes; in alloc_rbio()
1012 rbio->stripe_npages = stripe_npages; in alloc_rbio()
1013 rbio->faila = -1; in alloc_rbio()
1014 rbio->failb = -1; in alloc_rbio()
1015 refcount_set(&rbio->refs, 1); in alloc_rbio()
1016 atomic_set(&rbio->error, 0); in alloc_rbio()
1017 atomic_set(&rbio->stripes_pending, 0); in alloc_rbio()
1028 CONSUME_ALLOC(rbio->stripe_pages, num_pages); in alloc_rbio()
1029 CONSUME_ALLOC(rbio->bio_pages, num_pages); in alloc_rbio()
1030 CONSUME_ALLOC(rbio->finish_pointers, real_stripes); in alloc_rbio()
1031 CONSUME_ALLOC(rbio->dbitmap, BITS_TO_LONGS(stripe_npages)); in alloc_rbio()
1032 CONSUME_ALLOC(rbio->finish_pbitmap, BITS_TO_LONGS(stripe_npages)); in alloc_rbio()
1035 if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5) in alloc_rbio()
1036 nr_data = real_stripes - 1; in alloc_rbio()
1037 else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6) in alloc_rbio()
1038 nr_data = real_stripes - 2; in alloc_rbio()
1042 rbio->nr_data = nr_data; in alloc_rbio()
1052 for (i = 0; i < rbio->nr_pages; i++) { in alloc_rbio_pages()
1053 if (rbio->stripe_pages[i]) in alloc_rbio_pages()
1057 return -ENOMEM; in alloc_rbio_pages()
1058 rbio->stripe_pages[i] = page; in alloc_rbio_pages()
1069 i = rbio_stripe_page_index(rbio, rbio->nr_data, 0); in alloc_rbio_parity_pages()
1071 for (; i < rbio->nr_pages; i++) { in alloc_rbio_parity_pages()
1072 if (rbio->stripe_pages[i]) in alloc_rbio_parity_pages()
1076 return -ENOMEM; in alloc_rbio_parity_pages()
1077 rbio->stripe_pages[i] = page; in alloc_rbio_parity_pages()
1094 struct bio *last = bio_list->tail; in rbio_add_io_page()
1100 stripe = &rbio->bbio->stripes[stripe_nr]; in rbio_add_io_page()
1101 disk_start = stripe->physical + (page_index << PAGE_SHIFT); in rbio_add_io_page()
1104 if (!stripe->dev->bdev) in rbio_add_io_page()
1109 u64 last_end = (u64)last->bi_iter.bi_sector << 9; in rbio_add_io_page()
1110 last_end += last->bi_iter.bi_size; in rbio_add_io_page()
1116 if (last_end == disk_start && !last->bi_status && in rbio_add_io_page()
1117 last->bi_disk == stripe->dev->bdev->bd_disk && in rbio_add_io_page()
1118 last->bi_partno == stripe->dev->bdev->bd_partno) { in rbio_add_io_page()
1127 btrfs_io_bio(bio)->device = stripe->dev; in rbio_add_io_page()
1128 bio->bi_iter.bi_size = 0; in rbio_add_io_page()
1129 bio_set_dev(bio, stripe->dev->bdev); in rbio_add_io_page()
1130 bio->bi_iter.bi_sector = disk_start >> 9; in rbio_add_io_page()
1146 if (rbio->faila >= 0 || rbio->failb >= 0) { in validate_rbio_for_rmw()
1147 BUG_ON(rbio->faila == rbio->real_stripes - 1); in validate_rbio_for_rmw()
1169 spin_lock_irq(&rbio->bio_list_lock); in index_rbio_pages()
1170 bio_list_for_each(bio, &rbio->bio_list) { in index_rbio_pages()
1175 start = (u64)bio->bi_iter.bi_sector << 9; in index_rbio_pages()
1176 stripe_offset = start - rbio->bbio->raid_map[0]; in index_rbio_pages()
1180 bio->bi_iter = btrfs_io_bio(bio)->iter; in index_rbio_pages()
1183 rbio->bio_pages[page_index + i] = bvec.bv_page; in index_rbio_pages()
1187 spin_unlock_irq(&rbio->bio_list_lock); in index_rbio_pages()
1200 struct btrfs_bio *bbio = rbio->bbio; in finish_rmw()
1201 void **pointers = rbio->finish_pointers; in finish_rmw()
1202 int nr_data = rbio->nr_data; in finish_rmw()
1212 if (rbio->real_stripes - rbio->nr_data == 1) in finish_rmw()
1214 else if (rbio->real_stripes - rbio->nr_data == 2) in finish_rmw()
1220 ASSERT(bitmap_weight(rbio->dbitmap, rbio->stripe_npages)); in finish_rmw()
1230 spin_lock_irq(&rbio->bio_list_lock); in finish_rmw()
1231 set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags); in finish_rmw()
1232 spin_unlock_irq(&rbio->bio_list_lock); in finish_rmw()
1234 atomic_set(&rbio->error, 0); in finish_rmw()
1249 clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags); in finish_rmw()
1251 for (pagenr = 0; pagenr < rbio->stripe_npages; pagenr++) { in finish_rmw()
1274 raid6_call.gen_syndrome(rbio->real_stripes, PAGE_SIZE, in finish_rmw()
1279 run_xor(pointers + 1, nr_data - 1, PAGE_SIZE); in finish_rmw()
1283 for (stripe = 0; stripe < rbio->real_stripes; stripe++) in finish_rmw()
1292 for (stripe = 0; stripe < rbio->real_stripes; stripe++) { in finish_rmw()
1293 for (pagenr = 0; pagenr < rbio->stripe_npages; pagenr++) { in finish_rmw()
1297 if (!test_bit(pagenr, rbio->dbitmap)) in finish_rmw()
1300 if (stripe < rbio->nr_data) { in finish_rmw()
1309 page, stripe, pagenr, rbio->stripe_len); in finish_rmw()
1315 if (likely(!bbio->num_tgtdevs)) in finish_rmw()
1318 for (stripe = 0; stripe < rbio->real_stripes; stripe++) { in finish_rmw()
1319 if (!bbio->tgtdev_map[stripe]) in finish_rmw()
1322 for (pagenr = 0; pagenr < rbio->stripe_npages; pagenr++) { in finish_rmw()
1326 if (!test_bit(pagenr, rbio->dbitmap)) in finish_rmw()
1329 if (stripe < rbio->nr_data) { in finish_rmw()
1338 rbio->bbio->tgtdev_map[stripe], in finish_rmw()
1339 pagenr, rbio->stripe_len); in finish_rmw()
1346 atomic_set(&rbio->stripes_pending, bio_list_size(&bio_list)); in finish_rmw()
1347 BUG_ON(atomic_read(&rbio->stripes_pending) == 0); in finish_rmw()
1350 bio->bi_private = rbio; in finish_rmw()
1351 bio->bi_end_io = raid_write_end_io; in finish_rmw()
1352 bio->bi_opf = REQ_OP_WRITE; in finish_rmw()
1373 u64 physical = bio->bi_iter.bi_sector; in find_bio_stripe()
1379 for (i = 0; i < rbio->bbio->num_stripes; i++) { in find_bio_stripe()
1380 stripe = &rbio->bbio->stripes[i]; in find_bio_stripe()
1381 if (in_range(physical, stripe->physical, rbio->stripe_len) && in find_bio_stripe()
1382 stripe->dev->bdev && in find_bio_stripe()
1383 bio->bi_disk == stripe->dev->bdev->bd_disk && in find_bio_stripe()
1384 bio->bi_partno == stripe->dev->bdev->bd_partno) { in find_bio_stripe()
1388 return -1; in find_bio_stripe()
1399 u64 logical = (u64)bio->bi_iter.bi_sector << 9; in find_logical_bio_stripe()
1402 for (i = 0; i < rbio->nr_data; i++) { in find_logical_bio_stripe()
1403 u64 stripe_start = rbio->bbio->raid_map[i]; in find_logical_bio_stripe()
1405 if (in_range(logical, stripe_start, rbio->stripe_len)) in find_logical_bio_stripe()
1408 return -1; in find_logical_bio_stripe()
1412 * returns -EIO if we had too many failures
1419 spin_lock_irqsave(&rbio->bio_list_lock, flags); in fail_rbio_index()
1422 if (rbio->faila == failed || rbio->failb == failed) in fail_rbio_index()
1425 if (rbio->faila == -1) { in fail_rbio_index()
1427 rbio->faila = failed; in fail_rbio_index()
1428 atomic_inc(&rbio->error); in fail_rbio_index()
1429 } else if (rbio->failb == -1) { in fail_rbio_index()
1431 rbio->failb = failed; in fail_rbio_index()
1432 atomic_inc(&rbio->error); in fail_rbio_index()
1434 ret = -EIO; in fail_rbio_index()
1437 spin_unlock_irqrestore(&rbio->bio_list_lock, flags); in fail_rbio_index()
1452 return -EIO; in fail_bio_stripe()
1469 SetPageUptodate(bvec->bv_page); in set_bio_pages_uptodate()
1482 struct btrfs_raid_bio *rbio = bio->bi_private; in raid_rmw_end_io()
1484 if (bio->bi_status) in raid_rmw_end_io()
1491 if (!atomic_dec_and_test(&rbio->stripes_pending)) in raid_rmw_end_io()
1494 if (atomic_read(&rbio->error) > rbio->bbio->max_errors) in raid_rmw_end_io()
1512 * unlock after all the writes are done
1531 atomic_set(&rbio->error, 0); in raid56_rmw_stripe()
1536 for (stripe = 0; stripe < rbio->nr_data; stripe++) { in raid56_rmw_stripe()
1537 for (pagenr = 0; pagenr < rbio->stripe_npages; pagenr++) { in raid56_rmw_stripe()
1558 stripe, pagenr, rbio->stripe_len); in raid56_rmw_stripe()
1579 atomic_set(&rbio->stripes_pending, bios_to_read); in raid56_rmw_stripe()
1581 bio->bi_private = rbio; in raid56_rmw_stripe()
1582 bio->bi_end_io = raid_rmw_end_io; in raid56_rmw_stripe()
1583 bio->bi_opf = REQ_OP_READ; in raid56_rmw_stripe()
1585 btrfs_bio_wq_end_io(rbio->fs_info, bio, BTRFS_WQ_ENDIO_RAID56); in raid56_rmw_stripe()
1589 /* the actual write will happen once the reads are done */ in raid56_rmw_stripe()
1598 return -EIO; in raid56_rmw_stripe()
1678 u64 a_sector = ra->bio_list.head->bi_iter.bi_sector; in plug_cmp()
1679 u64 b_sector = rb->bio_list.head->bi_iter.bi_sector; in plug_cmp()
1682 return -1; in plug_cmp()
1698 list_sort(NULL, &plug->rbio_list, plug_cmp); in run_plug()
1699 while (!list_empty(&plug->rbio_list)) { in run_plug()
1700 cur = list_entry(plug->rbio_list.next, in run_plug()
1702 list_del_init(&cur->plug_list); in run_plug()
1746 btrfs_init_work(&plug->work, unplug_work, NULL, NULL); in btrfs_raid_unplug()
1747 btrfs_queue_work(plug->info->rmw_workers, in btrfs_raid_unplug()
1748 &plug->work); in btrfs_raid_unplug()
1754 /* Add the original bio into rbio->bio_list, and update rbio::dbitmap. */
1757 const struct btrfs_fs_info *fs_info = rbio->fs_info; in rbio_add_bio()
1758 const u64 orig_logical = orig_bio->bi_iter.bi_sector << SECTOR_SHIFT; in rbio_add_bio()
1759 const u64 full_stripe_start = rbio->bbio->raid_map[0]; in rbio_add_bio()
1760 const u32 orig_len = orig_bio->bi_iter.bi_size; in rbio_add_bio()
1761 const u32 sectorsize = fs_info->sectorsize; in rbio_add_bio()
1766 rbio->nr_data * rbio->stripe_len); in rbio_add_bio()
1768 bio_list_add(&rbio->bio_list, orig_bio); in rbio_add_bio()
1769 rbio->bio_list_bytes += orig_bio->bi_iter.bi_size; in rbio_add_bio()
1774 int bit = ((u32)(cur_logical - full_stripe_start) >> in rbio_add_bio()
1775 PAGE_SHIFT) % rbio->stripe_npages; in rbio_add_bio()
1777 set_bit(bit, rbio->dbitmap); in rbio_add_bio()
1797 rbio->operation = BTRFS_RBIO_WRITE; in raid56_parity_write()
1801 rbio->generic_bio_cnt = 1; in raid56_parity_write()
1817 if (!plug->info) { in raid56_parity_write()
1818 plug->info = fs_info; in raid56_parity_write()
1819 INIT_LIST_HEAD(&plug->rbio_list); in raid56_parity_write()
1821 list_add_tail(&rbio->plug_list, &plug->rbio_list); in raid56_parity_write()
1840 int faila = -1, failb = -1; in __raid_recover_end_io()
1845 pointers = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS); in __raid_recover_end_io()
1851 faila = rbio->faila; in __raid_recover_end_io()
1852 failb = rbio->failb; in __raid_recover_end_io()
1854 if (rbio->operation == BTRFS_RBIO_READ_REBUILD || in __raid_recover_end_io()
1855 rbio->operation == BTRFS_RBIO_REBUILD_MISSING) { in __raid_recover_end_io()
1856 spin_lock_irq(&rbio->bio_list_lock); in __raid_recover_end_io()
1857 set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags); in __raid_recover_end_io()
1858 spin_unlock_irq(&rbio->bio_list_lock); in __raid_recover_end_io()
1863 for (pagenr = 0; pagenr < rbio->stripe_npages; pagenr++) { in __raid_recover_end_io()
1868 if (rbio->operation == BTRFS_RBIO_PARITY_SCRUB && in __raid_recover_end_io()
1869 !test_bit(pagenr, rbio->dbitmap)) in __raid_recover_end_io()
1875 for (stripe = 0; stripe < rbio->real_stripes; stripe++) { in __raid_recover_end_io()
1880 if ((rbio->operation == BTRFS_RBIO_READ_REBUILD || in __raid_recover_end_io()
1881 rbio->operation == BTRFS_RBIO_REBUILD_MISSING) && in __raid_recover_end_io()
1891 if (rbio->bbio->map_type & BTRFS_BLOCK_GROUP_RAID6) { in __raid_recover_end_io()
1897 if (faila == rbio->nr_data) { in __raid_recover_end_io()
1923 if (rbio->bbio->raid_map[failb] == RAID6_Q_STRIPE) { in __raid_recover_end_io()
1924 if (rbio->bbio->raid_map[faila] == in __raid_recover_end_io()
1936 if (rbio->bbio->raid_map[failb] == RAID5_P_STRIPE) { in __raid_recover_end_io()
1937 raid6_datap_recov(rbio->real_stripes, in __raid_recover_end_io()
1940 raid6_2data_recov(rbio->real_stripes, in __raid_recover_end_io()
1948 BUG_ON(failb != -1); in __raid_recover_end_io()
1951 copy_page(pointers[faila], pointers[rbio->nr_data]); in __raid_recover_end_io()
1955 for (stripe = faila; stripe < rbio->nr_data - 1; stripe++) in __raid_recover_end_io()
1957 pointers[rbio->nr_data - 1] = p; in __raid_recover_end_io()
1960 run_xor(pointers, rbio->nr_data - 1, PAGE_SIZE); in __raid_recover_end_io()
1968 if (rbio->operation == BTRFS_RBIO_WRITE) { in __raid_recover_end_io()
1969 for (i = 0; i < rbio->stripe_npages; i++) { in __raid_recover_end_io()
1970 if (faila != -1) { in __raid_recover_end_io()
1974 if (failb != -1) { in __raid_recover_end_io()
1980 for (stripe = 0; stripe < rbio->real_stripes; stripe++) { in __raid_recover_end_io()
1985 if ((rbio->operation == BTRFS_RBIO_READ_REBUILD || in __raid_recover_end_io()
1986 rbio->operation == BTRFS_RBIO_REBUILD_MISSING) && in __raid_recover_end_io()
2006 if (rbio->operation == BTRFS_RBIO_READ_REBUILD || in __raid_recover_end_io()
2007 rbio->operation == BTRFS_RBIO_REBUILD_MISSING) { in __raid_recover_end_io()
2009 * - In case of two failures, where rbio->failb != -1: in __raid_recover_end_io()
2014 * on-disk content any more, otherwise, a later write/recover in __raid_recover_end_io()
2018 * - In case of single failure, where rbio->failb == -1: in __raid_recover_end_io()
2023 if (err == BLK_STS_OK && rbio->failb < 0) in __raid_recover_end_io()
2026 clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags); in __raid_recover_end_io()
2030 rbio->faila = -1; in __raid_recover_end_io()
2031 rbio->failb = -1; in __raid_recover_end_io()
2033 if (rbio->operation == BTRFS_RBIO_WRITE) in __raid_recover_end_io()
2035 else if (rbio->operation == BTRFS_RBIO_PARITY_SCRUB) in __raid_recover_end_io()
2050 struct btrfs_raid_bio *rbio = bio->bi_private; in raid_recover_end_io()
2056 if (bio->bi_status) in raid_recover_end_io()
2062 if (!atomic_dec_and_test(&rbio->stripes_pending)) in raid_recover_end_io()
2065 if (atomic_read(&rbio->error) > rbio->bbio->max_errors) in raid_recover_end_io()
2074 * when the IO is done.
2094 atomic_set(&rbio->error, 0); in __raid56_parity_recover()
2102 * So here we always re-read everything in recovery path. in __raid56_parity_recover()
2104 for (stripe = 0; stripe < rbio->real_stripes; stripe++) { in __raid56_parity_recover()
2105 if (rbio->faila == stripe || rbio->failb == stripe) { in __raid56_parity_recover()
2106 atomic_inc(&rbio->error); in __raid56_parity_recover()
2110 for (pagenr = 0; pagenr < rbio->stripe_npages; pagenr++) { in __raid56_parity_recover()
2113 stripe, pagenr, rbio->stripe_len); in __raid56_parity_recover()
2126 if (atomic_read(&rbio->error) <= rbio->bbio->max_errors) { in __raid56_parity_recover()
2138 atomic_set(&rbio->stripes_pending, bios_to_read); in __raid56_parity_recover()
2140 bio->bi_private = rbio; in __raid56_parity_recover()
2141 bio->bi_end_io = raid_recover_end_io; in __raid56_parity_recover()
2142 bio->bi_opf = REQ_OP_READ; in __raid56_parity_recover()
2144 btrfs_bio_wq_end_io(rbio->fs_info, bio, BTRFS_WQ_ENDIO_RAID56); in __raid56_parity_recover()
2152 if (rbio->operation == BTRFS_RBIO_READ_REBUILD || in __raid56_parity_recover()
2153 rbio->operation == BTRFS_RBIO_REBUILD_MISSING) in __raid56_parity_recover()
2159 return -EIO; in __raid56_parity_recover()
2176 ASSERT(bbio->mirror_num == mirror_num); in raid56_parity_recover()
2177 btrfs_io_bio(bio)->mirror_num = mirror_num; in raid56_parity_recover()
2187 rbio->operation = BTRFS_RBIO_READ_REBUILD; in raid56_parity_recover()
2190 rbio->faila = find_logical_bio_stripe(rbio, bio); in raid56_parity_recover()
2191 if (rbio->faila == -1) { in raid56_parity_recover()
2194 __func__, (u64)bio->bi_iter.bi_sector << 9, in raid56_parity_recover()
2195 (u64)bio->bi_iter.bi_size, bbio->map_type); in raid56_parity_recover()
2199 return -EIO; in raid56_parity_recover()
2204 rbio->generic_bio_cnt = 1; in raid56_parity_recover()
2220 rbio->failb = rbio->real_stripes - (mirror_num - 1); in raid56_parity_recover()
2221 ASSERT(rbio->failb > 0); in raid56_parity_recover()
2222 if (rbio->failb <= rbio->faila) in raid56_parity_recover()
2223 rbio->failb--; in raid56_parity_recover()
2240 * currently lock owner is done in raid56_parity_recover()
2284 bio_list_add(&rbio->bio_list, bio); in raid56_parity_alloc_scrub_rbio()
2289 ASSERT(!bio->bi_iter.bi_size); in raid56_parity_alloc_scrub_rbio()
2290 rbio->operation = BTRFS_RBIO_PARITY_SCRUB; in raid56_parity_alloc_scrub_rbio()
2297 for (i = rbio->nr_data; i < rbio->real_stripes; i++) { in raid56_parity_alloc_scrub_rbio()
2298 if (bbio->stripes[i].dev == scrub_dev) { in raid56_parity_alloc_scrub_rbio()
2299 rbio->scrubp = i; in raid56_parity_alloc_scrub_rbio()
2303 ASSERT(i < rbio->real_stripes); in raid56_parity_alloc_scrub_rbio()
2306 ASSERT(fs_info->sectorsize == PAGE_SIZE); in raid56_parity_alloc_scrub_rbio()
2307 ASSERT(rbio->stripe_npages == stripe_nsectors); in raid56_parity_alloc_scrub_rbio()
2308 bitmap_copy(rbio->dbitmap, dbitmap, stripe_nsectors); in raid56_parity_alloc_scrub_rbio()
2314 rbio->generic_bio_cnt = 1; in raid56_parity_alloc_scrub_rbio()
2326 ASSERT(logical >= rbio->bbio->raid_map[0]); in raid56_add_scrub_pages()
2327 ASSERT(logical + PAGE_SIZE <= rbio->bbio->raid_map[0] + in raid56_add_scrub_pages()
2328 rbio->stripe_len * rbio->nr_data); in raid56_add_scrub_pages()
2329 stripe_offset = (int)(logical - rbio->bbio->raid_map[0]); in raid56_add_scrub_pages()
2331 rbio->bio_pages[index] = page; in raid56_add_scrub_pages()
2345 for_each_set_bit(bit, rbio->dbitmap, rbio->stripe_npages) { in alloc_rbio_essential_pages()
2346 for (i = 0; i < rbio->real_stripes; i++) { in alloc_rbio_essential_pages()
2347 index = i * rbio->stripe_npages + bit; in alloc_rbio_essential_pages()
2348 if (rbio->stripe_pages[index]) in alloc_rbio_essential_pages()
2353 return -ENOMEM; in alloc_rbio_essential_pages()
2354 rbio->stripe_pages[index] = page; in alloc_rbio_essential_pages()
2363 struct btrfs_bio *bbio = rbio->bbio; in finish_parity_scrub()
2364 void **pointers = rbio->finish_pointers; in finish_parity_scrub()
2365 unsigned long *pbitmap = rbio->finish_pbitmap; in finish_parity_scrub()
2366 int nr_data = rbio->nr_data; in finish_parity_scrub()
2379 if (rbio->real_stripes - rbio->nr_data == 1) in finish_parity_scrub()
2381 else if (rbio->real_stripes - rbio->nr_data == 2) in finish_parity_scrub()
2386 if (bbio->num_tgtdevs && bbio->tgtdev_map[rbio->scrubp]) { in finish_parity_scrub()
2388 bitmap_copy(pbitmap, rbio->dbitmap, rbio->stripe_npages); in finish_parity_scrub()
2392 * Because the higher layers(scrubber) are unlikely to in finish_parity_scrub()
2396 clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags); in finish_parity_scrub()
2414 pointers[rbio->real_stripes - 1] = kmap(q_page); in finish_parity_scrub()
2417 atomic_set(&rbio->error, 0); in finish_parity_scrub()
2422 for_each_set_bit(pagenr, rbio->dbitmap, rbio->stripe_npages) { in finish_parity_scrub()
2433 raid6_call.gen_syndrome(rbio->real_stripes, PAGE_SIZE, in finish_parity_scrub()
2438 run_xor(pointers + 1, nr_data - 1, PAGE_SIZE); in finish_parity_scrub()
2442 p = rbio_stripe_page(rbio, rbio->scrubp, pagenr); in finish_parity_scrub()
2444 if (memcmp(parity, pointers[rbio->scrubp], PAGE_SIZE)) in finish_parity_scrub()
2445 copy_page(parity, pointers[rbio->scrubp]); in finish_parity_scrub()
2448 bitmap_clear(rbio->dbitmap, pagenr, 1); in finish_parity_scrub()
2468 for_each_set_bit(pagenr, rbio->dbitmap, rbio->stripe_npages) { in finish_parity_scrub()
2471 page = rbio_stripe_page(rbio, rbio->scrubp, pagenr); in finish_parity_scrub()
2473 page, rbio->scrubp, pagenr, rbio->stripe_len); in finish_parity_scrub()
2481 for_each_set_bit(pagenr, pbitmap, rbio->stripe_npages) { in finish_parity_scrub()
2484 page = rbio_stripe_page(rbio, rbio->scrubp, pagenr); in finish_parity_scrub()
2486 bbio->tgtdev_map[rbio->scrubp], in finish_parity_scrub()
2487 pagenr, rbio->stripe_len); in finish_parity_scrub()
2500 atomic_set(&rbio->stripes_pending, nr_data); in finish_parity_scrub()
2503 bio->bi_private = rbio; in finish_parity_scrub()
2504 bio->bi_end_io = raid_write_end_io; in finish_parity_scrub()
2505 bio->bi_opf = REQ_OP_WRITE; in finish_parity_scrub()
2520 if (stripe >= 0 && stripe < rbio->nr_data) in is_data_stripe()
2534 if (atomic_read(&rbio->error) > rbio->bbio->max_errors) in validate_rbio_for_parity_scrub()
2537 if (rbio->faila >= 0 || rbio->failb >= 0) { in validate_rbio_for_parity_scrub()
2538 int dfail = 0, failp = -1; in validate_rbio_for_parity_scrub()
2540 if (is_data_stripe(rbio, rbio->faila)) in validate_rbio_for_parity_scrub()
2542 else if (is_parity_stripe(rbio->faila)) in validate_rbio_for_parity_scrub()
2543 failp = rbio->faila; in validate_rbio_for_parity_scrub()
2545 if (is_data_stripe(rbio, rbio->failb)) in validate_rbio_for_parity_scrub()
2547 else if (is_parity_stripe(rbio->failb)) in validate_rbio_for_parity_scrub()
2548 failp = rbio->failb; in validate_rbio_for_parity_scrub()
2555 if (dfail > rbio->bbio->max_errors - 1) in validate_rbio_for_parity_scrub()
2573 if (failp != rbio->scrubp) in validate_rbio_for_parity_scrub()
2596 struct btrfs_raid_bio *rbio = bio->bi_private; in raid56_parity_scrub_end_io()
2598 if (bio->bi_status) in raid56_parity_scrub_end_io()
2605 if (!atomic_dec_and_test(&rbio->stripes_pending)) in raid56_parity_scrub_end_io()
2631 atomic_set(&rbio->error, 0); in raid56_parity_scrub_stripe()
2636 for (stripe = 0; stripe < rbio->real_stripes; stripe++) { in raid56_parity_scrub_stripe()
2637 for_each_set_bit(pagenr, rbio->dbitmap, rbio->stripe_npages) { in raid56_parity_scrub_stripe()
2658 stripe, pagenr, rbio->stripe_len); in raid56_parity_scrub_stripe()
2679 atomic_set(&rbio->stripes_pending, bios_to_read); in raid56_parity_scrub_stripe()
2681 bio->bi_private = rbio; in raid56_parity_scrub_stripe()
2682 bio->bi_end_io = raid56_parity_scrub_end_io; in raid56_parity_scrub_stripe()
2683 bio->bi_opf = REQ_OP_READ; in raid56_parity_scrub_stripe()
2685 btrfs_bio_wq_end_io(rbio->fs_info, bio, BTRFS_WQ_ENDIO_RAID56); in raid56_parity_scrub_stripe()
2689 /* the actual write will happen once the reads are done */ in raid56_parity_scrub_stripe()
2730 rbio->operation = BTRFS_RBIO_REBUILD_MISSING; in raid56_alloc_missing_rbio()
2731 bio_list_add(&rbio->bio_list, bio); in raid56_alloc_missing_rbio()
2736 ASSERT(!bio->bi_iter.bi_size); in raid56_alloc_missing_rbio()
2738 rbio->faila = find_logical_bio_stripe(rbio, bio); in raid56_alloc_missing_rbio()
2739 if (rbio->faila == -1) { in raid56_alloc_missing_rbio()
2749 rbio->generic_bio_cnt = 1; in raid56_alloc_missing_rbio()