Lines Matching refs:bio
40 static void bio_csum(struct bio *bio, struct bkey *k) in bio_csum() argument
46 bio_for_each_segment(bv, bio, iter) { in bio_csum()
111 struct bio *bio = op->bio; in bch_data_invalidate() local
114 bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector); in bch_data_invalidate()
116 while (bio_sectors(bio)) { in bch_data_invalidate()
117 unsigned int sectors = min(bio_sectors(bio), in bch_data_invalidate()
123 bio->bi_iter.bi_sector += sectors; in bch_data_invalidate()
124 bio->bi_iter.bi_size -= sectors << 9; in bch_data_invalidate()
128 bio->bi_iter.bi_sector, in bch_data_invalidate()
134 bio_put(bio); in bch_data_invalidate()
169 static void bch_data_insert_endio(struct bio *bio) in bch_data_insert_endio() argument
171 struct closure *cl = bio->bi_private; in bch_data_insert_endio()
174 if (bio->bi_status) { in bch_data_insert_endio()
177 op->status = bio->bi_status; in bch_data_insert_endio()
184 bch_bbio_endio(op->c, bio, bio->bi_status, "writing data to cache"); in bch_data_insert_endio()
190 struct bio *bio = op->bio, *n; in bch_data_insert_start() local
195 if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0) in bch_data_insert_start()
202 bio->bi_opf &= ~(REQ_PREFLUSH|REQ_FUA); in bch_data_insert_start()
220 SET_KEY_OFFSET(k, bio->bi_iter.bi_sector); in bch_data_insert_start()
222 if (!bch_alloc_sectors(op->c, k, bio_sectors(bio), in bch_data_insert_start()
227 n = bio_next_split(bio, KEY_SIZE(k), GFP_NOIO, split); in bch_data_insert_start()
249 } while (n != bio); in bch_data_insert_start()
279 bio_put(bio); in bch_data_insert_start()
312 trace_bcache_write(op->c, op->inode, op->bio, in bch_data_insert()
316 bio_get(op->bio); in bch_data_insert()
363 static bool check_should_bypass(struct cached_dev *dc, struct bio *bio) in check_should_bypass() argument
373 (bio_op(bio) == REQ_OP_DISCARD)) in check_should_bypass()
378 op_is_write(bio_op(bio)))) in check_should_bypass()
391 if ((bio->bi_opf & (REQ_RAHEAD|REQ_BACKGROUND))) { in check_should_bypass()
392 if (!(bio->bi_opf & (REQ_META|REQ_PRIO)) && in check_should_bypass()
397 if (bio->bi_iter.bi_sector & (c->cache->sb.block_size - 1) || in check_should_bypass()
398 bio_sectors(bio) & (c->cache->sb.block_size - 1)) { in check_should_bypass()
416 hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash) in check_should_bypass()
417 if (i->last == bio->bi_iter.bi_sector && in check_should_bypass()
426 if (i->sequential + bio->bi_iter.bi_size > i->sequential) in check_should_bypass()
427 i->sequential += bio->bi_iter.bi_size; in check_should_bypass()
429 i->last = bio_end_sector(bio); in check_should_bypass()
444 trace_bcache_bypass_sequential(bio); in check_should_bypass()
449 trace_bcache_bypass_congested(bio); in check_should_bypass()
454 bch_rescale_priorities(c, bio_sectors(bio)); in check_should_bypass()
457 bch_mark_sectors_bypassed(c, dc, bio_sectors(bio)); in check_should_bypass()
467 struct bbio bio; member
468 struct bio *orig_bio;
469 struct bio *cache_miss;
485 static void bch_cache_read_endio(struct bio *bio) in bch_cache_read_endio() argument
487 struct bbio *b = container_of(bio, struct bbio, bio); in bch_cache_read_endio()
488 struct closure *cl = bio->bi_private; in bch_cache_read_endio()
498 if (bio->bi_status) in bch_cache_read_endio()
499 s->iop.status = bio->bi_status; in bch_cache_read_endio()
506 bch_bbio_endio(s->iop.c, bio, bio->bi_status, "reading from cache"); in bch_cache_read_endio()
516 struct bio *n, *bio = &s->bio.bio; in cache_lookup_fn() local
520 if (bkey_cmp(k, &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0)) <= 0) in cache_lookup_fn()
524 KEY_START(k) > bio->bi_iter.bi_sector) { in cache_lookup_fn()
525 unsigned int bio_sectors = bio_sectors(bio); in cache_lookup_fn()
528 KEY_START(k) - bio->bi_iter.bi_sector) in cache_lookup_fn()
530 int ret = s->d->cache_miss(b, s, bio, sectors); in cache_lookup_fn()
550 n = bio_next_split(bio, min_t(uint64_t, INT_MAX, in cache_lookup_fn()
551 KEY_OFFSET(k) - bio->bi_iter.bi_sector), in cache_lookup_fn()
554 bio_key = &container_of(n, struct bbio, bio)->key; in cache_lookup_fn()
575 return n == bio ? MAP_DONE : MAP_CONTINUE; in cache_lookup_fn()
581 struct bio *bio = &s->bio.bio; in cache_lookup() local
588 &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0), in cache_lookup()
621 static void request_endio(struct bio *bio) in request_endio() argument
623 struct closure *cl = bio->bi_private; in request_endio()
625 if (bio->bi_status) { in request_endio()
628 s->iop.status = bio->bi_status; in request_endio()
633 bio_put(bio); in request_endio()
637 static void backing_request_endio(struct bio *bio) in backing_request_endio() argument
639 struct closure *cl = bio->bi_private; in backing_request_endio()
641 if (bio->bi_status) { in backing_request_endio()
653 bio->bi_opf & REQ_PREFLUSH)) { in backing_request_endio()
655 dc->backing_dev_name, bio->bi_status); in backing_request_endio()
658 s->iop.status = bio->bi_status; in backing_request_endio()
662 bch_count_backing_io_errors(dc, bio); in backing_request_endio()
665 bio_put(bio); in backing_request_endio()
683 struct bio *orig_bio, in do_bio_hook()
686 struct bio *bio = &s->bio.bio; in do_bio_hook() local
688 bio_init(bio, NULL, 0); in do_bio_hook()
689 __bio_clone_fast(bio, orig_bio); in do_bio_hook()
696 bio->bi_end_io = end_io_fn; in do_bio_hook()
697 bio->bi_private = &s->cl; in do_bio_hook()
699 bio_cnt_set(bio, 3); in do_bio_hook()
708 if (s->iop.bio) in search_free()
709 bio_put(s->iop.bio); in search_free()
716 static inline struct search *search_alloc(struct bio *bio, in search_alloc() argument
725 do_bio_hook(s, bio, request_endio); in search_alloc()
728 s->orig_bio = bio; in search_alloc()
733 s->write = op_is_write(bio_op(bio)); in search_alloc()
739 s->iop.bio = NULL; in search_alloc()
745 s->iop.flush_journal = op_is_flush(bio->bi_opf); in search_alloc()
771 if (s->iop.bio) in cached_dev_read_error_done()
772 bio_free_pages(s->iop.bio); in cached_dev_read_error_done()
780 struct bio *bio = &s->bio.bio; in cached_dev_read_error() local
799 closure_bio_submit(s->iop.c, bio, cl); in cached_dev_read_error()
813 if (s->iop.bio) in cached_dev_cache_miss_done()
814 bio_free_pages(s->iop.bio); in cached_dev_cache_miss_done()
833 if (s->iop.bio) { in cached_dev_read_done()
834 bio_reset(s->iop.bio); in cached_dev_read_done()
835 s->iop.bio->bi_iter.bi_sector = in cached_dev_read_done()
837 bio_copy_dev(s->iop.bio, s->cache_miss); in cached_dev_read_done()
838 s->iop.bio->bi_iter.bi_size = s->insert_bio_sectors << 9; in cached_dev_read_done()
839 bch_bio_map(s->iop.bio, NULL); in cached_dev_read_done()
841 bio_copy_data(s->cache_miss, s->iop.bio); in cached_dev_read_done()
853 if (s->iop.bio && in cached_dev_read_done()
873 else if (s->iop.bio || verify(dc)) in cached_dev_read_done_bh()
880 struct bio *bio, unsigned int sectors) in cached_dev_cache_miss() argument
884 struct bio *miss, *cache_bio; in cached_dev_cache_miss()
890 miss = bio_next_split(bio, sectors, GFP_NOIO, &s->d->bio_split); in cached_dev_cache_miss()
891 ret = miss == bio ? MAP_DONE : MAP_CONTINUE; in cached_dev_cache_miss()
898 s->insert_bio_sectors = min3(size_limit, sectors, bio_sectors(bio)); in cached_dev_cache_miss()
901 bio->bi_iter.bi_sector + s->insert_bio_sectors, in cached_dev_cache_miss()
910 miss = bio_next_split(bio, s->insert_bio_sectors, GFP_NOIO, in cached_dev_cache_miss()
914 ret = miss == bio ? MAP_DONE : -EINTR; in cached_dev_cache_miss()
934 s->iop.bio = cache_bio; in cached_dev_cache_miss()
972 struct bio *bio = &s->bio.bio; in cached_dev_write() local
973 struct bkey start = KEY(dc->disk.id, bio->bi_iter.bi_sector, 0); in cached_dev_write()
974 struct bkey end = KEY(dc->disk.id, bio_end_sector(bio), 0); in cached_dev_write()
995 if (bio_op(bio) == REQ_OP_DISCARD) in cached_dev_write()
1006 s->iop.bio = s->orig_bio; in cached_dev_write()
1007 bio_get(s->iop.bio); in cached_dev_write()
1009 if (bio_op(bio) == REQ_OP_DISCARD && in cached_dev_write()
1014 bio->bi_end_io = backing_request_endio; in cached_dev_write()
1015 closure_bio_submit(s->iop.c, bio, cl); in cached_dev_write()
1019 s->iop.bio = bio; in cached_dev_write()
1021 if (bio->bi_opf & REQ_PREFLUSH) { in cached_dev_write()
1026 struct bio *flush; in cached_dev_write()
1034 bio_copy_dev(flush, bio); in cached_dev_write()
1042 s->iop.bio = bio_clone_fast(bio, GFP_NOIO, &dc->disk.bio_split); in cached_dev_write()
1044 bio->bi_end_io = backing_request_endio; in cached_dev_write()
1045 closure_bio_submit(s->iop.c, bio, cl); in cached_dev_write()
1056 struct bio *bio = &s->bio.bio; in cached_dev_nodata() local
1062 bio->bi_end_io = backing_request_endio; in cached_dev_nodata()
1063 closure_bio_submit(s->iop.c, bio, cl); in cached_dev_nodata()
1076 static void detached_dev_end_io(struct bio *bio) in detached_dev_end_io() argument
1080 ddip = bio->bi_private; in detached_dev_end_io()
1081 bio->bi_end_io = ddip->bi_end_io; in detached_dev_end_io()
1082 bio->bi_private = ddip->bi_private; in detached_dev_end_io()
1085 bio_end_io_acct_remapped(bio, ddip->start_time, ddip->orig_bdev); in detached_dev_end_io()
1087 if (bio->bi_status) { in detached_dev_end_io()
1091 bch_count_backing_io_errors(dc, bio); in detached_dev_end_io()
1095 bio->bi_end_io(bio); in detached_dev_end_io()
1098 static void detached_dev_do_request(struct bcache_device *d, struct bio *bio, in detached_dev_do_request() argument
1111 bio->bi_status = BLK_STS_RESOURCE; in detached_dev_do_request()
1112 bio->bi_end_io(bio); in detached_dev_do_request()
1120 ddip->bi_end_io = bio->bi_end_io; in detached_dev_do_request()
1121 ddip->bi_private = bio->bi_private; in detached_dev_do_request()
1122 bio->bi_end_io = detached_dev_end_io; in detached_dev_do_request()
1123 bio->bi_private = ddip; in detached_dev_do_request()
1125 if ((bio_op(bio) == REQ_OP_DISCARD) && in detached_dev_do_request()
1127 bio->bi_end_io(bio); in detached_dev_do_request()
1129 submit_bio_noacct(bio); in detached_dev_do_request()
1172 blk_qc_t cached_dev_submit_bio(struct bio *bio) in cached_dev_submit_bio() argument
1175 struct block_device *orig_bdev = bio->bi_bdev; in cached_dev_submit_bio()
1179 int rw = bio_data_dir(bio); in cached_dev_submit_bio()
1183 bio->bi_status = BLK_STS_IOERR; in cached_dev_submit_bio()
1184 bio_endio(bio); in cached_dev_submit_bio()
1203 start_time = bio_start_io_acct(bio); in cached_dev_submit_bio()
1205 bio_set_dev(bio, dc->bdev); in cached_dev_submit_bio()
1206 bio->bi_iter.bi_sector += dc->sb.data_offset; in cached_dev_submit_bio()
1209 s = search_alloc(bio, d, orig_bdev, start_time); in cached_dev_submit_bio()
1210 trace_bcache_request_start(s->d, bio); in cached_dev_submit_bio()
1212 if (!bio->bi_iter.bi_size) { in cached_dev_submit_bio()
1221 s->iop.bypass = check_should_bypass(dc, bio); in cached_dev_submit_bio()
1230 detached_dev_do_request(d, bio, orig_bdev, start_time); in cached_dev_submit_bio()
1256 struct bio *bio, unsigned int sectors) in flash_dev_cache_miss() argument
1258 unsigned int bytes = min(sectors, bio_sectors(bio)) << 9; in flash_dev_cache_miss()
1260 swap(bio->bi_iter.bi_size, bytes); in flash_dev_cache_miss()
1261 zero_fill_bio(bio); in flash_dev_cache_miss()
1262 swap(bio->bi_iter.bi_size, bytes); in flash_dev_cache_miss()
1264 bio_advance(bio, bytes); in flash_dev_cache_miss()
1266 if (!bio->bi_iter.bi_size) in flash_dev_cache_miss()
1282 blk_qc_t flash_dev_submit_bio(struct bio *bio) in flash_dev_submit_bio() argument
1286 struct bcache_device *d = bio->bi_bdev->bd_disk->private_data; in flash_dev_submit_bio()
1289 bio->bi_status = BLK_STS_IOERR; in flash_dev_submit_bio()
1290 bio_endio(bio); in flash_dev_submit_bio()
1294 s = search_alloc(bio, d, bio->bi_bdev, bio_start_io_acct(bio)); in flash_dev_submit_bio()
1296 bio = &s->bio.bio; in flash_dev_submit_bio()
1298 trace_bcache_request_start(s->d, bio); in flash_dev_submit_bio()
1300 if (!bio->bi_iter.bi_size) { in flash_dev_submit_bio()
1308 } else if (bio_data_dir(bio)) { in flash_dev_submit_bio()
1310 &KEY(d->id, bio->bi_iter.bi_sector, 0), in flash_dev_submit_bio()
1311 &KEY(d->id, bio_end_sector(bio), 0)); in flash_dev_submit_bio()
1313 s->iop.bypass = (bio_op(bio) == REQ_OP_DISCARD) != 0; in flash_dev_submit_bio()
1315 s->iop.bio = bio; in flash_dev_submit_bio()