Lines Matching refs:dio
367 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
1379 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio) in submit_flush_bio() argument
1385 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in submit_flush_bio()
1408 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio) in do_endio_flush() argument
1410 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in do_endio_flush()
1412 if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic))) in do_endio_flush()
1413 submit_flush_bio(ic, dio); in do_endio_flush()
1418 static void dec_in_flight(struct dm_integrity_io *dio) in dec_in_flight() argument
1420 if (atomic_dec_and_test(&dio->in_flight)) { in dec_in_flight()
1421 struct dm_integrity_c *ic = dio->ic; in dec_in_flight()
1424 remove_range(ic, &dio->range); in dec_in_flight()
1426 if (unlikely(dio->write)) in dec_in_flight()
1429 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in dec_in_flight()
1431 if (unlikely(dio->bi_status) && !bio->bi_status) in dec_in_flight()
1432 bio->bi_status = dio->bi_status; in dec_in_flight()
1433 if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) { in dec_in_flight()
1434 dio->range.logical_sector += dio->range.n_sectors; in dec_in_flight()
1435 bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT); in dec_in_flight()
1436 INIT_WORK(&dio->work, integrity_bio_wait); in dec_in_flight()
1437 queue_work(ic->wait_wq, &dio->work); in dec_in_flight()
1440 do_endio_flush(ic, dio); in dec_in_flight()
1446 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); in integrity_end_io() local
1448 bio->bi_iter = dio->orig_bi_iter; in integrity_end_io()
1449 bio->bi_disk = dio->orig_bi_disk; in integrity_end_io()
1450 bio->bi_partno = dio->orig_bi_partno; in integrity_end_io()
1451 if (dio->orig_bi_integrity) { in integrity_end_io()
1452 bio->bi_integrity = dio->orig_bi_integrity; in integrity_end_io()
1455 bio->bi_end_io = dio->orig_bi_end_io; in integrity_end_io()
1457 if (dio->completion) in integrity_end_io()
1458 complete(dio->completion); in integrity_end_io()
1460 dec_in_flight(dio); in integrity_end_io()
1510 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); in integrity_metadata() local
1511 struct dm_integrity_c *ic = dio->ic; in integrity_metadata()
1519 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in integrity_metadata()
1523 unsigned sectors_to_process = dio->range.n_sectors; in integrity_metadata()
1524 sector_t sector = dio->range.logical_sector; in integrity_metadata()
1540 __bio_for_each_segment(bv, bio, iter, dio->orig_bi_iter) { in integrity_metadata()
1557 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1558 checksums_ptr - checksums, !dio->write ? TAG_CMP : TAG_WRITE); in integrity_metadata()
1584 struct bio_integrity_payload *bip = dio->orig_bi_integrity; in integrity_metadata()
1589 unsigned data_to_process = dio->range.n_sectors; in integrity_metadata()
1600 r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1601 this_len, !dio->write ? TAG_READ : TAG_WRITE); in integrity_metadata()
1611 dec_in_flight(dio); in integrity_metadata()
1614 dio->bi_status = errno_to_blk_status(r); in integrity_metadata()
1615 dec_in_flight(dio); in integrity_metadata()
1621 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); in dm_integrity_map() local
1626 dio->ic = ic; in dm_integrity_map()
1627 dio->bi_status = 0; in dm_integrity_map()
1630 submit_flush_bio(ic, dio); in dm_integrity_map()
1634 dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); in dm_integrity_map()
1635 dio->write = bio_op(bio) == REQ_OP_WRITE; in dm_integrity_map()
1636 dio->fua = dio->write && bio->bi_opf & REQ_FUA; in dm_integrity_map()
1637 if (unlikely(dio->fua)) { in dm_integrity_map()
1644 if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) { in dm_integrity_map()
1646 (unsigned long long)dio->range.logical_sector, bio_sectors(bio), in dm_integrity_map()
1650 …if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1… in dm_integrity_map()
1653 (unsigned long long)dio->range.logical_sector, bio_sectors(bio)); in dm_integrity_map()
1690 if (unlikely(ic->mode == 'R') && unlikely(dio->write)) in dm_integrity_map()
1693 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); in dm_integrity_map()
1694 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); in dm_integrity_map()
1697 dm_integrity_map_continue(dio, true); in dm_integrity_map()
1701 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio, in __journal_read_write() argument
1704 struct dm_integrity_c *ic = dio->ic; in __journal_read_write()
1708 logical_sector = dio->range.logical_sector; in __journal_read_write()
1709 n_sectors = dio->range.n_sectors; in __journal_read_write()
1720 if (likely(dio->write)) in __journal_read_write()
1726 if (unlikely(!dio->write)) { in __journal_read_write()
1773 if (likely(dio->write)) in __journal_read_write()
1781 if (likely(dio->write)) in __journal_read_write()
1786 if (likely(dio->write)) { in __journal_read_write()
1822 if (unlikely(!dio->write)) in __journal_read_write()
1827 if (likely(dio->write)) { in __journal_read_write()
1837 remove_range(ic, &dio->range); in __journal_read_write()
1843 dio->range.logical_sector = logical_sector; in __journal_read_write()
1844 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); in __journal_read_write()
1845 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); in __journal_read_write()
1852 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map) in dm_integrity_map_continue() argument
1854 struct dm_integrity_c *ic = dio->ic; in dm_integrity_map_continue()
1855 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in dm_integrity_map_continue()
1859 bool need_sync_io = ic->internal_hash && !dio->write; in dm_integrity_map_continue()
1862 INIT_WORK(&dio->work, integrity_bio_wait); in dm_integrity_map_continue()
1863 queue_work(ic->metadata_wq, &dio->work); in dm_integrity_map_continue()
1875 dio->range.n_sectors = bio_sectors(bio); in dm_integrity_map_continue()
1878 if (dio->write) { in dm_integrity_map_continue()
1882 dio->range.n_sectors = min(dio->range.n_sectors, in dm_integrity_map_continue()
1884 if (unlikely(!dio->range.n_sectors)) { in dm_integrity_map_continue()
1890 range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block; in dm_integrity_map_continue()
1908 add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i); in dm_integrity_map_continue()
1922 } while ((i += ic->sectors_per_block) < dio->range.n_sectors); in dm_integrity_map_continue()
1928 journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
1930 if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector)) in dm_integrity_map_continue()
1931 dio->range.n_sectors = next_sector - dio->range.logical_sector; in dm_integrity_map_continue()
1935 for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) { in dm_integrity_map_continue()
1936 if (!test_journal_node(ic, jp, dio->range.logical_sector + i)) in dm_integrity_map_continue()
1939 dio->range.n_sectors = i; in dm_integrity_map_continue()
1943 if (unlikely(!add_new_range(ic, &dio->range, true))) { in dm_integrity_map_continue()
1952 INIT_WORK(&dio->work, integrity_bio_wait); in dm_integrity_map_continue()
1953 queue_work(ic->wait_wq, &dio->work); in dm_integrity_map_continue()
1957 dio->range.n_sectors = ic->sectors_per_block; in dm_integrity_map_continue()
1958 wait_and_add_new_range(ic, &dio->range); in dm_integrity_map_continue()
1966 unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
1968 remove_range_unlocked(ic, &dio->range); in dm_integrity_map_continue()
1981 if (ic->mode == 'B' && dio->write) { in dm_integrity_map_continue()
1982 if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in dm_integrity_map_continue()
1983 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) { in dm_integrity_map_continue()
1986 bbs = sector_to_bitmap_block(ic, dio->range.logical_sector); in dm_integrity_map_continue()
1995 dio->in_flight = (atomic_t)ATOMIC_INIT(2); in dm_integrity_map_continue()
1999 dio->completion = &read_comp; in dm_integrity_map_continue()
2001 dio->completion = NULL; in dm_integrity_map_continue()
2003 dio->orig_bi_iter = bio->bi_iter; in dm_integrity_map_continue()
2005 dio->orig_bi_disk = bio->bi_disk; in dm_integrity_map_continue()
2006 dio->orig_bi_partno = bio->bi_partno; in dm_integrity_map_continue()
2009 dio->orig_bi_integrity = bio_integrity(bio); in dm_integrity_map_continue()
2013 dio->orig_bi_end_io = bio->bi_end_io; in dm_integrity_map_continue()
2016 bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT; in dm_integrity_map_continue()
2022 dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector)) in dm_integrity_map_continue()
2025 if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector, in dm_integrity_map_continue()
2026 dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR)) in dm_integrity_map_continue()
2031 integrity_metadata(&dio->work); in dm_integrity_map_continue()
2034 dec_in_flight(dio); in dm_integrity_map_continue()
2037 INIT_WORK(&dio->work, integrity_metadata); in dm_integrity_map_continue()
2038 queue_work(ic->metadata_wq, &dio->work); in dm_integrity_map_continue()
2044 if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry))) in dm_integrity_map_continue()
2047 do_endio_flush(ic, dio); in dm_integrity_map_continue()
2053 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); in integrity_bio_wait() local
2055 dm_integrity_map_continue(dio, false); in integrity_bio_wait()
2491 struct dm_integrity_io *dio; in bitmap_block_work() local
2493 dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); in bitmap_block_work()
2495 if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in bitmap_block_work()
2496 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) { in bitmap_block_work()
2497 remove_range(ic, &dio->range); in bitmap_block_work()
2498 INIT_WORK(&dio->work, integrity_bio_wait); in bitmap_block_work()
2499 queue_work(ic->wait_wq, &dio->work); in bitmap_block_work()
2501 block_bitmap_op(ic, ic->journal, dio->range.logical_sector, in bitmap_block_work()
2502 dio->range.n_sectors, BITMAP_OP_SET); in bitmap_block_work()
2515 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); in bitmap_block_work() local
2517 block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in bitmap_block_work()
2518 dio->range.n_sectors, BITMAP_OP_SET); in bitmap_block_work()
2520 remove_range(ic, &dio->range); in bitmap_block_work()
2521 INIT_WORK(&dio->work, integrity_bio_wait); in bitmap_block_work()
2522 queue_work(ic->wait_wq, &dio->work); in bitmap_block_work()