Lines Matching refs:sdio
162 static inline unsigned dio_pages_present(struct dio_submit *sdio) in dio_pages_present() argument
164 return sdio->tail - sdio->head; in dio_pages_present()
170 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) in dio_refill_pages() argument
174 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES, in dio_refill_pages()
175 &sdio->from); in dio_refill_pages()
177 if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) { in dio_refill_pages()
188 sdio->head = 0; in dio_refill_pages()
189 sdio->tail = 1; in dio_refill_pages()
190 sdio->from = 0; in dio_refill_pages()
191 sdio->to = PAGE_SIZE; in dio_refill_pages()
196 iov_iter_advance(sdio->iter, ret); in dio_refill_pages()
197 ret += sdio->from; in dio_refill_pages()
198 sdio->head = 0; in dio_refill_pages()
199 sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE; in dio_refill_pages()
200 sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1; in dio_refill_pages()
213 struct dio_submit *sdio) in dio_get_page() argument
215 if (dio_pages_present(sdio) == 0) { in dio_get_page()
218 ret = dio_refill_pages(dio, sdio); in dio_get_page()
221 BUG_ON(dio_pages_present(sdio) == 0); in dio_get_page()
223 return dio->pages[sdio->head]; in dio_get_page()
391 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, in dio_bio_alloc() argument
405 sdio->cur_page_fs_offset >> inode->i_blkbits, in dio_bio_alloc()
417 sdio->bio = bio; in dio_bio_alloc()
418 sdio->logical_offset_in_bio = sdio->cur_page_fs_offset; in dio_bio_alloc()
428 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) in dio_bio_submit() argument
430 struct bio *bio = sdio->bio; in dio_bio_submit()
444 if (sdio->submit_io) { in dio_bio_submit()
445 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); in dio_bio_submit()
450 sdio->bio = NULL; in dio_bio_submit()
451 sdio->boundary = 0; in dio_bio_submit()
452 sdio->logical_offset_in_bio = 0; in dio_bio_submit()
458 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) in dio_cleanup() argument
460 while (sdio->head < sdio->tail) in dio_cleanup()
461 put_page(dio->pages[sdio->head++]); in dio_cleanup()
550 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio) in dio_bio_reap() argument
554 if (sdio->reap_counter++ >= 64) { in dio_bio_reap()
568 sdio->reap_counter = 0; in dio_bio_reap()
632 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, in get_more_blocks() argument
640 unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor; in get_more_blocks()
649 BUG_ON(sdio->block_in_file >= sdio->final_block_in_request); in get_more_blocks()
650 fs_startblk = sdio->block_in_file >> sdio->blkfactor; in get_more_blocks()
651 fs_endblk = (sdio->final_block_in_request - 1) >> in get_more_blocks()
652 sdio->blkfactor; in get_more_blocks()
676 ret = (*sdio->get_block)(dio->inode, fs_startblk, in get_more_blocks()
691 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, in dio_new_bio() argument
697 ret = dio_bio_reap(dio, sdio); in dio_new_bio()
700 sector = start_sector << (sdio->blkbits - 9); in dio_new_bio()
701 nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES); in dio_new_bio()
703 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); in dio_new_bio()
704 sdio->boundary = 0; in dio_new_bio()
716 static inline int dio_bio_add_page(struct dio_submit *sdio) in dio_bio_add_page() argument
720 ret = bio_add_page(sdio->bio, sdio->cur_page, in dio_bio_add_page()
721 sdio->cur_page_len, sdio->cur_page_offset); in dio_bio_add_page()
722 if (ret == sdio->cur_page_len) { in dio_bio_add_page()
726 if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE) in dio_bio_add_page()
727 sdio->pages_in_io--; in dio_bio_add_page()
728 get_page(sdio->cur_page); in dio_bio_add_page()
729 sdio->final_block_in_bio = sdio->cur_page_block + in dio_bio_add_page()
730 (sdio->cur_page_len >> sdio->blkbits); in dio_bio_add_page()
748 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio, in dio_send_cur_page() argument
753 if (sdio->bio) { in dio_send_cur_page()
754 loff_t cur_offset = sdio->cur_page_fs_offset; in dio_send_cur_page()
755 loff_t bio_next_offset = sdio->logical_offset_in_bio + in dio_send_cur_page()
756 sdio->bio->bi_iter.bi_size; in dio_send_cur_page()
778 if (sdio->final_block_in_bio != sdio->cur_page_block || in dio_send_cur_page()
780 !fscrypt_mergeable_bio(sdio->bio, dio->inode, in dio_send_cur_page()
782 dio_bio_submit(dio, sdio); in dio_send_cur_page()
785 if (sdio->bio == NULL) { in dio_send_cur_page()
786 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
791 if (dio_bio_add_page(sdio) != 0) { in dio_send_cur_page()
792 dio_bio_submit(dio, sdio); in dio_send_cur_page()
793 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
795 ret = dio_bio_add_page(sdio); in dio_send_cur_page()
821 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, in submit_page_section() argument
826 int boundary = sdio->boundary; /* dio_send_cur_page may clear it */ in submit_page_section()
838 if (sdio->cur_page == page && in submit_page_section()
839 sdio->cur_page_offset + sdio->cur_page_len == offset && in submit_page_section()
840 sdio->cur_page_block + in submit_page_section()
841 (sdio->cur_page_len >> sdio->blkbits) == blocknr) { in submit_page_section()
842 sdio->cur_page_len += len; in submit_page_section()
849 if (sdio->cur_page) { in submit_page_section()
850 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
851 put_page(sdio->cur_page); in submit_page_section()
852 sdio->cur_page = NULL; in submit_page_section()
858 sdio->cur_page = page; in submit_page_section()
859 sdio->cur_page_offset = offset; in submit_page_section()
860 sdio->cur_page_len = len; in submit_page_section()
861 sdio->cur_page_block = blocknr; in submit_page_section()
862 sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits; in submit_page_section()
869 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
870 if (sdio->bio) in submit_page_section()
871 dio_bio_submit(dio, sdio); in submit_page_section()
872 put_page(sdio->cur_page); in submit_page_section()
873 sdio->cur_page = NULL; in submit_page_section()
887 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio, in dio_zero_block() argument
895 sdio->start_zero_done = 1; in dio_zero_block()
896 if (!sdio->blkfactor || !buffer_new(map_bh)) in dio_zero_block()
899 dio_blocks_per_fs_block = 1 << sdio->blkfactor; in dio_zero_block()
900 this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1); in dio_zero_block()
912 this_chunk_bytes = this_chunk_blocks << sdio->blkbits; in dio_zero_block()
915 if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes, in dio_zero_block()
916 sdio->next_block_for_io, map_bh)) in dio_zero_block()
919 sdio->next_block_for_io += this_chunk_blocks; in dio_zero_block()
938 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, in do_direct_IO() argument
941 const unsigned blkbits = sdio->blkbits; in do_direct_IO()
942 const unsigned i_blkbits = blkbits + sdio->blkfactor; in do_direct_IO()
945 while (sdio->block_in_file < sdio->final_block_in_request) { in do_direct_IO()
949 page = dio_get_page(dio, sdio); in do_direct_IO()
954 from = sdio->head ? 0 : sdio->from; in do_direct_IO()
955 to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE; in do_direct_IO()
956 sdio->head++; in do_direct_IO()
963 if (sdio->blocks_available == 0) { in do_direct_IO()
970 ret = get_more_blocks(dio, sdio, map_bh); in do_direct_IO()
978 sdio->blocks_available = in do_direct_IO()
980 sdio->next_block_for_io = in do_direct_IO()
981 map_bh->b_blocknr << sdio->blkfactor; in do_direct_IO()
989 if (!sdio->blkfactor) in do_direct_IO()
992 blkmask = (1 << sdio->blkfactor) - 1; in do_direct_IO()
993 dio_remainder = (sdio->block_in_file & blkmask); in do_direct_IO()
1007 sdio->next_block_for_io += dio_remainder; in do_direct_IO()
1008 sdio->blocks_available -= dio_remainder; in do_direct_IO()
1027 if (sdio->block_in_file >= in do_direct_IO()
1034 sdio->block_in_file++; in do_direct_IO()
1045 if (unlikely(sdio->blkfactor && !sdio->start_zero_done)) in do_direct_IO()
1046 dio_zero_block(dio, sdio, 0, map_bh); in do_direct_IO()
1052 this_chunk_blocks = sdio->blocks_available; in do_direct_IO()
1056 u = sdio->final_block_in_request - sdio->block_in_file; in do_direct_IO()
1062 if (this_chunk_blocks == sdio->blocks_available) in do_direct_IO()
1063 sdio->boundary = buffer_boundary(map_bh); in do_direct_IO()
1064 ret = submit_page_section(dio, sdio, page, in do_direct_IO()
1067 sdio->next_block_for_io, in do_direct_IO()
1073 sdio->next_block_for_io += this_chunk_blocks; in do_direct_IO()
1075 sdio->block_in_file += this_chunk_blocks; in do_direct_IO()
1078 sdio->blocks_available -= this_chunk_blocks; in do_direct_IO()
1080 BUG_ON(sdio->block_in_file > sdio->final_block_in_request); in do_direct_IO()
1081 if (sdio->block_in_file == sdio->final_block_in_request) in do_direct_IO()
1153 struct dio_submit sdio = { 0, }; in do_blockdev_direct_IO() local
1257 sdio.blkbits = blkbits; in do_blockdev_direct_IO()
1258 sdio.blkfactor = i_blkbits - blkbits; in do_blockdev_direct_IO()
1259 sdio.block_in_file = offset >> blkbits; in do_blockdev_direct_IO()
1261 sdio.get_block = get_block; in do_blockdev_direct_IO()
1263 sdio.submit_io = submit_io; in do_blockdev_direct_IO()
1264 sdio.final_block_in_bio = -1; in do_blockdev_direct_IO()
1265 sdio.next_block_for_io = -1; in do_blockdev_direct_IO()
1273 sdio.iter = iter; in do_blockdev_direct_IO()
1274 sdio.final_block_in_request = end >> blkbits; in do_blockdev_direct_IO()
1280 if (unlikely(sdio.blkfactor)) in do_blockdev_direct_IO()
1281 sdio.pages_in_io = 2; in do_blockdev_direct_IO()
1283 sdio.pages_in_io += iov_iter_npages(iter, INT_MAX); in do_blockdev_direct_IO()
1287 retval = do_direct_IO(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1289 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()
1302 dio_zero_block(dio, &sdio, 1, &map_bh); in do_blockdev_direct_IO()
1304 if (sdio.cur_page) { in do_blockdev_direct_IO()
1307 ret2 = dio_send_cur_page(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1310 put_page(sdio.cur_page); in do_blockdev_direct_IO()
1311 sdio.cur_page = NULL; in do_blockdev_direct_IO()
1313 if (sdio.bio) in do_blockdev_direct_IO()
1314 dio_bio_submit(dio, &sdio); in do_blockdev_direct_IO()
1322 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()