Lines Matching refs:sdio
160 static inline unsigned dio_pages_present(struct dio_submit *sdio) in dio_pages_present() argument
162 return sdio->tail - sdio->head; in dio_pages_present()
168 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) in dio_refill_pages() argument
172 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES, in dio_refill_pages()
173 &sdio->from); in dio_refill_pages()
175 if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) { in dio_refill_pages()
186 sdio->head = 0; in dio_refill_pages()
187 sdio->tail = 1; in dio_refill_pages()
188 sdio->from = 0; in dio_refill_pages()
189 sdio->to = PAGE_SIZE; in dio_refill_pages()
194 iov_iter_advance(sdio->iter, ret); in dio_refill_pages()
195 ret += sdio->from; in dio_refill_pages()
196 sdio->head = 0; in dio_refill_pages()
197 sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE; in dio_refill_pages()
198 sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1; in dio_refill_pages()
211 struct dio_submit *sdio) in dio_get_page() argument
213 if (dio_pages_present(sdio) == 0) { in dio_get_page()
216 ret = dio_refill_pages(dio, sdio); in dio_get_page()
219 BUG_ON(dio_pages_present(sdio) == 0); in dio_get_page()
221 return dio->pages[sdio->head]; in dio_get_page()
429 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, in dio_bio_alloc() argument
443 sdio->cur_page_fs_offset >> inode->i_blkbits, in dio_bio_alloc()
455 sdio->bio = bio; in dio_bio_alloc()
456 sdio->logical_offset_in_bio = sdio->cur_page_fs_offset; in dio_bio_alloc()
466 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) in dio_bio_submit() argument
468 struct bio *bio = sdio->bio; in dio_bio_submit()
482 if (sdio->submit_io) { in dio_bio_submit()
483 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); in dio_bio_submit()
488 sdio->bio = NULL; in dio_bio_submit()
489 sdio->boundary = 0; in dio_bio_submit()
490 sdio->logical_offset_in_bio = 0; in dio_bio_submit()
496 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) in dio_cleanup() argument
498 while (sdio->head < sdio->tail) in dio_cleanup()
499 put_page(dio->pages[sdio->head++]); in dio_cleanup()
588 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio) in dio_bio_reap() argument
592 if (sdio->reap_counter++ >= 64) { in dio_bio_reap()
606 sdio->reap_counter = 0; in dio_bio_reap()
670 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, in get_more_blocks() argument
678 unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor; in get_more_blocks()
687 BUG_ON(sdio->block_in_file >= sdio->final_block_in_request); in get_more_blocks()
688 fs_startblk = sdio->block_in_file >> sdio->blkfactor; in get_more_blocks()
689 fs_endblk = (sdio->final_block_in_request - 1) >> in get_more_blocks()
690 sdio->blkfactor; in get_more_blocks()
714 ret = (*sdio->get_block)(dio->inode, fs_startblk, in get_more_blocks()
729 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, in dio_new_bio() argument
735 ret = dio_bio_reap(dio, sdio); in dio_new_bio()
738 sector = start_sector << (sdio->blkbits - 9); in dio_new_bio()
739 nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES); in dio_new_bio()
741 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); in dio_new_bio()
742 sdio->boundary = 0; in dio_new_bio()
754 static inline int dio_bio_add_page(struct dio_submit *sdio) in dio_bio_add_page() argument
758 ret = bio_add_page(sdio->bio, sdio->cur_page, in dio_bio_add_page()
759 sdio->cur_page_len, sdio->cur_page_offset); in dio_bio_add_page()
760 if (ret == sdio->cur_page_len) { in dio_bio_add_page()
764 if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE) in dio_bio_add_page()
765 sdio->pages_in_io--; in dio_bio_add_page()
766 get_page(sdio->cur_page); in dio_bio_add_page()
767 sdio->final_block_in_bio = sdio->cur_page_block + in dio_bio_add_page()
768 (sdio->cur_page_len >> sdio->blkbits); in dio_bio_add_page()
786 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio, in dio_send_cur_page() argument
791 if (sdio->bio) { in dio_send_cur_page()
792 loff_t cur_offset = sdio->cur_page_fs_offset; in dio_send_cur_page()
793 loff_t bio_next_offset = sdio->logical_offset_in_bio + in dio_send_cur_page()
794 sdio->bio->bi_iter.bi_size; in dio_send_cur_page()
810 if (sdio->final_block_in_bio != sdio->cur_page_block || in dio_send_cur_page()
812 dio_bio_submit(dio, sdio); in dio_send_cur_page()
815 if (sdio->bio == NULL) { in dio_send_cur_page()
816 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
821 if (dio_bio_add_page(sdio) != 0) { in dio_send_cur_page()
822 dio_bio_submit(dio, sdio); in dio_send_cur_page()
823 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
825 ret = dio_bio_add_page(sdio); in dio_send_cur_page()
851 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, in submit_page_section() argument
867 if (sdio->cur_page == page && in submit_page_section()
868 sdio->cur_page_offset + sdio->cur_page_len == offset && in submit_page_section()
869 sdio->cur_page_block + in submit_page_section()
870 (sdio->cur_page_len >> sdio->blkbits) == blocknr) { in submit_page_section()
871 sdio->cur_page_len += len; in submit_page_section()
878 if (sdio->cur_page) { in submit_page_section()
879 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
880 put_page(sdio->cur_page); in submit_page_section()
881 sdio->cur_page = NULL; in submit_page_section()
887 sdio->cur_page = page; in submit_page_section()
888 sdio->cur_page_offset = offset; in submit_page_section()
889 sdio->cur_page_len = len; in submit_page_section()
890 sdio->cur_page_block = blocknr; in submit_page_section()
891 sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits; in submit_page_section()
897 if (sdio->boundary) { in submit_page_section()
898 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
899 if (sdio->bio) in submit_page_section()
900 dio_bio_submit(dio, sdio); in submit_page_section()
901 put_page(sdio->cur_page); in submit_page_section()
902 sdio->cur_page = NULL; in submit_page_section()
916 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio, in dio_zero_block() argument
924 sdio->start_zero_done = 1; in dio_zero_block()
925 if (!sdio->blkfactor || !buffer_new(map_bh)) in dio_zero_block()
928 dio_blocks_per_fs_block = 1 << sdio->blkfactor; in dio_zero_block()
929 this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1); in dio_zero_block()
941 this_chunk_bytes = this_chunk_blocks << sdio->blkbits; in dio_zero_block()
944 if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes, in dio_zero_block()
945 sdio->next_block_for_io, map_bh)) in dio_zero_block()
948 sdio->next_block_for_io += this_chunk_blocks; in dio_zero_block()
967 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, in do_direct_IO() argument
970 const unsigned blkbits = sdio->blkbits; in do_direct_IO()
971 const unsigned i_blkbits = blkbits + sdio->blkfactor; in do_direct_IO()
974 while (sdio->block_in_file < sdio->final_block_in_request) { in do_direct_IO()
978 page = dio_get_page(dio, sdio); in do_direct_IO()
983 from = sdio->head ? 0 : sdio->from; in do_direct_IO()
984 to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE; in do_direct_IO()
985 sdio->head++; in do_direct_IO()
992 if (sdio->blocks_available == 0) { in do_direct_IO()
999 ret = get_more_blocks(dio, sdio, map_bh); in do_direct_IO()
1007 sdio->blocks_available = in do_direct_IO()
1009 sdio->next_block_for_io = in do_direct_IO()
1010 map_bh->b_blocknr << sdio->blkfactor; in do_direct_IO()
1018 if (!sdio->blkfactor) in do_direct_IO()
1021 blkmask = (1 << sdio->blkfactor) - 1; in do_direct_IO()
1022 dio_remainder = (sdio->block_in_file & blkmask); in do_direct_IO()
1036 sdio->next_block_for_io += dio_remainder; in do_direct_IO()
1037 sdio->blocks_available -= dio_remainder; in do_direct_IO()
1056 if (sdio->block_in_file >= in do_direct_IO()
1063 sdio->block_in_file++; in do_direct_IO()
1074 if (unlikely(sdio->blkfactor && !sdio->start_zero_done)) in do_direct_IO()
1075 dio_zero_block(dio, sdio, 0, map_bh); in do_direct_IO()
1081 this_chunk_blocks = sdio->blocks_available; in do_direct_IO()
1085 u = sdio->final_block_in_request - sdio->block_in_file; in do_direct_IO()
1091 if (this_chunk_blocks == sdio->blocks_available) in do_direct_IO()
1092 sdio->boundary = buffer_boundary(map_bh); in do_direct_IO()
1093 ret = submit_page_section(dio, sdio, page, in do_direct_IO()
1096 sdio->next_block_for_io, in do_direct_IO()
1102 sdio->next_block_for_io += this_chunk_blocks; in do_direct_IO()
1104 sdio->block_in_file += this_chunk_blocks; in do_direct_IO()
1107 sdio->blocks_available -= this_chunk_blocks; in do_direct_IO()
1109 BUG_ON(sdio->block_in_file > sdio->final_block_in_request); in do_direct_IO()
1110 if (sdio->block_in_file == sdio->final_block_in_request) in do_direct_IO()
1182 struct dio_submit sdio = { 0, }; in do_blockdev_direct_IO() local
1301 sdio.blkbits = blkbits; in do_blockdev_direct_IO()
1302 sdio.blkfactor = i_blkbits - blkbits; in do_blockdev_direct_IO()
1303 sdio.block_in_file = offset >> blkbits; in do_blockdev_direct_IO()
1305 sdio.get_block = get_block; in do_blockdev_direct_IO()
1307 sdio.submit_io = submit_io; in do_blockdev_direct_IO()
1308 sdio.final_block_in_bio = -1; in do_blockdev_direct_IO()
1309 sdio.next_block_for_io = -1; in do_blockdev_direct_IO()
1317 sdio.iter = iter; in do_blockdev_direct_IO()
1318 sdio.final_block_in_request = end >> blkbits; in do_blockdev_direct_IO()
1324 if (unlikely(sdio.blkfactor)) in do_blockdev_direct_IO()
1325 sdio.pages_in_io = 2; in do_blockdev_direct_IO()
1327 sdio.pages_in_io += iov_iter_npages(iter, INT_MAX); in do_blockdev_direct_IO()
1331 retval = do_direct_IO(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1333 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()
1346 dio_zero_block(dio, &sdio, 1, &map_bh); in do_blockdev_direct_IO()
1348 if (sdio.cur_page) { in do_blockdev_direct_IO()
1351 ret2 = dio_send_cur_page(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1354 put_page(sdio.cur_page); in do_blockdev_direct_IO()
1355 sdio.cur_page = NULL; in do_blockdev_direct_IO()
1357 if (sdio.bio) in do_blockdev_direct_IO()
1358 dio_bio_submit(dio, &sdio); in do_blockdev_direct_IO()
1366 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()