Lines Matching full:sdio
161 static inline unsigned dio_pages_present(struct dio_submit *sdio) in dio_pages_present() argument
163 return sdio->tail - sdio->head; in dio_pages_present()
169 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) in dio_refill_pages() argument
173 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES, in dio_refill_pages()
174 &sdio->from); in dio_refill_pages()
176 if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) { in dio_refill_pages()
187 sdio->head = 0; in dio_refill_pages()
188 sdio->tail = 1; in dio_refill_pages()
189 sdio->from = 0; in dio_refill_pages()
190 sdio->to = PAGE_SIZE; in dio_refill_pages()
195 iov_iter_advance(sdio->iter, ret); in dio_refill_pages()
196 ret += sdio->from; in dio_refill_pages()
197 sdio->head = 0; in dio_refill_pages()
198 sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE; in dio_refill_pages()
199 sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1; in dio_refill_pages()
212 struct dio_submit *sdio) in dio_get_page() argument
214 if (dio_pages_present(sdio) == 0) { in dio_get_page()
217 ret = dio_refill_pages(dio, sdio); in dio_get_page()
220 BUG_ON(dio_pages_present(sdio) == 0); in dio_get_page()
222 return dio->pages[sdio->head]; in dio_get_page()
390 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, in dio_bio_alloc() argument
412 sdio->bio = bio; in dio_bio_alloc()
413 sdio->logical_offset_in_bio = sdio->cur_page_fs_offset; in dio_bio_alloc()
423 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) in dio_bio_submit() argument
425 struct bio *bio = sdio->bio; in dio_bio_submit()
439 if (sdio->submit_io) { in dio_bio_submit()
440 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); in dio_bio_submit()
445 sdio->bio = NULL; in dio_bio_submit()
446 sdio->boundary = 0; in dio_bio_submit()
447 sdio->logical_offset_in_bio = 0; in dio_bio_submit()
453 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) in dio_cleanup() argument
455 while (sdio->head < sdio->tail) in dio_cleanup()
456 put_page(dio->pages[sdio->head++]); in dio_cleanup()
545 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio) in dio_bio_reap() argument
549 if (sdio->reap_counter++ >= 64) { in dio_bio_reap()
563 sdio->reap_counter = 0; in dio_bio_reap()
606 * of available blocks at sdio->blocks_available. These are in units of the
627 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, in get_more_blocks() argument
635 unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor; in get_more_blocks()
644 BUG_ON(sdio->block_in_file >= sdio->final_block_in_request); in get_more_blocks()
645 fs_startblk = sdio->block_in_file >> sdio->blkfactor; in get_more_blocks()
646 fs_endblk = (sdio->final_block_in_request - 1) >> in get_more_blocks()
647 sdio->blkfactor; in get_more_blocks()
671 ret = (*sdio->get_block)(dio->inode, fs_startblk, in get_more_blocks()
686 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, in dio_new_bio() argument
692 ret = dio_bio_reap(dio, sdio); in dio_new_bio()
695 sector = start_sector << (sdio->blkbits - 9); in dio_new_bio()
696 nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES); in dio_new_bio()
698 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); in dio_new_bio()
699 sdio->boundary = 0; in dio_new_bio()
711 static inline int dio_bio_add_page(struct dio_submit *sdio) in dio_bio_add_page() argument
715 ret = bio_add_page(sdio->bio, sdio->cur_page, in dio_bio_add_page()
716 sdio->cur_page_len, sdio->cur_page_offset); in dio_bio_add_page()
717 if (ret == sdio->cur_page_len) { in dio_bio_add_page()
721 if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE) in dio_bio_add_page()
722 sdio->pages_in_io--; in dio_bio_add_page()
723 get_page(sdio->cur_page); in dio_bio_add_page()
724 sdio->final_block_in_bio = sdio->cur_page_block + in dio_bio_add_page()
725 (sdio->cur_page_len >> sdio->blkbits); in dio_bio_add_page()
743 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio, in dio_send_cur_page() argument
748 if (sdio->bio) { in dio_send_cur_page()
749 loff_t cur_offset = sdio->cur_page_fs_offset; in dio_send_cur_page()
750 loff_t bio_next_offset = sdio->logical_offset_in_bio + in dio_send_cur_page()
751 sdio->bio->bi_iter.bi_size; in dio_send_cur_page()
767 if (sdio->final_block_in_bio != sdio->cur_page_block || in dio_send_cur_page()
769 dio_bio_submit(dio, sdio); in dio_send_cur_page()
772 if (sdio->bio == NULL) { in dio_send_cur_page()
773 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
778 if (dio_bio_add_page(sdio) != 0) { in dio_send_cur_page()
779 dio_bio_submit(dio, sdio); in dio_send_cur_page()
780 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
782 ret = dio_bio_add_page(sdio); in dio_send_cur_page()
808 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, in submit_page_section() argument
813 int boundary = sdio->boundary; /* dio_send_cur_page may clear it */ in submit_page_section()
825 if (sdio->cur_page == page && in submit_page_section()
826 sdio->cur_page_offset + sdio->cur_page_len == offset && in submit_page_section()
827 sdio->cur_page_block + in submit_page_section()
828 (sdio->cur_page_len >> sdio->blkbits) == blocknr) { in submit_page_section()
829 sdio->cur_page_len += len; in submit_page_section()
836 if (sdio->cur_page) { in submit_page_section()
837 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
838 put_page(sdio->cur_page); in submit_page_section()
839 sdio->cur_page = NULL; in submit_page_section()
845 sdio->cur_page = page; in submit_page_section()
846 sdio->cur_page_offset = offset; in submit_page_section()
847 sdio->cur_page_len = len; in submit_page_section()
848 sdio->cur_page_block = blocknr; in submit_page_section()
849 sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits; in submit_page_section()
856 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
857 if (sdio->bio) in submit_page_section()
858 dio_bio_submit(dio, sdio); in submit_page_section()
859 put_page(sdio->cur_page); in submit_page_section()
860 sdio->cur_page = NULL; in submit_page_section()
874 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio, in dio_zero_block() argument
882 sdio->start_zero_done = 1; in dio_zero_block()
883 if (!sdio->blkfactor || !buffer_new(map_bh)) in dio_zero_block()
886 dio_blocks_per_fs_block = 1 << sdio->blkfactor; in dio_zero_block()
887 this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1); in dio_zero_block()
899 this_chunk_bytes = this_chunk_blocks << sdio->blkbits; in dio_zero_block()
902 if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes, in dio_zero_block()
903 sdio->next_block_for_io, map_bh)) in dio_zero_block()
906 sdio->next_block_for_io += this_chunk_blocks; in dio_zero_block()
925 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, in do_direct_IO() argument
928 const unsigned blkbits = sdio->blkbits; in do_direct_IO()
929 const unsigned i_blkbits = blkbits + sdio->blkfactor; in do_direct_IO()
932 while (sdio->block_in_file < sdio->final_block_in_request) { in do_direct_IO()
936 page = dio_get_page(dio, sdio); in do_direct_IO()
941 from = sdio->head ? 0 : sdio->from; in do_direct_IO()
942 to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE; in do_direct_IO()
943 sdio->head++; in do_direct_IO()
950 if (sdio->blocks_available == 0) { in do_direct_IO()
957 ret = get_more_blocks(dio, sdio, map_bh); in do_direct_IO()
965 sdio->blocks_available = in do_direct_IO()
967 sdio->next_block_for_io = in do_direct_IO()
968 map_bh->b_blocknr << sdio->blkfactor; in do_direct_IO()
976 if (!sdio->blkfactor) in do_direct_IO()
979 blkmask = (1 << sdio->blkfactor) - 1; in do_direct_IO()
980 dio_remainder = (sdio->block_in_file & blkmask); in do_direct_IO()
994 sdio->next_block_for_io += dio_remainder; in do_direct_IO()
995 sdio->blocks_available -= dio_remainder; in do_direct_IO()
1014 if (sdio->block_in_file >= in do_direct_IO()
1021 sdio->block_in_file++; in do_direct_IO()
1032 if (unlikely(sdio->blkfactor && !sdio->start_zero_done)) in do_direct_IO()
1033 dio_zero_block(dio, sdio, 0, map_bh); in do_direct_IO()
1039 this_chunk_blocks = sdio->blocks_available; in do_direct_IO()
1043 u = sdio->final_block_in_request - sdio->block_in_file; in do_direct_IO()
1049 if (this_chunk_blocks == sdio->blocks_available) in do_direct_IO()
1050 sdio->boundary = buffer_boundary(map_bh); in do_direct_IO()
1051 ret = submit_page_section(dio, sdio, page, in do_direct_IO()
1054 sdio->next_block_for_io, in do_direct_IO()
1060 sdio->next_block_for_io += this_chunk_blocks; in do_direct_IO()
1062 sdio->block_in_file += this_chunk_blocks; in do_direct_IO()
1065 sdio->blocks_available -= this_chunk_blocks; in do_direct_IO()
1067 BUG_ON(sdio->block_in_file > sdio->final_block_in_request); in do_direct_IO()
1068 if (sdio->block_in_file == sdio->final_block_in_request) in do_direct_IO()
1121 * NOTE: if you pass "sdio" to anything by pointer make sure that function
1140 struct dio_submit sdio = { 0, }; in do_blockdev_direct_IO() local
1244 sdio.blkbits = blkbits; in do_blockdev_direct_IO()
1245 sdio.blkfactor = i_blkbits - blkbits; in do_blockdev_direct_IO()
1246 sdio.block_in_file = offset >> blkbits; in do_blockdev_direct_IO()
1248 sdio.get_block = get_block; in do_blockdev_direct_IO()
1250 sdio.submit_io = submit_io; in do_blockdev_direct_IO()
1251 sdio.final_block_in_bio = -1; in do_blockdev_direct_IO()
1252 sdio.next_block_for_io = -1; in do_blockdev_direct_IO()
1260 sdio.iter = iter; in do_blockdev_direct_IO()
1261 sdio.final_block_in_request = end >> blkbits; in do_blockdev_direct_IO()
1267 if (unlikely(sdio.blkfactor)) in do_blockdev_direct_IO()
1268 sdio.pages_in_io = 2; in do_blockdev_direct_IO()
1270 sdio.pages_in_io += iov_iter_npages(iter, INT_MAX); in do_blockdev_direct_IO()
1274 retval = do_direct_IO(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1276 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()
1289 dio_zero_block(dio, &sdio, 1, &map_bh); in do_blockdev_direct_IO()
1291 if (sdio.cur_page) { in do_blockdev_direct_IO()
1294 ret2 = dio_send_cur_page(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1297 put_page(sdio.cur_page); in do_blockdev_direct_IO()
1298 sdio.cur_page = NULL; in do_blockdev_direct_IO()
1300 if (sdio.bio) in do_blockdev_direct_IO()
1301 dio_bio_submit(dio, &sdio); in do_blockdev_direct_IO()
1309 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()