Lines Matching refs:dreq
102 static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
105 static inline void get_dreq(struct nfs_direct_req *dreq) in get_dreq() argument
107 atomic_inc(&dreq->io_count); in get_dreq()
110 static inline int put_dreq(struct nfs_direct_req *dreq) in put_dreq() argument
112 return atomic_dec_and_test(&dreq->io_count); in put_dreq()
116 nfs_direct_handle_truncated(struct nfs_direct_req *dreq, in nfs_direct_handle_truncated() argument
123 if (dreq->max_count >= dreq_len) { in nfs_direct_handle_truncated()
124 dreq->max_count = dreq_len; in nfs_direct_handle_truncated()
125 if (dreq->count > dreq_len) in nfs_direct_handle_truncated()
126 dreq->count = dreq_len; in nfs_direct_handle_truncated()
129 dreq->error = hdr->error; in nfs_direct_handle_truncated()
131 dreq->error = 0; in nfs_direct_handle_truncated()
136 nfs_direct_count_bytes(struct nfs_direct_req *dreq, in nfs_direct_count_bytes() argument
142 if (hdr_end > dreq->io_start) in nfs_direct_count_bytes()
143 dreq_len = hdr_end - dreq->io_start; in nfs_direct_count_bytes()
145 nfs_direct_handle_truncated(dreq, hdr, dreq_len); in nfs_direct_count_bytes()
147 if (dreq_len > dreq->max_count) in nfs_direct_count_bytes()
148 dreq_len = dreq->max_count; in nfs_direct_count_bytes()
150 if (dreq->count < dreq_len) in nfs_direct_count_bytes()
151 dreq->count = dreq_len; in nfs_direct_count_bytes()
187 struct nfs_direct_req *dreq) in nfs_init_cinfo_from_dreq() argument
189 cinfo->inode = dreq->inode; in nfs_init_cinfo_from_dreq()
190 cinfo->mds = &dreq->mds_cinfo; in nfs_init_cinfo_from_dreq()
191 cinfo->ds = &dreq->ds_cinfo; in nfs_init_cinfo_from_dreq()
192 cinfo->dreq = dreq; in nfs_init_cinfo_from_dreq()
198 struct nfs_direct_req *dreq; in nfs_direct_req_alloc() local
200 dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL); in nfs_direct_req_alloc()
201 if (!dreq) in nfs_direct_req_alloc()
204 kref_init(&dreq->kref); in nfs_direct_req_alloc()
205 kref_get(&dreq->kref); in nfs_direct_req_alloc()
206 init_completion(&dreq->completion); in nfs_direct_req_alloc()
207 INIT_LIST_HEAD(&dreq->mds_cinfo.list); in nfs_direct_req_alloc()
208 pnfs_init_ds_commit_info(&dreq->ds_cinfo); in nfs_direct_req_alloc()
209 INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); in nfs_direct_req_alloc()
210 spin_lock_init(&dreq->lock); in nfs_direct_req_alloc()
212 return dreq; in nfs_direct_req_alloc()
217 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref); in nfs_direct_req_free() local
219 pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode); in nfs_direct_req_free()
220 if (dreq->l_ctx != NULL) in nfs_direct_req_free()
221 nfs_put_lock_context(dreq->l_ctx); in nfs_direct_req_free()
222 if (dreq->ctx != NULL) in nfs_direct_req_free()
223 put_nfs_open_context(dreq->ctx); in nfs_direct_req_free()
224 kmem_cache_free(nfs_direct_cachep, dreq); in nfs_direct_req_free()
227 static void nfs_direct_req_release(struct nfs_direct_req *dreq) in nfs_direct_req_release() argument
229 kref_put(&dreq->kref, nfs_direct_req_free); in nfs_direct_req_release()
232 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq) in nfs_dreq_bytes_left() argument
234 return dreq->bytes_left; in nfs_dreq_bytes_left()
241 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq) in nfs_direct_wait() argument
246 if (dreq->iocb) in nfs_direct_wait()
249 result = wait_for_completion_killable(&dreq->completion); in nfs_direct_wait()
252 result = dreq->count; in nfs_direct_wait()
253 WARN_ON_ONCE(dreq->count < 0); in nfs_direct_wait()
256 result = dreq->error; in nfs_direct_wait()
266 static void nfs_direct_complete(struct nfs_direct_req *dreq) in nfs_direct_complete() argument
268 struct inode *inode = dreq->inode; in nfs_direct_complete()
272 if (dreq->iocb) { in nfs_direct_complete()
273 long res = (long) dreq->error; in nfs_direct_complete()
274 if (dreq->count != 0) { in nfs_direct_complete()
275 res = (long) dreq->count; in nfs_direct_complete()
276 WARN_ON_ONCE(dreq->count < 0); in nfs_direct_complete()
278 dreq->iocb->ki_complete(dreq->iocb, res, 0); in nfs_direct_complete()
281 complete(&dreq->completion); in nfs_direct_complete()
283 nfs_direct_req_release(dreq); in nfs_direct_complete()
289 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_read_completion() local
291 spin_lock(&dreq->lock); in nfs_direct_read_completion()
293 spin_unlock(&dreq->lock); in nfs_direct_read_completion()
297 nfs_direct_count_bytes(dreq, hdr); in nfs_direct_read_completion()
298 spin_unlock(&dreq->lock); in nfs_direct_read_completion()
305 (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY)) in nfs_direct_read_completion()
312 if (put_dreq(dreq)) in nfs_direct_read_completion()
313 nfs_direct_complete(dreq); in nfs_direct_read_completion()
330 get_dreq(hdr->dreq); in nfs_direct_pgio_init()
347 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, in nfs_direct_read_schedule_iovec() argument
352 struct inode *inode = dreq->inode; in nfs_direct_read_schedule_iovec()
357 nfs_pageio_init_read(&desc, dreq->inode, false, in nfs_direct_read_schedule_iovec()
359 get_dreq(dreq); in nfs_direct_read_schedule_iovec()
360 desc.pg_dreq = dreq; in nfs_direct_read_schedule_iovec()
381 req = nfs_create_request(dreq->ctx, pagevec[i], in nfs_direct_read_schedule_iovec()
398 dreq->bytes_left -= req_len; in nfs_direct_read_schedule_iovec()
414 nfs_direct_req_release(dreq); in nfs_direct_read_schedule_iovec()
418 if (put_dreq(dreq)) in nfs_direct_read_schedule_iovec()
419 nfs_direct_complete(dreq); in nfs_direct_read_schedule_iovec()
449 struct nfs_direct_req *dreq; in nfs_file_direct_read() local
465 dreq = nfs_direct_req_alloc(); in nfs_file_direct_read()
466 if (dreq == NULL) in nfs_file_direct_read()
469 dreq->inode = inode; in nfs_file_direct_read()
470 dreq->bytes_left = dreq->max_count = count; in nfs_file_direct_read()
471 dreq->io_start = iocb->ki_pos; in nfs_file_direct_read()
472 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); in nfs_file_direct_read()
473 l_ctx = nfs_get_lock_context(dreq->ctx); in nfs_file_direct_read()
476 nfs_direct_req_release(dreq); in nfs_file_direct_read()
479 dreq->l_ctx = l_ctx; in nfs_file_direct_read()
481 dreq->iocb = iocb; in nfs_file_direct_read()
484 dreq->flags = NFS_ODIRECT_SHOULD_DIRTY; in nfs_file_direct_read()
490 requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos); in nfs_file_direct_read()
496 result = nfs_direct_wait(dreq); in nfs_file_direct_read()
507 nfs_direct_req_release(dreq); in nfs_file_direct_read()
541 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq) in nfs_direct_write_reschedule() argument
549 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_reschedule()
550 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); in nfs_direct_write_reschedule()
552 nfs_direct_join_group(&reqs, dreq->inode); in nfs_direct_write_reschedule()
554 dreq->count = 0; in nfs_direct_write_reschedule()
555 dreq->max_count = 0; in nfs_direct_write_reschedule()
557 dreq->max_count += req->wb_bytes; in nfs_direct_write_reschedule()
558 nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo); in nfs_direct_write_reschedule()
559 get_dreq(dreq); in nfs_direct_write_reschedule()
561 nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false, in nfs_direct_write_reschedule()
563 desc.pg_dreq = dreq; in nfs_direct_write_reschedule()
571 dreq->flags = 0; in nfs_direct_write_reschedule()
573 dreq->error = desc.pg_error; in nfs_direct_write_reschedule()
575 dreq->error = -EIO; in nfs_direct_write_reschedule()
588 if (put_dreq(dreq)) in nfs_direct_write_reschedule()
589 nfs_direct_write_complete(dreq); in nfs_direct_write_reschedule()
595 struct nfs_direct_req *dreq = data->dreq; in nfs_direct_commit_complete() local
602 dreq->error = status; in nfs_direct_commit_complete()
603 dreq->max_count = 0; in nfs_direct_commit_complete()
604 dreq->count = 0; in nfs_direct_commit_complete()
605 dreq->flags = NFS_ODIRECT_DONE; in nfs_direct_commit_complete()
606 } else if (dreq->flags == NFS_ODIRECT_DONE) in nfs_direct_commit_complete()
607 status = dreq->error; in nfs_direct_commit_complete()
609 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_commit_complete()
615 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_commit_complete()
628 nfs_direct_write_complete(dreq); in nfs_direct_commit_complete()
634 struct nfs_direct_req *dreq = cinfo->dreq; in nfs_direct_resched_write() local
636 spin_lock(&dreq->lock); in nfs_direct_resched_write()
637 if (dreq->flags != NFS_ODIRECT_DONE) in nfs_direct_resched_write()
638 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_resched_write()
639 spin_unlock(&dreq->lock); in nfs_direct_resched_write()
648 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq) in nfs_direct_commit_schedule() argument
654 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_commit_schedule()
655 nfs_scan_commit(dreq->inode, &mds_list, &cinfo); in nfs_direct_commit_schedule()
656 res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo); in nfs_direct_commit_schedule()
658 nfs_direct_write_reschedule(dreq); in nfs_direct_commit_schedule()
661 static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq) in nfs_direct_write_clear_reqs() argument
667 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_clear_reqs()
668 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); in nfs_direct_write_clear_reqs()
680 struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work); in nfs_direct_write_schedule_work() local
681 int flags = dreq->flags; in nfs_direct_write_schedule_work()
683 dreq->flags = 0; in nfs_direct_write_schedule_work()
686 nfs_direct_commit_schedule(dreq); in nfs_direct_write_schedule_work()
689 nfs_direct_write_reschedule(dreq); in nfs_direct_write_schedule_work()
692 nfs_direct_write_clear_reqs(dreq); in nfs_direct_write_schedule_work()
693 nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping); in nfs_direct_write_schedule_work()
694 nfs_direct_complete(dreq); in nfs_direct_write_schedule_work()
698 static void nfs_direct_write_complete(struct nfs_direct_req *dreq) in nfs_direct_write_complete() argument
700 queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */ in nfs_direct_write_complete()
705 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_write_completion() local
710 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_completion()
712 spin_lock(&dreq->lock); in nfs_direct_write_completion()
714 spin_unlock(&dreq->lock); in nfs_direct_write_completion()
718 nfs_direct_count_bytes(dreq, hdr); in nfs_direct_write_completion()
720 if (!dreq->flags) in nfs_direct_write_completion()
721 dreq->flags = NFS_ODIRECT_DO_COMMIT; in nfs_direct_write_completion()
722 flags = dreq->flags; in nfs_direct_write_completion()
724 spin_unlock(&dreq->lock); in nfs_direct_write_completion()
744 if (put_dreq(dreq)) in nfs_direct_write_completion()
745 nfs_direct_write_complete(dreq); in nfs_direct_write_completion()
762 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_write_reschedule_io() local
764 spin_lock(&dreq->lock); in nfs_direct_write_reschedule_io()
765 if (dreq->error == 0) { in nfs_direct_write_reschedule_io()
766 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_write_reschedule_io()
772 spin_unlock(&dreq->lock); in nfs_direct_write_reschedule_io()
794 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, in nfs_direct_write_schedule_iovec() argument
799 struct inode *inode = dreq->inode; in nfs_direct_write_schedule_iovec()
806 desc.pg_dreq = dreq; in nfs_direct_write_schedule_iovec()
807 get_dreq(dreq); in nfs_direct_write_schedule_iovec()
829 req = nfs_create_request(dreq->ctx, pagevec[i], in nfs_direct_write_schedule_iovec()
854 dreq->bytes_left -= req_len; in nfs_direct_write_schedule_iovec()
869 nfs_direct_req_release(dreq); in nfs_direct_write_schedule_iovec()
873 if (put_dreq(dreq)) in nfs_direct_write_schedule_iovec()
874 nfs_direct_write_complete(dreq); in nfs_direct_write_schedule_iovec()
907 struct nfs_direct_req *dreq; in nfs_file_direct_write() local
930 dreq = nfs_direct_req_alloc(); in nfs_file_direct_write()
931 if (!dreq) in nfs_file_direct_write()
934 dreq->inode = inode; in nfs_file_direct_write()
935 dreq->bytes_left = dreq->max_count = count; in nfs_file_direct_write()
936 dreq->io_start = pos; in nfs_file_direct_write()
937 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); in nfs_file_direct_write()
938 l_ctx = nfs_get_lock_context(dreq->ctx); in nfs_file_direct_write()
941 nfs_direct_req_release(dreq); in nfs_file_direct_write()
944 dreq->l_ctx = l_ctx; in nfs_file_direct_write()
946 dreq->iocb = iocb; in nfs_file_direct_write()
947 pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode); in nfs_file_direct_write()
950 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, in nfs_file_direct_write()
955 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, in nfs_file_direct_write()
967 result = nfs_direct_wait(dreq); in nfs_file_direct_write()
979 nfs_direct_req_release(dreq); in nfs_file_direct_write()