Lines Matching refs:dreq
102 static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
105 static inline void get_dreq(struct nfs_direct_req *dreq) in get_dreq() argument
107 atomic_inc(&dreq->io_count); in get_dreq()
110 static inline int put_dreq(struct nfs_direct_req *dreq) in put_dreq() argument
112 return atomic_dec_and_test(&dreq->io_count); in put_dreq()
116 nfs_direct_handle_truncated(struct nfs_direct_req *dreq, in nfs_direct_handle_truncated() argument
123 if (dreq->max_count >= dreq_len) { in nfs_direct_handle_truncated()
124 dreq->max_count = dreq_len; in nfs_direct_handle_truncated()
125 if (dreq->count > dreq_len) in nfs_direct_handle_truncated()
126 dreq->count = dreq_len; in nfs_direct_handle_truncated()
129 dreq->error = hdr->error; in nfs_direct_handle_truncated()
131 dreq->error = 0; in nfs_direct_handle_truncated()
136 nfs_direct_count_bytes(struct nfs_direct_req *dreq, in nfs_direct_count_bytes() argument
142 if (hdr_end > dreq->io_start) in nfs_direct_count_bytes()
143 dreq_len = hdr_end - dreq->io_start; in nfs_direct_count_bytes()
145 nfs_direct_handle_truncated(dreq, hdr, dreq_len); in nfs_direct_count_bytes()
147 if (dreq_len > dreq->max_count) in nfs_direct_count_bytes()
148 dreq_len = dreq->max_count; in nfs_direct_count_bytes()
150 if (dreq->count < dreq_len) in nfs_direct_count_bytes()
151 dreq->count = dreq_len; in nfs_direct_count_bytes()
163 nfs_direct_select_verf(struct nfs_direct_req *dreq, in nfs_direct_select_verf() argument
167 struct nfs_writeverf *verfp = &dreq->verf; in nfs_direct_select_verf()
174 if (ds_clp && dreq->ds_cinfo.nbuckets > 0) { in nfs_direct_select_verf()
175 if (commit_idx >= 0 && commit_idx < dreq->ds_cinfo.nbuckets) in nfs_direct_select_verf()
176 verfp = &dreq->ds_cinfo.buckets[commit_idx].direct_verf; in nfs_direct_select_verf()
192 static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq, in nfs_direct_set_hdr_verf() argument
197 verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx); in nfs_direct_set_hdr_verf()
218 static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq, in nfs_direct_set_or_cmp_hdr_verf() argument
223 verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx); in nfs_direct_set_or_cmp_hdr_verf()
225 nfs_direct_set_hdr_verf(dreq, hdr); in nfs_direct_set_or_cmp_hdr_verf()
239 static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq, in nfs_direct_cmp_commit_data_verf() argument
244 verfp = nfs_direct_select_verf(dreq, data->ds_clp, in nfs_direct_cmp_commit_data_verf()
287 struct nfs_direct_req *dreq) in nfs_init_cinfo_from_dreq() argument
289 cinfo->inode = dreq->inode; in nfs_init_cinfo_from_dreq()
290 cinfo->mds = &dreq->mds_cinfo; in nfs_init_cinfo_from_dreq()
291 cinfo->ds = &dreq->ds_cinfo; in nfs_init_cinfo_from_dreq()
292 cinfo->dreq = dreq; in nfs_init_cinfo_from_dreq()
298 struct nfs_direct_req *dreq; in nfs_direct_req_alloc() local
300 dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL); in nfs_direct_req_alloc()
301 if (!dreq) in nfs_direct_req_alloc()
304 kref_init(&dreq->kref); in nfs_direct_req_alloc()
305 kref_get(&dreq->kref); in nfs_direct_req_alloc()
306 init_completion(&dreq->completion); in nfs_direct_req_alloc()
307 INIT_LIST_HEAD(&dreq->mds_cinfo.list); in nfs_direct_req_alloc()
308 dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */ in nfs_direct_req_alloc()
309 INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); in nfs_direct_req_alloc()
310 spin_lock_init(&dreq->lock); in nfs_direct_req_alloc()
312 return dreq; in nfs_direct_req_alloc()
317 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref); in nfs_direct_req_free() local
319 nfs_free_pnfs_ds_cinfo(&dreq->ds_cinfo); in nfs_direct_req_free()
320 if (dreq->l_ctx != NULL) in nfs_direct_req_free()
321 nfs_put_lock_context(dreq->l_ctx); in nfs_direct_req_free()
322 if (dreq->ctx != NULL) in nfs_direct_req_free()
323 put_nfs_open_context(dreq->ctx); in nfs_direct_req_free()
324 kmem_cache_free(nfs_direct_cachep, dreq); in nfs_direct_req_free()
327 static void nfs_direct_req_release(struct nfs_direct_req *dreq) in nfs_direct_req_release() argument
329 kref_put(&dreq->kref, nfs_direct_req_free); in nfs_direct_req_release()
332 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq) in nfs_dreq_bytes_left() argument
334 return dreq->bytes_left; in nfs_dreq_bytes_left()
341 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq) in nfs_direct_wait() argument
346 if (dreq->iocb) in nfs_direct_wait()
349 result = wait_for_completion_killable(&dreq->completion); in nfs_direct_wait()
352 result = dreq->count; in nfs_direct_wait()
353 WARN_ON_ONCE(dreq->count < 0); in nfs_direct_wait()
356 result = dreq->error; in nfs_direct_wait()
366 static void nfs_direct_complete(struct nfs_direct_req *dreq) in nfs_direct_complete() argument
368 struct inode *inode = dreq->inode; in nfs_direct_complete()
372 if (dreq->iocb) { in nfs_direct_complete()
373 long res = (long) dreq->error; in nfs_direct_complete()
374 if (dreq->count != 0) { in nfs_direct_complete()
375 res = (long) dreq->count; in nfs_direct_complete()
376 WARN_ON_ONCE(dreq->count < 0); in nfs_direct_complete()
378 dreq->iocb->ki_complete(dreq->iocb, res, 0); in nfs_direct_complete()
381 complete(&dreq->completion); in nfs_direct_complete()
383 nfs_direct_req_release(dreq); in nfs_direct_complete()
389 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_read_completion() local
391 spin_lock(&dreq->lock); in nfs_direct_read_completion()
393 spin_unlock(&dreq->lock); in nfs_direct_read_completion()
397 nfs_direct_count_bytes(dreq, hdr); in nfs_direct_read_completion()
398 spin_unlock(&dreq->lock); in nfs_direct_read_completion()
405 (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY)) in nfs_direct_read_completion()
412 if (put_dreq(dreq)) in nfs_direct_read_completion()
413 nfs_direct_complete(dreq); in nfs_direct_read_completion()
430 get_dreq(hdr->dreq); in nfs_direct_pgio_init()
447 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, in nfs_direct_read_schedule_iovec() argument
452 struct inode *inode = dreq->inode; in nfs_direct_read_schedule_iovec()
457 nfs_pageio_init_read(&desc, dreq->inode, false, in nfs_direct_read_schedule_iovec()
459 get_dreq(dreq); in nfs_direct_read_schedule_iovec()
460 desc.pg_dreq = dreq; in nfs_direct_read_schedule_iovec()
481 req = nfs_create_request(dreq->ctx, pagevec[i], in nfs_direct_read_schedule_iovec()
498 dreq->bytes_left -= req_len; in nfs_direct_read_schedule_iovec()
514 nfs_direct_req_release(dreq); in nfs_direct_read_schedule_iovec()
518 if (put_dreq(dreq)) in nfs_direct_read_schedule_iovec()
519 nfs_direct_complete(dreq); in nfs_direct_read_schedule_iovec()
549 struct nfs_direct_req *dreq; in nfs_file_direct_read() local
565 dreq = nfs_direct_req_alloc(); in nfs_file_direct_read()
566 if (dreq == NULL) in nfs_file_direct_read()
569 dreq->inode = inode; in nfs_file_direct_read()
570 dreq->bytes_left = dreq->max_count = count; in nfs_file_direct_read()
571 dreq->io_start = iocb->ki_pos; in nfs_file_direct_read()
572 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); in nfs_file_direct_read()
573 l_ctx = nfs_get_lock_context(dreq->ctx); in nfs_file_direct_read()
576 nfs_direct_req_release(dreq); in nfs_file_direct_read()
579 dreq->l_ctx = l_ctx; in nfs_file_direct_read()
581 dreq->iocb = iocb; in nfs_file_direct_read()
584 dreq->flags = NFS_ODIRECT_SHOULD_DIRTY; in nfs_file_direct_read()
590 requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos); in nfs_file_direct_read()
596 result = nfs_direct_wait(dreq); in nfs_file_direct_read()
607 nfs_direct_req_release(dreq); in nfs_file_direct_read()
626 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq) in nfs_direct_write_reschedule() argument
634 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_reschedule()
635 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); in nfs_direct_write_reschedule()
637 dreq->count = 0; in nfs_direct_write_reschedule()
638 dreq->max_count = 0; in nfs_direct_write_reschedule()
640 dreq->max_count += req->wb_bytes; in nfs_direct_write_reschedule()
641 dreq->verf.committed = NFS_INVALID_STABLE_HOW; in nfs_direct_write_reschedule()
642 nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo); in nfs_direct_write_reschedule()
643 get_dreq(dreq); in nfs_direct_write_reschedule()
645 nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false, in nfs_direct_write_reschedule()
647 desc.pg_dreq = dreq; in nfs_direct_write_reschedule()
655 dreq->flags = 0; in nfs_direct_write_reschedule()
657 dreq->error = desc.pg_error; in nfs_direct_write_reschedule()
659 dreq->error = -EIO; in nfs_direct_write_reschedule()
672 if (put_dreq(dreq)) in nfs_direct_write_reschedule()
673 nfs_direct_write_complete(dreq); in nfs_direct_write_reschedule()
678 struct nfs_direct_req *dreq = data->dreq; in nfs_direct_commit_complete() local
683 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_commit_complete()
684 if (status < 0 || nfs_direct_cmp_commit_data_verf(dreq, data)) in nfs_direct_commit_complete()
685 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_commit_complete()
690 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) { in nfs_direct_commit_complete()
704 nfs_direct_write_complete(dreq); in nfs_direct_commit_complete()
710 struct nfs_direct_req *dreq = cinfo->dreq; in nfs_direct_resched_write() local
712 spin_lock(&dreq->lock); in nfs_direct_resched_write()
713 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_resched_write()
714 spin_unlock(&dreq->lock); in nfs_direct_resched_write()
723 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq) in nfs_direct_commit_schedule() argument
729 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_commit_schedule()
730 nfs_scan_commit(dreq->inode, &mds_list, &cinfo); in nfs_direct_commit_schedule()
731 res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo); in nfs_direct_commit_schedule()
733 nfs_direct_write_reschedule(dreq); in nfs_direct_commit_schedule()
738 struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work); in nfs_direct_write_schedule_work() local
739 int flags = dreq->flags; in nfs_direct_write_schedule_work()
741 dreq->flags = 0; in nfs_direct_write_schedule_work()
744 nfs_direct_commit_schedule(dreq); in nfs_direct_write_schedule_work()
747 nfs_direct_write_reschedule(dreq); in nfs_direct_write_schedule_work()
750 nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping); in nfs_direct_write_schedule_work()
751 nfs_direct_complete(dreq); in nfs_direct_write_schedule_work()
755 static void nfs_direct_write_complete(struct nfs_direct_req *dreq) in nfs_direct_write_complete() argument
757 queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */ in nfs_direct_write_complete()
762 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_write_completion() local
767 nfs_init_cinfo_from_dreq(&cinfo, dreq); in nfs_direct_write_completion()
769 spin_lock(&dreq->lock); in nfs_direct_write_completion()
771 spin_unlock(&dreq->lock); in nfs_direct_write_completion()
775 nfs_direct_count_bytes(dreq, hdr); in nfs_direct_write_completion()
778 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) in nfs_direct_write_completion()
780 else if (dreq->flags == 0) { in nfs_direct_write_completion()
781 nfs_direct_set_hdr_verf(dreq, hdr); in nfs_direct_write_completion()
783 dreq->flags = NFS_ODIRECT_DO_COMMIT; in nfs_direct_write_completion()
784 } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) { in nfs_direct_write_completion()
786 if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr)) in nfs_direct_write_completion()
787 dreq->flags = in nfs_direct_write_completion()
792 spin_unlock(&dreq->lock); in nfs_direct_write_completion()
807 if (put_dreq(dreq)) in nfs_direct_write_completion()
808 nfs_direct_write_complete(dreq); in nfs_direct_write_completion()
825 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_write_reschedule_io() local
827 spin_lock(&dreq->lock); in nfs_direct_write_reschedule_io()
828 if (dreq->error == 0) { in nfs_direct_write_reschedule_io()
829 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_write_reschedule_io()
834 spin_unlock(&dreq->lock); in nfs_direct_write_reschedule_io()
856 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, in nfs_direct_write_schedule_iovec() argument
861 struct inode *inode = dreq->inode; in nfs_direct_write_schedule_iovec()
868 desc.pg_dreq = dreq; in nfs_direct_write_schedule_iovec()
869 get_dreq(dreq); in nfs_direct_write_schedule_iovec()
891 req = nfs_create_request(dreq->ctx, pagevec[i], in nfs_direct_write_schedule_iovec()
916 dreq->bytes_left -= req_len; in nfs_direct_write_schedule_iovec()
931 nfs_direct_req_release(dreq); in nfs_direct_write_schedule_iovec()
935 if (put_dreq(dreq)) in nfs_direct_write_schedule_iovec()
936 nfs_direct_write_complete(dreq); in nfs_direct_write_schedule_iovec()
969 struct nfs_direct_req *dreq; in nfs_file_direct_write() local
992 dreq = nfs_direct_req_alloc(); in nfs_file_direct_write()
993 if (!dreq) in nfs_file_direct_write()
996 dreq->inode = inode; in nfs_file_direct_write()
997 dreq->bytes_left = dreq->max_count = count; in nfs_file_direct_write()
998 dreq->io_start = pos; in nfs_file_direct_write()
999 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); in nfs_file_direct_write()
1000 l_ctx = nfs_get_lock_context(dreq->ctx); in nfs_file_direct_write()
1003 nfs_direct_req_release(dreq); in nfs_file_direct_write()
1006 dreq->l_ctx = l_ctx; in nfs_file_direct_write()
1008 dreq->iocb = iocb; in nfs_file_direct_write()
1011 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, in nfs_file_direct_write()
1016 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, in nfs_file_direct_write()
1028 result = nfs_direct_wait(dreq); in nfs_file_direct_write()
1040 nfs_direct_req_release(dreq); in nfs_file_direct_write()