102 static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
105 static inline void get_dreq(struct nfs_direct_req *dreq)  in get_dreq()  argument
107 	atomic_inc(&dreq->io_count);  in get_dreq()
110 static inline int put_dreq(struct nfs_direct_req *dreq)  in put_dreq()  argument
112 	return atomic_dec_and_test(&dreq->io_count);  in put_dreq()
116 nfs_direct_handle_truncated(struct nfs_direct_req *dreq,  in nfs_direct_handle_truncated()  argument
123 	if (dreq->max_count >= dreq_len) {  in nfs_direct_handle_truncated()
124 		dreq->max_count = dreq_len;  in nfs_direct_handle_truncated()
125 		if (dreq->count > dreq_len)  in nfs_direct_handle_truncated()
126 			dreq->count = dreq_len;  in nfs_direct_handle_truncated()
129 			dreq->error = hdr->error;  in nfs_direct_handle_truncated()
131 			dreq->error = 0;  in nfs_direct_handle_truncated()
136 nfs_direct_count_bytes(struct nfs_direct_req *dreq,  in nfs_direct_count_bytes()  argument
142 	if (hdr_end > dreq->io_start)  in nfs_direct_count_bytes()
143 		dreq_len = hdr_end - dreq->io_start;  in nfs_direct_count_bytes()
145 	nfs_direct_handle_truncated(dreq, hdr, dreq_len);  in nfs_direct_count_bytes()
147 	if (dreq_len > dreq->max_count)  in nfs_direct_count_bytes()
148 		dreq_len = dreq->max_count;  in nfs_direct_count_bytes()
150 	if (dreq->count < dreq_len)  in nfs_direct_count_bytes()
151 		dreq->count = dreq_len;  in nfs_direct_count_bytes()
187 			      struct nfs_direct_req *dreq)  in nfs_init_cinfo_from_dreq()  argument
189 	cinfo->inode = dreq->inode;  in nfs_init_cinfo_from_dreq()
190 	cinfo->mds = &dreq->mds_cinfo;  in nfs_init_cinfo_from_dreq()
191 	cinfo->ds = &dreq->ds_cinfo;  in nfs_init_cinfo_from_dreq()
192 	cinfo->dreq = dreq;  in nfs_init_cinfo_from_dreq()
198 	struct nfs_direct_req *dreq;  in nfs_direct_req_alloc()  local
200 	dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);  in nfs_direct_req_alloc()
201 	if (!dreq)  in nfs_direct_req_alloc()
204 	kref_init(&dreq->kref);  in nfs_direct_req_alloc()
205 	kref_get(&dreq->kref);  in nfs_direct_req_alloc()
206 	init_completion(&dreq->completion);  in nfs_direct_req_alloc()
207 	INIT_LIST_HEAD(&dreq->mds_cinfo.list);  in nfs_direct_req_alloc()
208 	pnfs_init_ds_commit_info(&dreq->ds_cinfo);  in nfs_direct_req_alloc()
209 	INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);  in nfs_direct_req_alloc()
210 	spin_lock_init(&dreq->lock);  in nfs_direct_req_alloc()
212 	return dreq;  in nfs_direct_req_alloc()
217 	struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);  in nfs_direct_req_free()  local
219 	pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode);  in nfs_direct_req_free()
220 	if (dreq->l_ctx != NULL)  in nfs_direct_req_free()
221 		nfs_put_lock_context(dreq->l_ctx);  in nfs_direct_req_free()
222 	if (dreq->ctx != NULL)  in nfs_direct_req_free()
223 		put_nfs_open_context(dreq->ctx);  in nfs_direct_req_free()
224 	kmem_cache_free(nfs_direct_cachep, dreq);  in nfs_direct_req_free()
227 static void nfs_direct_req_release(struct nfs_direct_req *dreq)  in nfs_direct_req_release()  argument
229 	kref_put(&dreq->kref, nfs_direct_req_free);  in nfs_direct_req_release()
232 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)  in nfs_dreq_bytes_left()  argument
234 	return dreq->bytes_left;  in nfs_dreq_bytes_left()
241 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)  in nfs_direct_wait()  argument
246 	if (dreq->iocb)  in nfs_direct_wait()
249 	result = wait_for_completion_killable(&dreq->completion);  in nfs_direct_wait()
252 		result = dreq->count;  in nfs_direct_wait()
253 		WARN_ON_ONCE(dreq->count < 0);  in nfs_direct_wait()
256 		result = dreq->error;  in nfs_direct_wait()
266 static void nfs_direct_complete(struct nfs_direct_req *dreq)  in nfs_direct_complete()  argument
268 	struct inode *inode = dreq->inode;  in nfs_direct_complete()
272 	if (dreq->iocb) {  in nfs_direct_complete()
273 		long res = (long) dreq->error;  in nfs_direct_complete()
274 		if (dreq->count != 0) {  in nfs_direct_complete()
275 			res = (long) dreq->count;  in nfs_direct_complete()
276 			WARN_ON_ONCE(dreq->count < 0);  in nfs_direct_complete()
278 		dreq->iocb->ki_complete(dreq->iocb, res, 0);  in nfs_direct_complete()
281 	complete(&dreq->completion);  in nfs_direct_complete()
283 	nfs_direct_req_release(dreq);  in nfs_direct_complete()
289 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_read_completion()  local
291 	spin_lock(&dreq->lock);  in nfs_direct_read_completion()
293 		spin_unlock(&dreq->lock);  in nfs_direct_read_completion()
297 	nfs_direct_count_bytes(dreq, hdr);  in nfs_direct_read_completion()
298 	spin_unlock(&dreq->lock);  in nfs_direct_read_completion()
305 		    (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY))  in nfs_direct_read_completion()
312 	if (put_dreq(dreq))  in nfs_direct_read_completion()
313 		nfs_direct_complete(dreq);  in nfs_direct_read_completion()
330 	get_dreq(hdr->dreq);  in nfs_direct_pgio_init()
347 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,  in nfs_direct_read_schedule_iovec()  argument
352 	struct inode *inode = dreq->inode;  in nfs_direct_read_schedule_iovec()
357 	nfs_pageio_init_read(&desc, dreq->inode, false,  in nfs_direct_read_schedule_iovec()
359 	get_dreq(dreq);  in nfs_direct_read_schedule_iovec()
360 	desc.pg_dreq = dreq;  in nfs_direct_read_schedule_iovec()
381 			req = nfs_create_request(dreq->ctx, pagevec[i],  in nfs_direct_read_schedule_iovec()
398 			dreq->bytes_left -= req_len;  in nfs_direct_read_schedule_iovec()
414 		nfs_direct_req_release(dreq);  in nfs_direct_read_schedule_iovec()
418 	if (put_dreq(dreq))  in nfs_direct_read_schedule_iovec()
419 		nfs_direct_complete(dreq);  in nfs_direct_read_schedule_iovec()
449 	struct nfs_direct_req *dreq;  in nfs_file_direct_read()  local
465 	dreq = nfs_direct_req_alloc();  in nfs_file_direct_read()
466 	if (dreq == NULL)  in nfs_file_direct_read()
469 	dreq->inode = inode;  in nfs_file_direct_read()
470 	dreq->bytes_left = dreq->max_count = count;  in nfs_file_direct_read()
471 	dreq->io_start = iocb->ki_pos;  in nfs_file_direct_read()
472 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));  in nfs_file_direct_read()
473 	l_ctx = nfs_get_lock_context(dreq->ctx);  in nfs_file_direct_read()
476 		nfs_direct_req_release(dreq);  in nfs_file_direct_read()
479 	dreq->l_ctx = l_ctx;  in nfs_file_direct_read()
481 		dreq->iocb = iocb;  in nfs_file_direct_read()
484 		dreq->flags = NFS_ODIRECT_SHOULD_DIRTY;  in nfs_file_direct_read()
490 	requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);  in nfs_file_direct_read()
496 		result = nfs_direct_wait(dreq);  in nfs_file_direct_read()
507 	nfs_direct_req_release(dreq);  in nfs_file_direct_read()
567 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)  in nfs_direct_write_reschedule()  argument
575 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_reschedule()
576 	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);  in nfs_direct_write_reschedule()
578 	nfs_direct_join_group(&reqs, &cinfo, dreq->inode);  in nfs_direct_write_reschedule()
580 	dreq->count = 0;  in nfs_direct_write_reschedule()
581 	dreq->max_count = 0;  in nfs_direct_write_reschedule()
583 		dreq->max_count += req->wb_bytes;  in nfs_direct_write_reschedule()
584 	nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo);  in nfs_direct_write_reschedule()
585 	get_dreq(dreq);  in nfs_direct_write_reschedule()
587 	nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,  in nfs_direct_write_reschedule()
589 	desc.pg_dreq = dreq;  in nfs_direct_write_reschedule()
597 			dreq->flags = 0;  in nfs_direct_write_reschedule()
599 				dreq->error = desc.pg_error;  in nfs_direct_write_reschedule()
601 				dreq->error = -EIO;  in nfs_direct_write_reschedule()
614 	if (put_dreq(dreq))  in nfs_direct_write_reschedule()
615 		nfs_direct_write_complete(dreq);  in nfs_direct_write_reschedule()
621 	struct nfs_direct_req *dreq = data->dreq;  in nfs_direct_commit_complete()  local
628 		dreq->error = status;  in nfs_direct_commit_complete()
629 		dreq->max_count = 0;  in nfs_direct_commit_complete()
630 		dreq->count = 0;  in nfs_direct_commit_complete()
631 		dreq->flags = NFS_ODIRECT_DONE;  in nfs_direct_commit_complete()
632 	} else if (dreq->flags == NFS_ODIRECT_DONE)  in nfs_direct_commit_complete()
633 		status = dreq->error;  in nfs_direct_commit_complete()
635 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_commit_complete()
641 			dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_commit_complete()
654 		nfs_direct_write_complete(dreq);  in nfs_direct_commit_complete()
660 	struct nfs_direct_req *dreq = cinfo->dreq;  in nfs_direct_resched_write()  local
662 	spin_lock(&dreq->lock);  in nfs_direct_resched_write()
663 	if (dreq->flags != NFS_ODIRECT_DONE)  in nfs_direct_resched_write()
664 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_resched_write()
665 	spin_unlock(&dreq->lock);  in nfs_direct_resched_write()
674 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)  in nfs_direct_commit_schedule()  argument
680 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_commit_schedule()
681 	nfs_scan_commit(dreq->inode, &mds_list, &cinfo);  in nfs_direct_commit_schedule()
682 	res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);  in nfs_direct_commit_schedule()
684 		nfs_direct_write_reschedule(dreq);  in nfs_direct_commit_schedule()
687 static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq)  in nfs_direct_write_clear_reqs()  argument
693 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_clear_reqs()
694 	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);  in nfs_direct_write_clear_reqs()
706 	struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);  in nfs_direct_write_schedule_work()  local
707 	int flags = dreq->flags;  in nfs_direct_write_schedule_work()
709 	dreq->flags = 0;  in nfs_direct_write_schedule_work()
712 			nfs_direct_commit_schedule(dreq);  in nfs_direct_write_schedule_work()
715 			nfs_direct_write_reschedule(dreq);  in nfs_direct_write_schedule_work()
718 			nfs_direct_write_clear_reqs(dreq);  in nfs_direct_write_schedule_work()
719 			nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping);  in nfs_direct_write_schedule_work()
720 			nfs_direct_complete(dreq);  in nfs_direct_write_schedule_work()
724 static void nfs_direct_write_complete(struct nfs_direct_req *dreq)  in nfs_direct_write_complete()  argument
726 	queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */  in nfs_direct_write_complete()
731 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_write_completion()  local
736 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_completion()
738 	spin_lock(&dreq->lock);  in nfs_direct_write_completion()
740 		spin_unlock(&dreq->lock);  in nfs_direct_write_completion()
744 	nfs_direct_count_bytes(dreq, hdr);  in nfs_direct_write_completion()
746 		if (!dreq->flags)  in nfs_direct_write_completion()
747 			dreq->flags = NFS_ODIRECT_DO_COMMIT;  in nfs_direct_write_completion()
748 		flags = dreq->flags;  in nfs_direct_write_completion()
750 	spin_unlock(&dreq->lock);  in nfs_direct_write_completion()
770 	if (put_dreq(dreq))  in nfs_direct_write_completion()
771 		nfs_direct_write_complete(dreq);  in nfs_direct_write_completion()
788 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_write_reschedule_io()  local
792 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_reschedule_io()
793 	spin_lock(&dreq->lock);  in nfs_direct_write_reschedule_io()
794 	if (dreq->error == 0)  in nfs_direct_write_reschedule_io()
795 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_write_reschedule_io()
797 	spin_unlock(&dreq->lock);  in nfs_direct_write_reschedule_io()
825 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,  in nfs_direct_write_schedule_iovec()  argument
830 	struct inode *inode = dreq->inode;  in nfs_direct_write_schedule_iovec()
837 	desc.pg_dreq = dreq;  in nfs_direct_write_schedule_iovec()
838 	get_dreq(dreq);  in nfs_direct_write_schedule_iovec()
860 			req = nfs_create_request(dreq->ctx, pagevec[i],  in nfs_direct_write_schedule_iovec()
885 			dreq->bytes_left -= req_len;  in nfs_direct_write_schedule_iovec()
900 		nfs_direct_req_release(dreq);  in nfs_direct_write_schedule_iovec()
904 	if (put_dreq(dreq))  in nfs_direct_write_schedule_iovec()
905 		nfs_direct_write_complete(dreq);  in nfs_direct_write_schedule_iovec()
938 	struct nfs_direct_req *dreq;  in nfs_file_direct_write()  local
961 	dreq = nfs_direct_req_alloc();  in nfs_file_direct_write()
962 	if (!dreq)  in nfs_file_direct_write()
965 	dreq->inode = inode;  in nfs_file_direct_write()
966 	dreq->bytes_left = dreq->max_count = count;  in nfs_file_direct_write()
967 	dreq->io_start = pos;  in nfs_file_direct_write()
968 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));  in nfs_file_direct_write()
969 	l_ctx = nfs_get_lock_context(dreq->ctx);  in nfs_file_direct_write()
972 		nfs_direct_req_release(dreq);  in nfs_file_direct_write()
975 	dreq->l_ctx = l_ctx;  in nfs_file_direct_write()
977 		dreq->iocb = iocb;  in nfs_file_direct_write()
978 	pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode);  in nfs_file_direct_write()
981 		requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,  in nfs_file_direct_write()
986 		requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,  in nfs_file_direct_write()
998 		result = nfs_direct_wait(dreq);  in nfs_file_direct_write()
1010 	nfs_direct_req_release(dreq);  in nfs_file_direct_write()