Lines Matching +full:data +full:- +full:mirror
36 &desc->pg_mirrors[desc->pg_mirror_idx] : in nfs_pgio_current_mirror()
37 &desc->pg_mirrors[0]; in nfs_pgio_current_mirror()
45 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_pgheader_init() local
48 hdr->req = nfs_list_entry(mirror->pg_list.next); in nfs_pgheader_init()
49 hdr->inode = desc->pg_inode; in nfs_pgheader_init()
50 hdr->cred = hdr->req->wb_context->cred; in nfs_pgheader_init()
51 hdr->io_start = req_offset(hdr->req); in nfs_pgheader_init()
52 hdr->good_bytes = mirror->pg_count; in nfs_pgheader_init()
53 hdr->io_completion = desc->pg_io_completion; in nfs_pgheader_init()
54 hdr->dreq = desc->pg_dreq; in nfs_pgheader_init()
55 hdr->release = release; in nfs_pgheader_init()
56 hdr->completion_ops = desc->pg_completion_ops; in nfs_pgheader_init()
57 if (hdr->completion_ops->init_hdr) in nfs_pgheader_init()
58 hdr->completion_ops->init_hdr(hdr); in nfs_pgheader_init()
60 hdr->pgio_mirror_idx = desc->pg_mirror_idx; in nfs_pgheader_init()
66 spin_lock(&hdr->lock); in nfs_set_pgio_error()
67 if (!test_and_set_bit(NFS_IOHDR_ERROR, &hdr->flags) in nfs_set_pgio_error()
68 || pos < hdr->io_start + hdr->good_bytes) { in nfs_set_pgio_error()
69 clear_bit(NFS_IOHDR_EOF, &hdr->flags); in nfs_set_pgio_error()
70 hdr->good_bytes = pos - hdr->io_start; in nfs_set_pgio_error()
71 hdr->error = error; in nfs_set_pgio_error()
73 spin_unlock(&hdr->lock); in nfs_set_pgio_error()
81 INIT_LIST_HEAD(&p->wb_list); in nfs_page_alloc()
92 * nfs_iocounter_wait - wait for i/o to complete
95 * returns -ERESTARTSYS if interrupted by a fatal signal.
101 return wait_var_event_killable(&l_ctx->io_count, in nfs_iocounter_wait()
102 !atomic_read(&l_ctx->io_count)); in nfs_iocounter_wait()
106 * nfs_async_iocounter_wait - wait on a rpc_waitqueue for I/O
117 struct inode *inode = d_inode(l_ctx->open_context->dentry); in nfs_async_iocounter_wait()
120 if (atomic_read(&l_ctx->io_count) > 0) { in nfs_async_iocounter_wait()
121 rpc_sleep_on(&NFS_SERVER(inode)->uoc_rpcwaitq, task, NULL); in nfs_async_iocounter_wait()
125 if (atomic_read(&l_ctx->io_count) == 0) { in nfs_async_iocounter_wait()
126 rpc_wake_up_queued_task(&NFS_SERVER(inode)->uoc_rpcwaitq, task); in nfs_async_iocounter_wait()
135 * nfs_page_set_headlock - set the request PG_HEADLOCK
138 * this lock must be held when modifying req->wb_head
145 if (!test_and_set_bit(PG_HEADLOCK, &req->wb_flags)) in nfs_page_set_headlock()
148 set_bit(PG_CONTENDED1, &req->wb_flags); in nfs_page_set_headlock()
150 return wait_on_bit_lock(&req->wb_flags, PG_HEADLOCK, in nfs_page_set_headlock()
155 * nfs_page_clear_headlock - clear the request PG_HEADLOCK
162 clear_bit(PG_HEADLOCK, &req->wb_flags); in nfs_page_clear_headlock()
164 if (!test_bit(PG_CONTENDED1, &req->wb_flags)) in nfs_page_clear_headlock()
166 wake_up_bit(&req->wb_flags, PG_HEADLOCK); in nfs_page_clear_headlock()
170 * nfs_page_group_lock - lock the head of the page group
184 if (ret || req->wb_head == req) in nfs_page_group_lock()
186 return nfs_page_set_headlock(req->wb_head); in nfs_page_group_lock()
190 * nfs_page_group_unlock - unlock the head of the page group
196 if (req != req->wb_head) in nfs_page_group_unlock()
197 nfs_page_clear_headlock(req->wb_head); in nfs_page_group_unlock()
209 struct nfs_page *head = req->wb_head; in nfs_page_group_sync_on_bit_locked()
212 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags)); in nfs_page_group_sync_on_bit_locked()
213 WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags)); in nfs_page_group_sync_on_bit_locked()
215 tmp = req->wb_this_page; in nfs_page_group_sync_on_bit_locked()
217 if (!test_bit(bit, &tmp->wb_flags)) in nfs_page_group_sync_on_bit_locked()
219 tmp = tmp->wb_this_page; in nfs_page_group_sync_on_bit_locked()
225 clear_bit(bit, &tmp->wb_flags); in nfs_page_group_sync_on_bit_locked()
226 tmp = tmp->wb_this_page; in nfs_page_group_sync_on_bit_locked()
233 * nfs_page_group_sync_on_bit - set bit on current request, but only
235 * @req - request in page group
236 * @bit - PG_* bit that is used to sync page group
250 * nfs_page_group_init - Initialize the page group linkage for @req
251 * @req - a new nfs request
252 * @prev - the previous request in page group, or NULL if @req is the first
263 req->wb_head = req; in nfs_page_group_init()
264 req->wb_this_page = req; in nfs_page_group_init()
267 WARN_ON_ONCE(prev->wb_this_page != prev->wb_head); in nfs_page_group_init()
268 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags)); in nfs_page_group_init()
269 req->wb_head = prev->wb_head; in nfs_page_group_init()
270 req->wb_this_page = prev->wb_this_page; in nfs_page_group_init()
271 prev->wb_this_page = req; in nfs_page_group_init()
275 kref_get(&req->wb_head->wb_kref); in nfs_page_group_init()
280 if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) { in nfs_page_group_init()
281 inode = page_file_mapping(req->wb_page)->host; in nfs_page_group_init()
282 set_bit(PG_INODE_REF, &req->wb_flags); in nfs_page_group_init()
283 kref_get(&req->wb_kref); in nfs_page_group_init()
284 atomic_long_inc(&NFS_I(inode)->nrequests); in nfs_page_group_init()
290 * nfs_page_group_destroy - sync the destruction of page groups
291 * @req - request that no longer needs the page group
300 struct nfs_page *head = req->wb_head; in nfs_page_group_destroy()
308 next = tmp->wb_this_page; in nfs_page_group_destroy()
310 tmp->wb_this_page = tmp; in nfs_page_group_destroy()
311 tmp->wb_head = tmp; in nfs_page_group_destroy()
322 * nfs_create_request - Create an NFS read/write request.
341 if (test_bit(NFS_CONTEXT_BAD, &ctx->flags)) in nfs_create_request()
342 return ERR_PTR(-EBADF); in nfs_create_request()
346 return ERR_PTR(-ENOMEM); in nfs_create_request()
354 req->wb_lock_context = l_ctx; in nfs_create_request()
355 atomic_inc(&l_ctx->io_count); in nfs_create_request()
358 * long write-back delay. This will be adjusted in in nfs_create_request()
360 req->wb_page = page; in nfs_create_request()
362 req->wb_index = page_index(page); in nfs_create_request()
365 req->wb_offset = offset; in nfs_create_request()
366 req->wb_pgbase = offset; in nfs_create_request()
367 req->wb_bytes = count; in nfs_create_request()
368 req->wb_context = get_nfs_open_context(ctx); in nfs_create_request()
369 kref_init(&req->wb_kref); in nfs_create_request()
375 * nfs_unlock_request - Unlock request and wake up sleepers.
385 clear_bit(PG_BUSY, &req->wb_flags); in nfs_unlock_request()
387 if (!test_bit(PG_CONTENDED2, &req->wb_flags)) in nfs_unlock_request()
389 wake_up_bit(&req->wb_flags, PG_BUSY); in nfs_unlock_request()
393 * nfs_unlock_and_release_request - Unlock request and release the nfs_page
403 * nfs_clear_request - Free up all resources allocated to the request
411 struct page *page = req->wb_page; in nfs_clear_request()
412 struct nfs_open_context *ctx = req->wb_context; in nfs_clear_request()
413 struct nfs_lock_context *l_ctx = req->wb_lock_context; in nfs_clear_request()
417 req->wb_page = NULL; in nfs_clear_request()
420 if (atomic_dec_and_test(&l_ctx->io_count)) { in nfs_clear_request()
421 wake_up_var(&l_ctx->io_count); in nfs_clear_request()
422 if (test_bit(NFS_CONTEXT_UNLOCK, &ctx->flags)) in nfs_clear_request()
423 rpc_wake_up(&NFS_SERVER(d_inode(ctx->dentry))->uoc_rpcwaitq); in nfs_clear_request()
426 req->wb_lock_context = NULL; in nfs_clear_request()
430 req->wb_context = NULL; in nfs_clear_request()
435 * nfs_release_request - Release the count on an NFS read/write request
442 WARN_ON_ONCE(req->wb_this_page != req); in nfs_free_request()
445 WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags)); in nfs_free_request()
446 WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags)); in nfs_free_request()
447 WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags)); in nfs_free_request()
448 WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags)); in nfs_free_request()
449 WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags)); in nfs_free_request()
458 kref_put(&req->wb_kref, nfs_page_group_destroy); in nfs_release_request()
463 * nfs_wait_on_request - Wait for a request to complete.
472 if (!test_bit(PG_BUSY, &req->wb_flags)) in nfs_wait_on_request()
474 set_bit(PG_CONTENDED2, &req->wb_flags); in nfs_wait_on_request()
476 return wait_on_bit_io(&req->wb_flags, PG_BUSY, in nfs_wait_on_request()
482 * nfs_generic_pg_test - determine if requests can be coalesced
493 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_generic_pg_test() local
496 if (mirror->pg_count > mirror->pg_bsize) { in nfs_generic_pg_test()
506 if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) * in nfs_generic_pg_test()
510 return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes); in nfs_generic_pg_test()
516 struct nfs_pgio_header *hdr = ops->rw_alloc_header(); in nfs_pgio_header_alloc()
519 INIT_LIST_HEAD(&hdr->pages); in nfs_pgio_header_alloc()
520 spin_lock_init(&hdr->lock); in nfs_pgio_header_alloc()
521 hdr->rw_ops = ops; in nfs_pgio_header_alloc()
528 * nfs_pgio_data_destroy - make @hdr suitable for reuse
537 if (hdr->args.context) in nfs_pgio_data_destroy()
538 put_nfs_open_context(hdr->args.context); in nfs_pgio_data_destroy()
539 if (hdr->page_array.pagevec != hdr->page_array.page_array) in nfs_pgio_data_destroy()
540 kfree(hdr->page_array.pagevec); in nfs_pgio_data_destroy()
544 * nfs_pgio_header_free - Free a read or write header
550 hdr->rw_ops->rw_free_header(hdr); in nfs_pgio_header_free()
555 * nfs_pgio_rpcsetup - Set up arguments for a pageio call
559 * @how: How to commit data (writes only)
566 struct nfs_page *req = hdr->req; in nfs_pgio_rpcsetup()
569 * NB: take care not to mess about with hdr->commit et al. */ in nfs_pgio_rpcsetup()
571 hdr->args.fh = NFS_FH(hdr->inode); in nfs_pgio_rpcsetup()
572 hdr->args.offset = req_offset(req); in nfs_pgio_rpcsetup()
574 hdr->mds_offset = hdr->args.offset; in nfs_pgio_rpcsetup()
575 hdr->args.pgbase = req->wb_pgbase; in nfs_pgio_rpcsetup()
576 hdr->args.pages = hdr->page_array.pagevec; in nfs_pgio_rpcsetup()
577 hdr->args.count = count; in nfs_pgio_rpcsetup()
578 hdr->args.context = get_nfs_open_context(req->wb_context); in nfs_pgio_rpcsetup()
579 hdr->args.lock_context = req->wb_lock_context; in nfs_pgio_rpcsetup()
580 hdr->args.stable = NFS_UNSTABLE; in nfs_pgio_rpcsetup()
589 hdr->args.stable = NFS_FILE_SYNC; in nfs_pgio_rpcsetup()
592 hdr->res.fattr = &hdr->fattr; in nfs_pgio_rpcsetup()
593 hdr->res.count = 0; in nfs_pgio_rpcsetup()
594 hdr->res.eof = 0; in nfs_pgio_rpcsetup()
595 hdr->res.verf = &hdr->verf; in nfs_pgio_rpcsetup()
596 nfs_fattr_init(&hdr->fattr); in nfs_pgio_rpcsetup()
600 * nfs_pgio_prepare - Prepare pageio hdr to go over the wire
608 err = NFS_PROTO(hdr->inode)->pgio_rpc_prepare(task, hdr); in nfs_pgio_prepare()
619 .rpc_argp = &hdr->args, in nfs_initiate_pgio()
620 .rpc_resp = &hdr->res, in nfs_initiate_pgio()
625 .task = &hdr->task, in nfs_initiate_pgio()
634 hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how); in nfs_initiate_pgio()
638 hdr->inode->i_sb->s_id, in nfs_initiate_pgio()
639 (unsigned long long)NFS_FILEID(hdr->inode), in nfs_initiate_pgio()
640 hdr->args.count, in nfs_initiate_pgio()
641 (unsigned long long)hdr->args.offset); in nfs_initiate_pgio()
651 ret = task->tk_status; in nfs_initiate_pgio()
660 * nfs_pgio_error - Clean up from a pageio error
666 set_bit(NFS_IOHDR_REDO, &hdr->flags); in nfs_pgio_error()
667 hdr->completion_ops->completion(hdr); in nfs_pgio_error()
671 * nfs_pgio_release - Release pageio data
677 hdr->completion_ops->completion(hdr); in nfs_pgio_release()
680 static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror, in nfs_pageio_mirror_init() argument
683 INIT_LIST_HEAD(&mirror->pg_list); in nfs_pageio_mirror_init()
684 mirror->pg_bytes_written = 0; in nfs_pageio_mirror_init()
685 mirror->pg_count = 0; in nfs_pageio_mirror_init()
686 mirror->pg_bsize = bsize; in nfs_pageio_mirror_init()
687 mirror->pg_base = 0; in nfs_pageio_mirror_init()
688 mirror->pg_recoalesce = 0; in nfs_pageio_mirror_init()
692 * nfs_pageio_init - initialise a page io descriptor
709 desc->pg_moreio = 0; in nfs_pageio_init()
710 desc->pg_inode = inode; in nfs_pageio_init()
711 desc->pg_ops = pg_ops; in nfs_pageio_init()
712 desc->pg_completion_ops = compl_ops; in nfs_pageio_init()
713 desc->pg_rw_ops = rw_ops; in nfs_pageio_init()
714 desc->pg_ioflags = io_flags; in nfs_pageio_init()
715 desc->pg_error = 0; in nfs_pageio_init()
716 desc->pg_lseg = NULL; in nfs_pageio_init()
717 desc->pg_io_completion = NULL; in nfs_pageio_init()
718 desc->pg_dreq = NULL; in nfs_pageio_init()
719 desc->pg_bsize = bsize; in nfs_pageio_init()
721 desc->pg_mirror_count = 1; in nfs_pageio_init()
722 desc->pg_mirror_idx = 0; in nfs_pageio_init()
724 desc->pg_mirrors_dynamic = NULL; in nfs_pageio_init()
725 desc->pg_mirrors = desc->pg_mirrors_static; in nfs_pageio_init()
726 nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize); in nfs_pageio_init()
730 * nfs_pgio_result - Basic pageio error handling
737 struct inode *inode = hdr->inode; in nfs_pgio_result()
740 task->tk_pid, task->tk_status); in nfs_pgio_result()
742 if (hdr->rw_ops->rw_done(task, hdr, inode) != 0) in nfs_pgio_result()
744 if (task->tk_status < 0) in nfs_pgio_result()
745 nfs_set_pgio_error(hdr, task->tk_status, hdr->args.offset); in nfs_pgio_result()
747 hdr->rw_ops->rw_result(task, hdr); in nfs_pgio_result()
761 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_generic_pgio() local
766 struct list_head *head = &mirror->pg_list; in nfs_generic_pgio()
768 struct nfs_page_array *pg_array = &hdr->page_array; in nfs_generic_pgio()
772 pagecount = nfs_page_array_len(mirror->pg_base, mirror->pg_count); in nfs_generic_pgio()
773 pg_array->npages = pagecount; in nfs_generic_pgio()
775 if (pagecount <= ARRAY_SIZE(pg_array->page_array)) in nfs_generic_pgio()
776 pg_array->pagevec = pg_array->page_array; in nfs_generic_pgio()
778 if (hdr->rw_mode == FMODE_WRITE) in nfs_generic_pgio()
780 pg_array->pagevec = kcalloc(pagecount, sizeof(struct page *), gfp_flags); in nfs_generic_pgio()
781 if (!pg_array->pagevec) { in nfs_generic_pgio()
782 pg_array->npages = 0; in nfs_generic_pgio()
784 desc->pg_error = -ENOMEM; in nfs_generic_pgio()
785 return desc->pg_error; in nfs_generic_pgio()
789 nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq); in nfs_generic_pgio()
790 pages = hdr->page_array.pagevec; in nfs_generic_pgio()
794 req = nfs_list_entry(head->next); in nfs_generic_pgio()
795 nfs_list_move_request(req, &hdr->pages); in nfs_generic_pgio()
797 if (!last_page || last_page != req->wb_page) { in nfs_generic_pgio()
801 *pages++ = last_page = req->wb_page; in nfs_generic_pgio()
806 desc->pg_error = -EINVAL; in nfs_generic_pgio()
807 return desc->pg_error; in nfs_generic_pgio()
810 if ((desc->pg_ioflags & FLUSH_COND_STABLE) && in nfs_generic_pgio()
811 (desc->pg_moreio || nfs_reqs_to_commit(&cinfo))) in nfs_generic_pgio()
812 desc->pg_ioflags &= ~FLUSH_COND_STABLE; in nfs_generic_pgio()
815 nfs_pgio_rpcsetup(hdr, mirror->pg_count, desc->pg_ioflags, &cinfo); in nfs_generic_pgio()
816 desc->pg_rpc_callops = &nfs_pgio_common_ops; in nfs_generic_pgio()
826 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); in nfs_generic_pg_pgios()
828 desc->pg_error = -ENOMEM; in nfs_generic_pg_pgios()
829 return desc->pg_error; in nfs_generic_pg_pgios()
834 ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode), in nfs_generic_pg_pgios()
836 hdr->cred, in nfs_generic_pg_pgios()
837 NFS_PROTO(hdr->inode), in nfs_generic_pg_pgios()
838 desc->pg_rpc_callops, in nfs_generic_pg_pgios()
839 desc->pg_ioflags, 0); in nfs_generic_pg_pgios()
850 kfree(desc->pg_mirrors_dynamic); in nfs_pageio_alloc_mirrors()
851 desc->pg_mirrors_dynamic = NULL; in nfs_pageio_alloc_mirrors()
853 return desc->pg_mirrors_static; in nfs_pageio_alloc_mirrors()
857 nfs_pageio_mirror_init(&ret[i], desc->pg_bsize); in nfs_pageio_alloc_mirrors()
858 desc->pg_mirrors_dynamic = ret; in nfs_pageio_alloc_mirrors()
864 * nfs_pageio_setup_mirroring - determine if mirroring is to be used
872 if (pgio->pg_ops->pg_get_mirror_count) in nfs_pageio_setup_mirroring()
873 mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req); in nfs_pageio_setup_mirroring()
874 if (mirror_count == pgio->pg_mirror_count || pgio->pg_error < 0) in nfs_pageio_setup_mirroring()
878 pgio->pg_error = -EINVAL; in nfs_pageio_setup_mirroring()
882 pgio->pg_mirrors = nfs_pageio_alloc_mirrors(pgio, mirror_count); in nfs_pageio_setup_mirroring()
883 if (pgio->pg_mirrors == NULL) { in nfs_pageio_setup_mirroring()
884 pgio->pg_error = -ENOMEM; in nfs_pageio_setup_mirroring()
885 pgio->pg_mirrors = pgio->pg_mirrors_static; in nfs_pageio_setup_mirroring()
888 pgio->pg_mirror_count = mirror_count; in nfs_pageio_setup_mirroring()
893 pgio->pg_mirror_count = 1; in nfs_pageio_cleanup_mirroring()
894 pgio->pg_mirror_idx = 0; in nfs_pageio_cleanup_mirroring()
895 pgio->pg_mirrors = pgio->pg_mirrors_static; in nfs_pageio_cleanup_mirroring()
896 kfree(pgio->pg_mirrors_dynamic); in nfs_pageio_cleanup_mirroring()
897 pgio->pg_mirrors_dynamic = NULL; in nfs_pageio_cleanup_mirroring()
903 return l1->lockowner == l2->lockowner; in nfs_match_lock_context()
907 * nfs_can_coalesce_requests - test two requests for compatibility
912 * page data area they describe is contiguous, and that their RPC
925 if (!nfs_match_open_context(req->wb_context, prev->wb_context)) in nfs_can_coalesce_requests()
927 flctx = d_inode(req->wb_context->dentry)->i_flctx; in nfs_can_coalesce_requests()
929 !(list_empty_careful(&flctx->flc_posix) && in nfs_can_coalesce_requests()
930 list_empty_careful(&flctx->flc_flock)) && in nfs_can_coalesce_requests()
931 !nfs_match_lock_context(req->wb_lock_context, in nfs_can_coalesce_requests()
932 prev->wb_lock_context)) in nfs_can_coalesce_requests()
934 if (req_offset(req) != req_offset(prev) + prev->wb_bytes) in nfs_can_coalesce_requests()
936 if (req->wb_page == prev->wb_page) { in nfs_can_coalesce_requests()
937 if (req->wb_pgbase != prev->wb_pgbase + prev->wb_bytes) in nfs_can_coalesce_requests()
940 if (req->wb_pgbase != 0 || in nfs_can_coalesce_requests()
941 prev->wb_pgbase + prev->wb_bytes != PAGE_SIZE) in nfs_can_coalesce_requests()
945 size = pgio->pg_ops->pg_test(pgio, prev, req); in nfs_can_coalesce_requests()
946 WARN_ON_ONCE(size > req->wb_bytes); in nfs_can_coalesce_requests()
947 if (size && size < req->wb_bytes) in nfs_can_coalesce_requests()
948 req->wb_bytes = size; in nfs_can_coalesce_requests()
953 * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
963 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_pageio_do_add_request() local
967 if (mirror->pg_count != 0) { in nfs_pageio_do_add_request()
968 prev = nfs_list_entry(mirror->pg_list.prev); in nfs_pageio_do_add_request()
970 if (desc->pg_ops->pg_init) in nfs_pageio_do_add_request()
971 desc->pg_ops->pg_init(desc, req); in nfs_pageio_do_add_request()
972 if (desc->pg_error < 0) in nfs_pageio_do_add_request()
974 mirror->pg_base = req->wb_pgbase; in nfs_pageio_do_add_request()
978 nfs_list_move_request(req, &mirror->pg_list); in nfs_pageio_do_add_request()
979 mirror->pg_count += req->wb_bytes; in nfs_pageio_do_add_request()
988 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_pageio_doio() local
991 if (!list_empty(&mirror->pg_list)) { in nfs_pageio_doio()
992 int error = desc->pg_ops->pg_doio(desc); in nfs_pageio_doio()
994 desc->pg_error = error; in nfs_pageio_doio()
996 mirror->pg_bytes_written += mirror->pg_count; in nfs_pageio_doio()
998 if (list_empty(&mirror->pg_list)) { in nfs_pageio_doio()
999 mirror->pg_count = 0; in nfs_pageio_doio()
1000 mirror->pg_base = 0; in nfs_pageio_doio()
1011 desc->pg_completion_ops->error_cleanup(&head, desc->pg_error); in nfs_pageio_cleanup_request()
1015 * nfs_pageio_add_request - Attempt to coalesce a request into a page list.
1028 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in __nfs_pageio_add_request() local
1037 bytes_left = subreq->wb_bytes; in __nfs_pageio_add_request()
1038 offset = subreq->wb_offset; in __nfs_pageio_add_request()
1039 pgbase = subreq->wb_pgbase; in __nfs_pageio_add_request()
1044 WARN_ON_ONCE(subreq->wb_bytes != bytes_left); in __nfs_pageio_add_request()
1045 WARN_ON_ONCE(subreq->wb_offset != offset); in __nfs_pageio_add_request()
1046 WARN_ON_ONCE(subreq->wb_pgbase != pgbase); in __nfs_pageio_add_request()
1049 desc->pg_moreio = 1; in __nfs_pageio_add_request()
1051 if (desc->pg_error < 0 || mirror->pg_recoalesce) in __nfs_pageio_add_request()
1059 WARN_ON_ONCE(subreq->wb_bytes + subreq->wb_pgbase > PAGE_SIZE); in __nfs_pageio_add_request()
1060 WARN_ON_ONCE(subreq->wb_bytes > bytes_left); in __nfs_pageio_add_request()
1061 WARN_ON_ONCE(subreq->wb_bytes == 0); in __nfs_pageio_add_request()
1063 bytes_left -= subreq->wb_bytes; in __nfs_pageio_add_request()
1064 offset += subreq->wb_bytes; in __nfs_pageio_add_request()
1065 pgbase += subreq->wb_bytes; in __nfs_pageio_add_request()
1068 subreq = nfs_create_request(req->wb_context, in __nfs_pageio_add_request()
1069 req->wb_page, in __nfs_pageio_add_request()
1074 subreq->wb_offset = offset; in __nfs_pageio_add_request()
1075 subreq->wb_index = req->wb_index; in __nfs_pageio_add_request()
1082 desc->pg_error = PTR_ERR(subreq); in __nfs_pageio_add_request()
1093 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_do_recoalesce() local
1097 list_splice_init(&mirror->pg_list, &head); in nfs_do_recoalesce()
1098 mirror->pg_bytes_written -= mirror->pg_count; in nfs_do_recoalesce()
1099 mirror->pg_count = 0; in nfs_do_recoalesce()
1100 mirror->pg_base = 0; in nfs_do_recoalesce()
1101 mirror->pg_recoalesce = 0; in nfs_do_recoalesce()
1109 if (desc->pg_error < 0) { in nfs_do_recoalesce()
1110 list_splice_tail(&head, &mirror->pg_list); in nfs_do_recoalesce()
1111 mirror->pg_recoalesce = 1; in nfs_do_recoalesce()
1116 } while (mirror->pg_recoalesce); in nfs_do_recoalesce()
1129 if (desc->pg_error < 0) in nfs_pageio_add_request_mirror()
1140 struct nfs_pgio_mirror *mirror; in nfs_pageio_error_cleanup() local
1142 if (!desc->pg_error) in nfs_pageio_error_cleanup()
1145 for (midx = 0; midx < desc->pg_mirror_count; midx++) { in nfs_pageio_error_cleanup()
1146 mirror = &desc->pg_mirrors[midx]; in nfs_pageio_error_cleanup()
1147 desc->pg_completion_ops->error_cleanup(&mirror->pg_list, in nfs_pageio_error_cleanup()
1148 desc->pg_error); in nfs_pageio_error_cleanup()
1159 pgbase = req->wb_pgbase; in nfs_pageio_add_request()
1160 offset = req->wb_offset; in nfs_pageio_add_request()
1161 bytes = req->wb_bytes; in nfs_pageio_add_request()
1164 if (desc->pg_error < 0) in nfs_pageio_add_request()
1167 for (midx = 0; midx < desc->pg_mirror_count; midx++) { in nfs_pageio_add_request()
1172 for (lastreq = req->wb_head; in nfs_pageio_add_request()
1173 lastreq->wb_this_page != req->wb_head; in nfs_pageio_add_request()
1174 lastreq = lastreq->wb_this_page) in nfs_pageio_add_request()
1177 dupreq = nfs_create_request(req->wb_context, in nfs_pageio_add_request()
1178 req->wb_page, lastreq, pgbase, bytes); in nfs_pageio_add_request()
1182 desc->pg_error = PTR_ERR(dupreq); in nfs_pageio_add_request()
1188 dupreq->wb_offset = offset; in nfs_pageio_add_request()
1189 dupreq->wb_index = req->wb_index; in nfs_pageio_add_request()
1194 desc->pg_mirror_idx = midx; in nfs_pageio_add_request()
1206 if (nfs_error_is_fatal(desc->pg_error)) in nfs_pageio_add_request()
1207 nfs_context_set_write_error(req->wb_context, in nfs_pageio_add_request()
1208 desc->pg_error); in nfs_pageio_add_request()
1214 * nfs_pageio_complete_mirror - Complete I/O on the current mirror of an
1217 * @mirror_idx: pointer to mirror index
1222 struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[mirror_idx]; in nfs_pageio_complete_mirror() local
1223 u32 restore_idx = desc->pg_mirror_idx; in nfs_pageio_complete_mirror()
1226 desc->pg_mirror_idx = mirror_idx; in nfs_pageio_complete_mirror()
1229 if (desc->pg_error < 0 || !mirror->pg_recoalesce) in nfs_pageio_complete_mirror()
1234 desc->pg_mirror_idx = restore_idx; in nfs_pageio_complete_mirror()
1238 * nfs_pageio_resend - Transfer requests to new descriptor and resend
1239 * @hdr - the pgio header to move request from
1240 * @desc - the pageio descriptor to add requests to
1252 desc->pg_io_completion = hdr->io_completion; in nfs_pageio_resend()
1253 desc->pg_dreq = hdr->dreq; in nfs_pageio_resend()
1254 list_splice_init(&hdr->pages, &pages); in nfs_pageio_resend()
1263 int err = desc->pg_error < 0 ? desc->pg_error : -EIO; in nfs_pageio_resend()
1264 hdr->completion_ops->error_cleanup(&pages, err); in nfs_pageio_resend()
1265 nfs_set_pgio_error(hdr, err, hdr->io_start); in nfs_pageio_resend()
1273 * nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor
1280 for (midx = 0; midx < desc->pg_mirror_count; midx++) in nfs_pageio_complete()
1283 if (desc->pg_error < 0) in nfs_pageio_complete()
1285 if (desc->pg_ops->pg_cleanup) in nfs_pageio_complete()
1286 desc->pg_ops->pg_cleanup(desc); in nfs_pageio_complete()
1291 * nfs_pageio_cond_complete - Conditional I/O completion
1296 * on non-contiguous ranges of pages as that might deadlock. This
1303 struct nfs_pgio_mirror *mirror; in nfs_pageio_cond_complete() local
1307 for (midx = 0; midx < desc->pg_mirror_count; midx++) { in nfs_pageio_cond_complete()
1308 mirror = &desc->pg_mirrors[midx]; in nfs_pageio_cond_complete()
1309 if (!list_empty(&mirror->pg_list)) { in nfs_pageio_cond_complete()
1310 prev = nfs_list_entry(mirror->pg_list.prev); in nfs_pageio_cond_complete()
1311 if (index != prev->wb_index + 1) { in nfs_pageio_cond_complete()
1320 * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1)
1334 return -ENOMEM; in nfs_init_nfspagecache()