Lines Matching full:lo
57 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo);
58 static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo,
264 pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo) in pnfs_get_layout_hdr() argument
266 refcount_inc(&lo->plh_refcount); in pnfs_get_layout_hdr()
277 pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) in pnfs_free_layout_hdr() argument
279 struct nfs_server *server = NFS_SERVER(lo->plh_inode); in pnfs_free_layout_hdr()
282 if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { in pnfs_free_layout_hdr()
286 list_del_rcu(&lo->plh_layouts); in pnfs_free_layout_hdr()
289 put_cred(lo->plh_lc_cred); in pnfs_free_layout_hdr()
290 return ld->free_layout_hdr(lo); in pnfs_free_layout_hdr()
294 pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo) in pnfs_detach_layout_hdr() argument
296 struct nfs_inode *nfsi = NFS_I(lo->plh_inode); in pnfs_detach_layout_hdr()
297 dprintk("%s: freeing layout cache %p\n", __func__, lo); in pnfs_detach_layout_hdr()
305 pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) in pnfs_put_layout_hdr() argument
310 if (!lo) in pnfs_put_layout_hdr()
312 inode = lo->plh_inode; in pnfs_put_layout_hdr()
313 pnfs_layoutreturn_before_put_layout_hdr(lo); in pnfs_put_layout_hdr()
315 if (refcount_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) { in pnfs_put_layout_hdr()
316 if (!list_empty(&lo->plh_segs)) in pnfs_put_layout_hdr()
318 pnfs_detach_layout_hdr(lo); in pnfs_put_layout_hdr()
321 pnfs_free_layout_hdr(lo); in pnfs_put_layout_hdr()
324 wake_up_var(lo); in pnfs_put_layout_hdr()
329 pnfs_grab_inode_layout_hdr(struct pnfs_layout_hdr *lo) in pnfs_grab_inode_layout_hdr() argument
331 struct inode *inode = igrab(lo->plh_inode); in pnfs_grab_inode_layout_hdr()
334 set_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags); in pnfs_grab_inode_layout_hdr()
347 static void pnfs_barrier_update(struct pnfs_layout_hdr *lo, u32 newseq) in pnfs_barrier_update() argument
349 if (pnfs_seqid_is_newer(newseq, lo->plh_barrier) || !lo->plh_barrier) in pnfs_barrier_update()
350 lo->plh_barrier = newseq; in pnfs_barrier_update()
354 pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, in pnfs_set_plh_return_info() argument
357 if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode) in pnfs_set_plh_return_info()
359 lo->plh_return_iomode = iomode; in pnfs_set_plh_return_info()
360 set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); in pnfs_set_plh_return_info()
362 * We must set lo->plh_return_seq to avoid livelocks with in pnfs_set_plh_return_info()
366 seq = be32_to_cpu(lo->plh_stateid.seqid); in pnfs_set_plh_return_info()
367 if (!lo->plh_return_seq || pnfs_seqid_is_newer(seq, lo->plh_return_seq)) in pnfs_set_plh_return_info()
368 lo->plh_return_seq = seq; in pnfs_set_plh_return_info()
369 pnfs_barrier_update(lo, seq); in pnfs_set_plh_return_info()
373 pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo) in pnfs_clear_layoutreturn_info() argument
376 lo->plh_return_iomode = 0; in pnfs_clear_layoutreturn_info()
377 lo->plh_return_seq = 0; in pnfs_clear_layoutreturn_info()
378 clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); in pnfs_clear_layoutreturn_info()
379 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { in pnfs_clear_layoutreturn_info()
382 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); in pnfs_clear_layoutreturn_info()
386 static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) in pnfs_clear_layoutreturn_waitbit() argument
388 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); in pnfs_clear_layoutreturn_waitbit()
389 clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags); in pnfs_clear_layoutreturn_waitbit()
391 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); in pnfs_clear_layoutreturn_waitbit()
392 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); in pnfs_clear_layoutreturn_waitbit()
415 struct pnfs_layout_hdr *lo; in nfs4_layout_refresh_old_stateid() local
426 lo = NFS_I(inode)->layout; in nfs4_layout_refresh_old_stateid()
427 if (lo && pnfs_layout_is_valid(lo) && in nfs4_layout_refresh_old_stateid()
428 nfs4_stateid_match_other(dst, &lo->plh_stateid)) { in nfs4_layout_refresh_old_stateid()
430 if (!nfs4_stateid_is_newer(&lo->plh_stateid, dst)) { in nfs4_layout_refresh_old_stateid()
436 err = pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); in nfs4_layout_refresh_old_stateid()
438 dst->seqid = lo->plh_stateid.seqid; in nfs4_layout_refresh_old_stateid()
457 pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, in pnfs_mark_layout_stateid_invalid() argument
467 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); in pnfs_mark_layout_stateid_invalid()
468 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) in pnfs_mark_layout_stateid_invalid()
470 pnfs_clear_layoutreturn_info(lo); in pnfs_mark_layout_stateid_invalid()
471 pnfs_free_returned_lsegs(lo, lseg_list, &range, 0); in pnfs_mark_layout_stateid_invalid()
472 set_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags); in pnfs_mark_layout_stateid_invalid()
473 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) && in pnfs_mark_layout_stateid_invalid()
474 !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) in pnfs_mark_layout_stateid_invalid()
475 pnfs_clear_layoutreturn_waitbit(lo); in pnfs_mark_layout_stateid_invalid()
476 return !list_empty(&lo->plh_segs); in pnfs_mark_layout_stateid_invalid()
487 pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) in pnfs_layout_set_fail_bit() argument
489 lo->plh_retry_timestamp = jiffies; in pnfs_layout_set_fail_bit()
490 if (!test_and_set_bit(fail_bit, &lo->plh_flags)) in pnfs_layout_set_fail_bit()
491 refcount_inc(&lo->plh_refcount); in pnfs_layout_set_fail_bit()
495 pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) in pnfs_layout_clear_fail_bit() argument
497 if (test_and_clear_bit(fail_bit, &lo->plh_flags)) in pnfs_layout_clear_fail_bit()
498 refcount_dec(&lo->plh_refcount); in pnfs_layout_clear_fail_bit()
502 pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) in pnfs_layout_io_set_failed() argument
504 struct inode *inode = lo->plh_inode; in pnfs_layout_io_set_failed()
513 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); in pnfs_layout_io_set_failed()
514 pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); in pnfs_layout_io_set_failed()
522 pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode) in pnfs_layout_io_test_failed() argument
527 if (test_bit(fail_bit, &lo->plh_flags) == 0) in pnfs_layout_io_test_failed()
531 if (!time_in_range(lo->plh_retry_timestamp, start, end)) { in pnfs_layout_io_test_failed()
533 pnfs_layout_clear_fail_bit(lo, fail_bit); in pnfs_layout_io_test_failed()
540 pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg, in pnfs_init_lseg() argument
549 lseg->pls_layout = lo; in pnfs_init_lseg()
563 pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, in pnfs_layout_remove_lseg() argument
569 refcount_dec(&lo->plh_refcount); in pnfs_layout_remove_lseg()
572 if (list_empty(&lo->plh_segs) && in pnfs_layout_remove_lseg()
573 !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) && in pnfs_layout_remove_lseg()
574 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { in pnfs_layout_remove_lseg()
575 if (atomic_read(&lo->plh_outstanding) == 0) in pnfs_layout_remove_lseg()
576 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); in pnfs_layout_remove_lseg()
577 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); in pnfs_layout_remove_lseg()
582 pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo, in pnfs_cache_lseg_for_layoutreturn() argument
586 pnfs_layout_is_valid(lo)) { in pnfs_cache_lseg_for_layoutreturn()
587 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); in pnfs_cache_lseg_for_layoutreturn()
588 list_move_tail(&lseg->pls_list, &lo->plh_return_segs); in pnfs_cache_lseg_for_layoutreturn()
597 struct pnfs_layout_hdr *lo; in pnfs_put_lseg() local
607 lo = lseg->pls_layout; in pnfs_put_lseg()
608 inode = lo->plh_inode; in pnfs_put_lseg()
611 pnfs_get_layout_hdr(lo); in pnfs_put_lseg()
612 pnfs_layout_remove_lseg(lo, lseg); in pnfs_put_lseg()
613 if (pnfs_cache_lseg_for_layoutreturn(lo, lseg)) in pnfs_put_lseg()
617 pnfs_put_layout_hdr(lo); in pnfs_put_lseg()
693 * @lo: layout header containing the lsegs
707 pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, in pnfs_mark_matching_lsegs_invalid() argument
713 struct nfs_server *server = NFS_SERVER(lo->plh_inode); in pnfs_mark_matching_lsegs_invalid()
716 dprintk("%s:Begin lo %p\n", __func__, lo); in pnfs_mark_matching_lsegs_invalid()
718 if (list_empty(&lo->plh_segs)) in pnfs_mark_matching_lsegs_invalid()
720 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) in pnfs_mark_matching_lsegs_invalid()
735 static void pnfs_reset_return_info(struct pnfs_layout_hdr *lo) in pnfs_reset_return_info() argument
739 list_for_each_entry(lseg, &lo->plh_return_segs, pls_list) in pnfs_reset_return_info()
740 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); in pnfs_reset_return_info()
744 pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, in pnfs_free_returned_lsegs() argument
751 list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) { in pnfs_free_returned_lsegs()
774 struct pnfs_layout_hdr *lo; in __pnfs_destroy_layout() local
778 lo = nfsi->layout; in __pnfs_destroy_layout()
779 if (lo) { in __pnfs_destroy_layout()
780 pnfs_get_layout_hdr(lo); in __pnfs_destroy_layout()
781 pnfs_mark_layout_stateid_invalid(lo, &tmp_list); in __pnfs_destroy_layout()
782 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED); in __pnfs_destroy_layout()
783 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED); in __pnfs_destroy_layout()
787 pnfs_put_layout_hdr(lo); in __pnfs_destroy_layout()
790 return lo; in __pnfs_destroy_layout()
800 struct pnfs_layout_hdr *lo) in pnfs_layout_removed() argument
805 ret = nfsi->layout != lo; in pnfs_layout_removed()
812 struct pnfs_layout_hdr *lo = __pnfs_destroy_layout(nfsi); in pnfs_destroy_layout_final() local
814 if (lo) in pnfs_destroy_layout_final()
815 wait_var_event(lo, pnfs_layout_removed(nfsi, lo)); in pnfs_destroy_layout_final()
822 struct pnfs_layout_hdr *lo; in pnfs_layout_add_bulk_destroy_list() local
826 lo = NFS_I(inode)->layout; in pnfs_layout_add_bulk_destroy_list()
827 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) { in pnfs_layout_add_bulk_destroy_list()
828 pnfs_get_layout_hdr(lo); in pnfs_layout_add_bulk_destroy_list()
829 list_add(&lo->plh_bulk_destroy, layout_list); in pnfs_layout_add_bulk_destroy_list()
844 struct pnfs_layout_hdr *lo, *next; in pnfs_layout_bulk_destroy_byserver_locked() local
847 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) { in pnfs_layout_bulk_destroy_byserver_locked()
848 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags) || in pnfs_layout_bulk_destroy_byserver_locked()
849 test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) || in pnfs_layout_bulk_destroy_byserver_locked()
850 !list_empty(&lo->plh_bulk_destroy)) in pnfs_layout_bulk_destroy_byserver_locked()
855 inode = pnfs_grab_inode_layout_hdr(lo); in pnfs_layout_bulk_destroy_byserver_locked()
857 if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) in pnfs_layout_bulk_destroy_byserver_locked()
858 list_del_rcu(&lo->plh_layouts); in pnfs_layout_bulk_destroy_byserver_locked()
881 struct pnfs_layout_hdr *lo; in pnfs_layout_free_bulk_destroy_list() local
887 lo = list_entry(layout_list->next, struct pnfs_layout_hdr, in pnfs_layout_free_bulk_destroy_list()
890 lo->plh_inode->i_ino); in pnfs_layout_free_bulk_destroy_list()
891 inode = lo->plh_inode; in pnfs_layout_free_bulk_destroy_list()
896 list_del_init(&lo->plh_bulk_destroy); in pnfs_layout_free_bulk_destroy_list()
897 if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) { in pnfs_layout_free_bulk_destroy_list()
899 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); in pnfs_layout_free_bulk_destroy_list()
906 pnfs_put_layout_hdr(lo); in pnfs_layout_free_bulk_destroy_list()
977 pnfs_set_layout_cred(struct pnfs_layout_hdr *lo, const struct cred *cred) in pnfs_set_layout_cred() argument
981 if (cred && cred_fscmp(lo->plh_lc_cred, cred) != 0) { in pnfs_set_layout_cred()
982 old = xchg(&lo->plh_lc_cred, get_cred(cred)); in pnfs_set_layout_cred()
987 /* update lo->plh_stateid with new if is more recent */
989 pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, in pnfs_set_layout_stateid() argument
992 u32 oldseq = be32_to_cpu(lo->plh_stateid.seqid); in pnfs_set_layout_stateid()
995 if (!pnfs_layout_is_valid(lo)) { in pnfs_set_layout_stateid()
996 pnfs_set_layout_cred(lo, cred); in pnfs_set_layout_stateid()
997 nfs4_stateid_copy(&lo->plh_stateid, new); in pnfs_set_layout_stateid()
998 lo->plh_barrier = newseq; in pnfs_set_layout_stateid()
999 pnfs_clear_layoutreturn_info(lo); in pnfs_set_layout_stateid()
1000 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); in pnfs_set_layout_stateid()
1005 nfs4_stateid_copy(&lo->plh_stateid, new); in pnfs_set_layout_stateid()
1008 pnfs_barrier_update(lo, newseq); in pnfs_set_layout_stateid()
1016 if (atomic_read(&lo->plh_outstanding) == 1) in pnfs_set_layout_stateid()
1017 pnfs_barrier_update(lo, be32_to_cpu(lo->plh_stateid.seqid)); in pnfs_set_layout_stateid()
1021 pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, in pnfs_layout_stateid_blocked() argument
1026 return lo->plh_barrier && pnfs_seqid_is_newer(lo->plh_barrier, seqid); in pnfs_layout_stateid_blocked()
1031 pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo) in pnfs_layoutgets_blocked() argument
1033 return lo->plh_block_lgets || in pnfs_layoutgets_blocked()
1034 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); in pnfs_layoutgets_blocked()
1154 pnfs_put_layout_hdr(lgp->lo); in pnfs_layoutget_free()
1174 void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo, in pnfs_layoutreturn_free_lsegs() argument
1179 struct inode *inode = lo->plh_inode; in pnfs_layoutreturn_free_lsegs()
1183 if (!nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) in pnfs_layoutreturn_free_lsegs()
1185 if (stateid && pnfs_layout_is_valid(lo)) { in pnfs_layoutreturn_free_lsegs()
1188 pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq); in pnfs_layoutreturn_free_lsegs()
1189 pnfs_free_returned_lsegs(lo, &freeme, range, seq); in pnfs_layoutreturn_free_lsegs()
1190 pnfs_set_layout_stateid(lo, stateid, NULL, true); in pnfs_layoutreturn_free_lsegs()
1191 pnfs_reset_return_info(lo); in pnfs_layoutreturn_free_lsegs()
1193 pnfs_mark_layout_stateid_invalid(lo, &freeme); in pnfs_layoutreturn_free_lsegs()
1195 pnfs_clear_layoutreturn_waitbit(lo); in pnfs_layoutreturn_free_lsegs()
1202 pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, in pnfs_prepare_layoutreturn() argument
1208 if (atomic_read(&lo->plh_outstanding) != 0 && lo->plh_return_seq == 0) in pnfs_prepare_layoutreturn()
1210 if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) in pnfs_prepare_layoutreturn()
1212 set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); in pnfs_prepare_layoutreturn()
1213 pnfs_get_layout_hdr(lo); in pnfs_prepare_layoutreturn()
1214 nfs4_stateid_copy(stateid, &lo->plh_stateid); in pnfs_prepare_layoutreturn()
1215 *cred = get_cred(lo->plh_lc_cred); in pnfs_prepare_layoutreturn()
1216 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { in pnfs_prepare_layoutreturn()
1217 if (lo->plh_return_seq != 0) in pnfs_prepare_layoutreturn()
1218 stateid->seqid = cpu_to_be32(lo->plh_return_seq); in pnfs_prepare_layoutreturn()
1220 *iomode = lo->plh_return_iomode; in pnfs_prepare_layoutreturn()
1221 pnfs_clear_layoutreturn_info(lo); in pnfs_prepare_layoutreturn()
1224 pnfs_barrier_update(lo, be32_to_cpu(stateid->seqid)); in pnfs_prepare_layoutreturn()
1230 struct pnfs_layout_hdr *lo, in pnfs_init_layoutreturn_args() argument
1234 struct inode *inode = lo->plh_inode; in pnfs_init_layoutreturn_args()
1241 args->layout = lo; in pnfs_init_layoutreturn_args()
1246 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, in pnfs_send_layoutreturn() argument
1252 struct inode *ino = lo->plh_inode; in pnfs_send_layoutreturn()
1263 pnfs_clear_layoutreturn_waitbit(lo); in pnfs_send_layoutreturn()
1266 pnfs_put_layout_hdr(lo); in pnfs_send_layoutreturn()
1270 pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode); in pnfs_send_layoutreturn()
1284 pnfs_layout_segments_returnable(struct pnfs_layout_hdr *lo, in pnfs_layout_segments_returnable() argument
1292 return pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, in pnfs_layout_segments_returnable()
1298 pnfs_layout_need_return(struct pnfs_layout_hdr *lo) in pnfs_layout_need_return() argument
1300 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) in pnfs_layout_need_return()
1302 return pnfs_layout_segments_returnable(lo, lo->plh_return_iomode, in pnfs_layout_need_return()
1303 lo->plh_return_seq); in pnfs_layout_need_return()
1306 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) in pnfs_layoutreturn_before_put_layout_hdr() argument
1308 struct inode *inode= lo->plh_inode; in pnfs_layoutreturn_before_put_layout_hdr()
1310 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) in pnfs_layoutreturn_before_put_layout_hdr()
1313 if (pnfs_layout_need_return(lo)) { in pnfs_layoutreturn_before_put_layout_hdr()
1319 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); in pnfs_layoutreturn_before_put_layout_hdr()
1323 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, false); in pnfs_layoutreturn_before_put_layout_hdr()
1340 struct pnfs_layout_hdr *lo = NULL; in _pnfs_return_layout() local
1356 lo = nfsi->layout; in _pnfs_return_layout()
1357 if (!lo) { in _pnfs_return_layout()
1363 pnfs_get_layout_hdr(lo); in _pnfs_return_layout()
1365 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { in _pnfs_return_layout()
1367 if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, in _pnfs_return_layout()
1372 valid_layout = pnfs_layout_is_valid(lo); in _pnfs_return_layout()
1374 pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0); in _pnfs_return_layout()
1377 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); in _pnfs_return_layout()
1380 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) || in _pnfs_return_layout()
1387 send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, NULL); in _pnfs_return_layout()
1390 status = pnfs_send_layoutreturn(lo, &stateid, &cred, IOMODE_ANY, true); in _pnfs_return_layout()
1392 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, TASK_UNINTERRUPTIBLE); in _pnfs_return_layout()
1395 pnfs_put_layout_hdr(lo); in _pnfs_return_layout()
1404 struct pnfs_layout_hdr *lo; in pnfs_commit_and_return_layout() local
1408 lo = NFS_I(inode)->layout; in pnfs_commit_and_return_layout()
1409 if (lo == NULL) { in pnfs_commit_and_return_layout()
1413 pnfs_get_layout_hdr(lo); in pnfs_commit_and_return_layout()
1415 lo->plh_block_lgets++; in pnfs_commit_and_return_layout()
1422 lo->plh_block_lgets--; in pnfs_commit_and_return_layout()
1424 pnfs_put_layout_hdr(lo); in pnfs_commit_and_return_layout()
1436 struct pnfs_layout_hdr *lo; in pnfs_roc() local
1449 lo = nfsi->layout; in pnfs_roc()
1450 if (!lo || !pnfs_layout_is_valid(lo) || in pnfs_roc()
1451 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { in pnfs_roc()
1452 lo = NULL; in pnfs_roc()
1455 pnfs_get_layout_hdr(lo); in pnfs_roc()
1456 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { in pnfs_roc()
1459 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, in pnfs_roc()
1461 pnfs_put_layout_hdr(lo); in pnfs_roc()
1484 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) { in pnfs_roc()
1495 if (!mark_lseg_invalid(lseg, &lo->plh_return_segs)) in pnfs_roc()
1497 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); in pnfs_roc()
1500 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) in pnfs_roc()
1507 /* lo ref dropped in pnfs_roc_release() */ in pnfs_roc()
1508 layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &lc_cred, &iomode); in pnfs_roc()
1514 pnfs_init_layoutreturn_args(args, lo, &stateid, iomode); in pnfs_roc()
1527 pnfs_put_layout_hdr(lo); in pnfs_roc()
1531 pnfs_send_layoutreturn(lo, &stateid, &lc_cred, iomode, true); in pnfs_roc()
1532 pnfs_put_layout_hdr(lo); in pnfs_roc()
1584 struct pnfs_layout_hdr *lo = args->layout; in pnfs_roc_release() local
1592 if (pnfs_layout_is_valid(lo) && in pnfs_roc_release()
1593 nfs4_stateid_match_other(&args->stateid, &lo->plh_stateid)) in pnfs_roc_release()
1594 pnfs_set_plh_return_info(lo, args->range.iomode, 0); in pnfs_roc_release()
1595 pnfs_clear_layoutreturn_waitbit(lo); in pnfs_roc_release()
1603 pnfs_layoutreturn_free_lsegs(lo, &args->stateid, &args->range, in pnfs_roc_release()
1609 pnfs_put_layout_hdr(lo); in pnfs_roc_release()
1615 struct pnfs_layout_hdr *lo; in pnfs_wait_on_layoutreturn() local
1618 /* we might not have grabbed lo reference. so need to check under in pnfs_wait_on_layoutreturn()
1621 lo = nfsi->layout; in pnfs_wait_on_layoutreturn()
1622 if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { in pnfs_wait_on_layoutreturn()
1670 pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo, in pnfs_generic_layout_insert_lseg() argument
1682 list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) { in pnfs_generic_layout_insert_lseg()
1701 list_add_tail(&lseg->pls_list, &lo->plh_segs); in pnfs_generic_layout_insert_lseg()
1707 pnfs_get_layout_hdr(lo); in pnfs_generic_layout_insert_lseg()
1714 pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo, in pnfs_layout_insert_lseg() argument
1718 struct inode *inode = lo->plh_inode; in pnfs_layout_insert_lseg()
1722 ld->add_lseg(lo, lseg, free_me); in pnfs_layout_insert_lseg()
1724 pnfs_generic_layout_insert_lseg(lo, lseg, in pnfs_layout_insert_lseg()
1735 struct pnfs_layout_hdr *lo; in alloc_init_layout_hdr() local
1737 lo = pnfs_alloc_layout_hdr(ino, gfp_flags); in alloc_init_layout_hdr()
1738 if (!lo) in alloc_init_layout_hdr()
1740 refcount_set(&lo->plh_refcount, 1); in alloc_init_layout_hdr()
1741 INIT_LIST_HEAD(&lo->plh_layouts); in alloc_init_layout_hdr()
1742 INIT_LIST_HEAD(&lo->plh_segs); in alloc_init_layout_hdr()
1743 INIT_LIST_HEAD(&lo->plh_return_segs); in alloc_init_layout_hdr()
1744 INIT_LIST_HEAD(&lo->plh_bulk_destroy); in alloc_init_layout_hdr()
1745 lo->plh_inode = ino; in alloc_init_layout_hdr()
1746 lo->plh_lc_cred = get_cred(ctx->cred); in alloc_init_layout_hdr()
1747 lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID; in alloc_init_layout_hdr()
1748 return lo; in alloc_init_layout_hdr()
1816 pnfs_find_lseg(struct pnfs_layout_hdr *lo, in pnfs_find_lseg() argument
1824 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { in pnfs_find_lseg()
1913 static int pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) in pnfs_prepare_to_retry_layoutget() argument
1919 pnfs_layoutcommit_inode(lo->plh_inode, false); in pnfs_prepare_to_retry_layoutget()
1920 return wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, in pnfs_prepare_to_retry_layoutget()
1925 static void nfs_layoutget_begin(struct pnfs_layout_hdr *lo) in nfs_layoutget_begin() argument
1927 atomic_inc(&lo->plh_outstanding); in nfs_layoutget_begin()
1930 static void nfs_layoutget_end(struct pnfs_layout_hdr *lo) in nfs_layoutget_end() argument
1932 if (atomic_dec_and_test(&lo->plh_outstanding) && in nfs_layoutget_end()
1933 test_and_clear_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags)) { in nfs_layoutget_end()
1935 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN); in nfs_layoutget_end()
1939 static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) in pnfs_is_first_layoutget() argument
1941 return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); in pnfs_is_first_layoutget()
1944 static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) in pnfs_clear_first_layoutget() argument
1946 unsigned long *bitlock = &lo->plh_flags; in pnfs_clear_first_layoutget()
1953 static void _add_to_server_list(struct pnfs_layout_hdr *lo, in _add_to_server_list() argument
1956 if (!test_and_set_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { in _add_to_server_list()
1959 /* The lo must be on the clp list if there is any in _add_to_server_list()
1963 list_add_tail_rcu(&lo->plh_layouts, &server->layouts); in _add_to_server_list()
1989 struct pnfs_layout_hdr *lo = NULL; in pnfs_update_layout() local
1998 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, in pnfs_update_layout()
2004 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, in pnfs_update_layout()
2012 iomode, lo, lseg, in pnfs_update_layout()
2023 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); in pnfs_update_layout()
2024 if (lo == NULL) { in pnfs_update_layout()
2027 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, in pnfs_update_layout()
2033 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { in pnfs_update_layout()
2034 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, in pnfs_update_layout()
2041 if (pnfs_layout_io_test_failed(lo, iomode)) { in pnfs_update_layout()
2042 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, in pnfs_update_layout()
2051 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && in pnfs_update_layout()
2052 atomic_read(&lo->plh_outstanding) != 0) { in pnfs_update_layout()
2054 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN, in pnfs_update_layout()
2058 pnfs_put_layout_hdr(lo); in pnfs_update_layout()
2066 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { in pnfs_update_layout()
2069 lseg = ERR_PTR(pnfs_prepare_to_retry_layoutget(lo)); in pnfs_update_layout()
2071 pnfs_put_layout_hdr(lo); in pnfs_update_layout()
2073 trace_pnfs_update_layout(ino, pos, count, iomode, lo, in pnfs_update_layout()
2078 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, in pnfs_update_layout()
2083 lseg = pnfs_find_lseg(lo, &arg, strict_iomode); in pnfs_update_layout()
2085 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, in pnfs_update_layout()
2095 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) { in pnfs_update_layout()
2103 &lo->plh_flags)) { in pnfs_update_layout()
2105 lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, in pnfs_update_layout()
2110 pnfs_put_layout_hdr(lo); in pnfs_update_layout()
2123 iomode, lo, lseg, in pnfs_update_layout()
2126 pnfs_clear_first_layoutget(lo); in pnfs_update_layout()
2127 pnfs_put_layout_hdr(lo); in pnfs_update_layout()
2132 nfs4_stateid_copy(&stateid, &lo->plh_stateid); in pnfs_update_layout()
2135 if (pnfs_layoutgets_blocked(lo)) { in pnfs_update_layout()
2136 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, in pnfs_update_layout()
2140 nfs_layoutget_begin(lo); in pnfs_update_layout()
2143 _add_to_server_list(lo, server); in pnfs_update_layout()
2156 trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL, in pnfs_update_layout()
2158 nfs_layoutget_end(lo); in pnfs_update_layout()
2162 lgp->lo = lo; in pnfs_update_layout()
2163 pnfs_get_layout_hdr(lo); in pnfs_update_layout()
2166 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, in pnfs_update_layout()
2168 nfs_layoutget_end(lo); in pnfs_update_layout()
2181 lo, pnfs_iomode_to_fail_bit(iomode)); in pnfs_update_layout()
2186 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); in pnfs_update_layout()
2193 pnfs_clear_first_layoutget(lo); in pnfs_update_layout()
2195 iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); in pnfs_update_layout()
2196 pnfs_put_layout_hdr(lo); in pnfs_update_layout()
2200 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); in pnfs_update_layout()
2205 pnfs_clear_first_layoutget(lo); in pnfs_update_layout()
2206 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, in pnfs_update_layout()
2208 pnfs_put_layout_hdr(lo); in pnfs_update_layout()
2248 struct pnfs_layout_hdr *lo; in _pnfs_grab_empty_layout() local
2251 lo = pnfs_find_alloc_layout(ino, ctx, nfs_io_gfp_mask()); in _pnfs_grab_empty_layout()
2252 if (!lo) in _pnfs_grab_empty_layout()
2254 if (!test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) in _pnfs_grab_empty_layout()
2256 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) in _pnfs_grab_empty_layout()
2258 if (pnfs_layoutgets_blocked(lo)) in _pnfs_grab_empty_layout()
2260 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags)) in _pnfs_grab_empty_layout()
2262 nfs_layoutget_begin(lo); in _pnfs_grab_empty_layout()
2264 _add_to_server_list(lo, NFS_SERVER(ino)); in _pnfs_grab_empty_layout()
2265 return lo; in _pnfs_grab_empty_layout()
2269 pnfs_put_layout_hdr(lo); in _pnfs_grab_empty_layout()
2284 struct pnfs_layout_hdr *lo; in _lgopen_prepare_attached() local
2291 lo = _pnfs_grab_empty_layout(ino, ctx); in _lgopen_prepare_attached()
2292 if (!lo) in _lgopen_prepare_attached()
2297 pnfs_clear_first_layoutget(lo); in _lgopen_prepare_attached()
2298 nfs_layoutget_end(lo); in _lgopen_prepare_attached()
2299 pnfs_put_layout_hdr(lo); in _lgopen_prepare_attached()
2302 lgp->lo = lo; in _lgopen_prepare_attached()
2351 struct pnfs_layout_hdr *lo; in pnfs_parse_lgopen() local
2381 if (!lgp->lo) { in pnfs_parse_lgopen()
2382 lo = _pnfs_grab_empty_layout(ino, ctx); in pnfs_parse_lgopen()
2383 if (!lo) in pnfs_parse_lgopen()
2385 lgp->lo = lo; in pnfs_parse_lgopen()
2387 lo = lgp->lo; in pnfs_parse_lgopen()
2392 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); in pnfs_parse_lgopen()
2400 if (lgp->lo) { in nfs4_lgopen_release()
2401 pnfs_clear_first_layoutget(lgp->lo); in nfs4_lgopen_release()
2402 nfs_layoutget_end(lgp->lo); in nfs4_lgopen_release()
2411 struct pnfs_layout_hdr *lo = lgp->lo; in pnfs_layout_process() local
2414 struct inode *ino = lo->plh_inode; in pnfs_layout_process()
2421 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags); in pnfs_layout_process()
2431 pnfs_init_lseg(lo, lseg, &res->range, &res->stateid); in pnfs_layout_process()
2434 if (pnfs_layoutgets_blocked(lo)) { in pnfs_layout_process()
2439 if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && in pnfs_layout_process()
2440 !pnfs_is_first_layoutget(lo)) in pnfs_layout_process()
2443 if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { in pnfs_layout_process()
2445 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { in pnfs_layout_process()
2446 if (!pnfs_layout_is_valid(lo)) in pnfs_layout_process()
2447 lo->plh_barrier = 0; in pnfs_layout_process()
2451 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, false); in pnfs_layout_process()
2452 } else if (pnfs_layout_is_valid(lo)) { in pnfs_layout_process()
2461 pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0); in pnfs_layout_process()
2465 pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, true); in pnfs_layout_process()
2469 pnfs_layout_insert_lseg(lo, lseg, &free_me); in pnfs_layout_process()
2481 lseg->pls_layout = lo; in pnfs_layout_process()
2488 * @lo: pointer to layout header
2503 pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, in pnfs_mark_matching_lsegs_return() argument
2509 struct nfs_server *server = NFS_SERVER(lo->plh_inode); in pnfs_mark_matching_lsegs_return()
2512 dprintk("%s:Begin lo %p\n", __func__, lo); in pnfs_mark_matching_lsegs_return()
2514 assert_spin_locked(&lo->plh_inode->i_lock); in pnfs_mark_matching_lsegs_return()
2516 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) in pnfs_mark_matching_lsegs_return()
2517 tmp_list = &lo->plh_return_segs; in pnfs_mark_matching_lsegs_return()
2519 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) in pnfs_mark_matching_lsegs_return()
2527 tmp_list = &lo->plh_return_segs; in pnfs_mark_matching_lsegs_return()
2536 pnfs_set_plh_return_info(lo, return_range->iomode, seq); in pnfs_mark_matching_lsegs_return()
2540 if (!list_empty(&lo->plh_return_segs)) { in pnfs_mark_matching_lsegs_return()
2541 pnfs_set_plh_return_info(lo, return_range->iomode, seq); in pnfs_mark_matching_lsegs_return()
2552 struct pnfs_layout_hdr *lo; in pnfs_mark_layout_for_return() local
2556 lo = NFS_I(inode)->layout; in pnfs_mark_layout_for_return()
2557 if (!pnfs_layout_is_valid(lo)) { in pnfs_mark_layout_for_return()
2561 pnfs_set_plh_return_info(lo, range->iomode, 0); in pnfs_mark_layout_for_return()
2567 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0) != -EBUSY) { in pnfs_mark_layout_for_return()
2572 return_now = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); in pnfs_mark_layout_for_return()
2575 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, false); in pnfs_mark_layout_for_return()
2596 pnfs_layout_can_be_returned(struct pnfs_layout_hdr *lo) in pnfs_layout_can_be_returned() argument
2598 return pnfs_layout_is_valid(lo) && in pnfs_layout_can_be_returned()
2599 !test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) && in pnfs_layout_can_be_returned()
2600 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); in pnfs_layout_can_be_returned()
2604 pnfs_find_first_lseg(struct pnfs_layout_hdr *lo, in pnfs_find_first_lseg() argument
2610 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { in pnfs_find_first_lseg()
2625 pnfs_should_return_unused_layout(struct pnfs_layout_hdr *lo, in pnfs_should_return_unused_layout() argument
2632 if (!pnfs_layout_can_be_returned(lo) || in pnfs_should_return_unused_layout()
2633 !pnfs_find_first_lseg(lo, range, range->iomode)) in pnfs_should_return_unused_layout()
2636 head = &NFS_I(lo->plh_inode)->open_files; in pnfs_should_return_unused_layout()
2649 if (pnfs_find_first_lseg(lo, range, IOMODE_READ)) in pnfs_should_return_unused_layout()
2660 struct pnfs_layout_hdr *lo; in pnfs_layout_return_unused_byserver() local
2667 list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { in pnfs_layout_return_unused_byserver()
2668 inode = lo->plh_inode; in pnfs_layout_return_unused_byserver()
2669 if (!inode || !pnfs_layout_can_be_returned(lo) || in pnfs_layout_return_unused_byserver()
2670 test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) in pnfs_layout_return_unused_byserver()
2673 if (!lo->plh_inode || in pnfs_layout_return_unused_byserver()
2674 !pnfs_should_return_unused_layout(lo, range)) { in pnfs_layout_return_unused_byserver()
2678 pnfs_get_layout_hdr(lo); in pnfs_layout_return_unused_byserver()
2679 pnfs_set_plh_return_info(lo, range->iomode, 0); in pnfs_layout_return_unused_byserver()
2680 if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, in pnfs_layout_return_unused_byserver()
2682 !pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode)) { in pnfs_layout_return_unused_byserver()
2685 pnfs_put_layout_hdr(lo); in pnfs_layout_return_unused_byserver()
2691 pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, false); in pnfs_layout_return_unused_byserver()
2692 pnfs_put_layout_hdr(lo); in pnfs_layout_return_unused_byserver()