27 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)  in mr_check_range()  argument
35 		if (iova < mr->ibmr.iova ||  in mr_check_range()
36 		    iova + length > mr->ibmr.iova + mr->ibmr.length) {  in mr_check_range()
37 			rxe_dbg_mr(mr, "iova/length out of range");  in mr_check_range()
75 static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova)  in rxe_mr_iova_to_index()  argument
77 	return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift);  in rxe_mr_iova_to_index()
80 static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova)  in rxe_mr_iova_to_page_offset()  argument
82 	return iova & (mr_page_size(mr) - 1);  in rxe_mr_iova_to_page_offset()
129 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,  in rxe_mr_init_user()  argument
240 	mr->page_offset = mr->ibmr.iova & (page_size - 1);  in rxe_map_mr_sg()
245 static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr,  in rxe_mr_copy_xarray()  argument
248 	unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova);  in rxe_mr_copy_xarray()
249 	unsigned long index = rxe_mr_iova_to_index(mr, iova);  in rxe_mr_copy_xarray()
304 int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,  in rxe_mr_copy()  argument
316 		rxe_mr_copy_dma(mr, iova, addr, length, dir);  in rxe_mr_copy()
320 	err = mr_check_range(mr, iova, length);  in rxe_mr_copy()
322 		rxe_dbg_mr(mr, "iova out of range");  in rxe_mr_copy()
326 	return rxe_mr_copy_xarray(mr, iova, addr, length, dir);  in rxe_mr_copy()
345 	u64			iova;  in copy_data()  local
397 			iova = sge->addr + offset;  in copy_data()
398 			err = rxe_mr_copy(mr, iova, addr, bytes, dir);  in copy_data()
424 int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length)  in rxe_flush_pmem_iova()  argument
443 	err = mr_check_range(mr, iova, length);  in rxe_flush_pmem_iova()
448 		index = rxe_mr_iova_to_index(mr, iova);  in rxe_flush_pmem_iova()
450 		page_offset = rxe_mr_iova_to_page_offset(mr, iova);  in rxe_flush_pmem_iova()
461 		iova += bytes;  in rxe_flush_pmem_iova()
471 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,  in rxe_mr_do_atomic_op()  argument
485 		page_offset = iova & (PAGE_SIZE - 1);  in rxe_mr_do_atomic_op()
486 		page = ib_virt_dma_to_page(iova);  in rxe_mr_do_atomic_op()
491 		err = mr_check_range(mr, iova, sizeof(value));  in rxe_mr_do_atomic_op()
493 			rxe_dbg_mr(mr, "iova out of range");  in rxe_mr_do_atomic_op()
496 		page_offset = rxe_mr_iova_to_page_offset(mr, iova);  in rxe_mr_do_atomic_op()
497 		index = rxe_mr_iova_to_index(mr, iova);  in rxe_mr_do_atomic_op()
504 		rxe_dbg_mr(mr, "iova not aligned");  in rxe_mr_do_atomic_op()
529 int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)  in rxe_mr_do_atomic_write()  argument
542 		page_offset = iova & (PAGE_SIZE - 1);  in rxe_mr_do_atomic_write()
543 		page = ib_virt_dma_to_page(iova);  in rxe_mr_do_atomic_write()
549 		err = mr_check_range(mr, iova, sizeof(value));  in rxe_mr_do_atomic_write()
551 			rxe_dbg_mr(mr, "iova out of range");  in rxe_mr_do_atomic_write()
554 		page_offset = rxe_mr_iova_to_page_offset(mr, iova);  in rxe_mr_do_atomic_write()
555 		index = rxe_mr_iova_to_index(mr, iova);  in rxe_mr_do_atomic_write()
577 int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)  in rxe_mr_do_atomic_write()  argument
716 	mr->ibmr.iova = wqe->wr.wr.reg.mr->iova;  in rxe_reg_fast_mr()