50 	struct siw_mem *mem;  in siw_mem_id2obj()  local
53 	mem = xa_load(&sdev->mem_xa, stag_index);  in siw_mem_id2obj()
54 	if (likely(mem && kref_get_unless_zero(&mem->ref))) {  in siw_mem_id2obj()
56 		return mem;  in siw_mem_id2obj()
93 	struct siw_mem *mem = kzalloc(sizeof(*mem), GFP_KERNEL);  in siw_mr_add_mem()  local
97 	if (!mem)  in siw_mr_add_mem()
100 	mem->mem_obj = mem_obj;  in siw_mr_add_mem()
101 	mem->stag_valid = 0;  in siw_mr_add_mem()
102 	mem->sdev = sdev;  in siw_mr_add_mem()
103 	mem->va = start;  in siw_mr_add_mem()
104 	mem->len = len;  in siw_mr_add_mem()
105 	mem->pd = pd;  in siw_mr_add_mem()
106 	mem->perms = rights & IWARP_ACCESS_MASK;  in siw_mr_add_mem()
107 	kref_init(&mem->ref);  in siw_mr_add_mem()
112 	if (xa_alloc_cyclic(&sdev->mem_xa, &id, mem, limit, &next,  in siw_mr_add_mem()
114 		kfree(mem);  in siw_mr_add_mem()
118 	mr->mem = mem;  in siw_mr_add_mem()
120 	mem->stag = id << 8;  in siw_mr_add_mem()
121 	mr->base_mr.lkey = mr->base_mr.rkey = mem->stag;  in siw_mr_add_mem()
128 	struct siw_mem *mem = mr->mem, *found;  in siw_mr_drop_mem()  local
130 	mem->stag_valid = 0;  in siw_mr_drop_mem()
135 	found = xa_erase(&mem->sdev->mem_xa, mem->stag >> 8);  in siw_mr_drop_mem()
136 	WARN_ON(found != mem);  in siw_mr_drop_mem()
137 	siw_mem_put(mem);  in siw_mr_drop_mem()
142 	struct siw_mem *mem = container_of(ref, struct siw_mem, ref);  in siw_free_mem()  local
144 	siw_dbg_mem(mem, "free mem, pbl: %s\n", mem->is_pbl ? "y" : "n");  in siw_free_mem()
146 	if (!mem->is_mw && mem->mem_obj) {  in siw_free_mem()
147 		if (mem->is_pbl == 0)  in siw_free_mem()
148 			siw_umem_release(mem->umem, true);  in siw_free_mem()
150 			kfree(mem->pbl);  in siw_free_mem()
152 	kfree(mem);  in siw_free_mem()
168 int siw_check_mem(struct ib_pd *pd, struct siw_mem *mem, u64 addr,  in siw_check_mem()  argument
171 	if (!mem->stag_valid) {  in siw_check_mem()
172 		siw_dbg_pd(pd, "STag 0x%08x invalid\n", mem->stag);  in siw_check_mem()
175 	if (mem->pd != pd) {  in siw_check_mem()
176 		siw_dbg_pd(pd, "STag 0x%08x: PD mismatch\n", mem->stag);  in siw_check_mem()
182 	if ((mem->perms & perms) < perms) {  in siw_check_mem()
184 			   mem->perms, perms);  in siw_check_mem()
190 	if (addr < mem->va || addr + len > mem->va + mem->len) {  in siw_check_mem()
196 			   (void *)(uintptr_t)mem->va,  in siw_check_mem()
197 			   (void *)(uintptr_t)(mem->va + mem->len),  in siw_check_mem()
198 			   mem->stag);  in siw_check_mem()
222 int siw_check_sge(struct ib_pd *pd, struct siw_sge *sge, struct siw_mem *mem[],  in siw_check_sge()  argument
233 	if (*mem == NULL) {  in siw_check_sge()
240 		*mem = new;  in siw_check_sge()
243 	if (unlikely((*mem)->stag != sge->lkey)) {  in siw_check_sge()
244 		siw_dbg_mem((*mem), "STag mismatch: 0x%08x\n", sge->lkey);  in siw_check_sge()
248 	rv = siw_check_mem(pd, *mem, sge->laddr + off, perms, len);  in siw_check_sge()
256 		*mem = NULL;  in siw_check_sge()
272 			siw_unref_mem_sgl(wqe->mem, wqe->sqe.num_sge);  in siw_wqe_put_mem()
276 		siw_unref_mem_sgl(wqe->mem, wqe->rqe.num_sge);  in siw_wqe_put_mem()
280 		siw_unref_mem_sgl(wqe->mem, 1);  in siw_wqe_put_mem()
295 	struct siw_mem *mem = siw_mem_id2obj(sdev, stag >> 8);  in siw_invalidate_stag()  local
298 	if (unlikely(!mem)) {  in siw_invalidate_stag()
302 	if (unlikely(mem->pd != pd)) {  in siw_invalidate_stag()
311 	mem->stag_valid = 0;  in siw_invalidate_stag()
315 	siw_mem_put(mem);  in siw_invalidate_stag()