58 static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp,  in uverbs_response()  argument
63 	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))  in uverbs_response()
65 			attrs, UVERBS_ATTR_CORE_OUT, resp, resp_len);  in uverbs_response()
67 	if (copy_to_user(attrs->ucore.outbuf, resp,  in uverbs_response()
68 			 min(attrs->ucore.outlen, resp_len)))  in uverbs_response()
71 	if (resp_len < attrs->ucore.outlen) {  in uverbs_response()
76 		ret = clear_user(attrs->ucore.outbuf + resp_len,  in uverbs_response()
77 				 attrs->ucore.outlen - resp_len);  in uverbs_response()
91 static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req,  in uverbs_request()  argument
94 	if (copy_from_user(req, attrs->ucore.inbuf,  in uverbs_request()
95 			   min(attrs->ucore.inlen, req_len)))  in uverbs_request()
98 	if (attrs->ucore.inlen < req_len) {  in uverbs_request()
99 		memset(req + attrs->ucore.inlen, 0,  in uverbs_request()
100 		       req_len - attrs->ucore.inlen);  in uverbs_request()
101 	} else if (attrs->ucore.inlen > req_len) {  in uverbs_request()
102 		if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len,  in uverbs_request()
103 					  attrs->ucore.inlen - req_len))  in uverbs_request()
115 static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs,  in uverbs_response_length()  argument
118 	return min_t(size_t, attrs->ucore.outlen, resp_len);  in uverbs_response_length()
130 static int uverbs_request_start(struct uverbs_attr_bundle *attrs,  in uverbs_request_start()  argument
135 	if (attrs->ucore.inlen < req_len)  in uverbs_request_start()
138 	if (copy_from_user(req, attrs->ucore.inbuf, req_len))  in uverbs_request_start()
141 	iter->cur = attrs->ucore.inbuf + req_len;  in uverbs_request_start()
142 	iter->end = attrs->ucore.inbuf + attrs->ucore.inlen;  in uverbs_request_start()
182 struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)  in uverbs_get_cleared_udata()  argument
184 	attrs->driver_udata = (struct ib_udata){};  in uverbs_get_cleared_udata()
185 	return &attrs->driver_udata;  in uverbs_get_cleared_udata()
189 _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)  in _ib_uverbs_lookup_comp_file()  argument
192 					       fd, attrs);  in _ib_uverbs_lookup_comp_file()
206 int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs)  in ib_alloc_ucontext()  argument
208 	struct ib_uverbs_file *ufile = attrs->ufile;  in ib_alloc_ucontext()
227 	attrs->context = ucontext;  in ib_alloc_ucontext()
231 int ib_init_ucontext(struct uverbs_attr_bundle *attrs)  in ib_init_ucontext()  argument
233 	struct ib_ucontext *ucontext = attrs->context;  in ib_init_ucontext()
234 	struct ib_uverbs_file *file = attrs->ufile;  in ib_init_ucontext()
251 						   &attrs->driver_udata);  in ib_init_ucontext()
276 static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)  in ib_uverbs_get_context()  argument
284 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_get_context()
288 	ret = ib_alloc_ucontext(attrs);  in ib_uverbs_get_context()
292 	uobj = uobj_alloc(UVERBS_OBJECT_ASYNC_EVENT, attrs, &ib_dev);  in ib_uverbs_get_context()
299 		.num_comp_vectors = attrs->ufile->device->num_comp_vectors,  in ib_uverbs_get_context()
302 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_get_context()
306 	ret = ib_init_ucontext(attrs);  in ib_uverbs_get_context()
312 	rdma_alloc_commit_uobject(uobj, attrs);  in ib_uverbs_get_context()
316 	rdma_alloc_abort_uobject(uobj, attrs, false);  in ib_uverbs_get_context()
318 	rdma_restrack_put(&attrs->context->res);  in ib_uverbs_get_context()
319 	kfree(attrs->context);  in ib_uverbs_get_context()
320 	attrs->context = NULL;  in ib_uverbs_get_context()
370 static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)  in ib_uverbs_query_device()  argument
377 	ucontext = ib_uverbs_get_ucontext(attrs);  in ib_uverbs_query_device()
381 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_query_device()
386 	copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs);  in ib_uverbs_query_device()
388 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_query_device()
391 static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)  in ib_uverbs_query_port()  argument
400 	ucontext = ib_uverbs_get_ucontext(attrs);  in ib_uverbs_query_port()
405 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_query_port()
416 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_query_port()
419 static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)  in ib_uverbs_alloc_pd()  argument
428 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_alloc_pd()
432 	uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev);  in ib_uverbs_alloc_pd()
449 	ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);  in ib_uverbs_alloc_pd()
455 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_alloc_pd()
458 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_alloc_pd()
464 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_alloc_pd()
468 static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs)  in ib_uverbs_dealloc_pd()  argument
473 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_dealloc_pd()
477 	return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_dealloc_pd()
565 static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)  in ib_uverbs_open_xrcd()  argument
567 	struct ib_uverbs_device *ibudev = attrs->ufile->device;  in ib_uverbs_open_xrcd()
578 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_open_xrcd()
606 	obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs,  in ib_uverbs_open_xrcd()
614 		xrcd = ib_alloc_xrcd_user(ib_dev, inode, &attrs->driver_udata);  in ib_uverbs_open_xrcd()
639 	uobj_finalize_uobj_create(&obj->uobject, attrs);  in ib_uverbs_open_xrcd()
642 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_open_xrcd()
645 	ib_dealloc_xrcd_user(xrcd, uverbs_get_cleared_udata(attrs));  in ib_uverbs_open_xrcd()
648 	uobj_alloc_abort(&obj->uobject, attrs);  in ib_uverbs_open_xrcd()
659 static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)  in ib_uverbs_close_xrcd()  argument
664 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_close_xrcd()
668 	return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);  in ib_uverbs_close_xrcd()
673 			   struct uverbs_attr_bundle *attrs)  in ib_uverbs_dealloc_xrcd()  argument
677 	struct ib_uverbs_device *dev = attrs->ufile->device;  in ib_uverbs_dealloc_xrcd()
683 	ret = ib_dealloc_xrcd_user(xrcd, &attrs->driver_udata);  in ib_uverbs_dealloc_xrcd()
695 static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)  in ib_uverbs_reg_mr()  argument
705 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_reg_mr()
712 	uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);  in ib_uverbs_reg_mr()
720 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_reg_mr()
728 					 &attrs->driver_udata);  in ib_uverbs_reg_mr()
750 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_reg_mr()
755 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_reg_mr()
760 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_reg_mr()
764 static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)  in ib_uverbs_rereg_mr()  argument
777 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_rereg_mr()
791 	uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);  in ib_uverbs_rereg_mr()
811 					   attrs);  in ib_uverbs_rereg_mr()
824 	new_uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);  in ib_uverbs_rereg_mr()
832 					   &attrs->driver_udata);  in ib_uverbs_rereg_mr()
853 		rdma_assign_uobject(uobj, new_uobj, attrs);  in ib_uverbs_rereg_mr()
854 		rdma_alloc_commit_uobject(new_uobj, attrs);  in ib_uverbs_rereg_mr()
875 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_rereg_mr()
879 		uobj_alloc_abort(new_uobj, attrs);  in ib_uverbs_rereg_mr()
891 static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)  in ib_uverbs_dereg_mr()  argument
896 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_dereg_mr()
900 	return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);  in ib_uverbs_dereg_mr()
903 static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)  in ib_uverbs_alloc_mw()  argument
913 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_alloc_mw()
917 	uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev);  in ib_uverbs_alloc_mw()
921 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_alloc_mw()
943 	ret = pd->device->ops.alloc_mw(mw, &attrs->driver_udata);  in ib_uverbs_alloc_mw()
951 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_alloc_mw()
955 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_alloc_mw()
962 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_alloc_mw()
966 static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs)  in ib_uverbs_dealloc_mw()  argument
971 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_dealloc_mw()
975 	return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs);  in ib_uverbs_dealloc_mw()
978 static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_comp_channel()  argument
987 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_comp_channel()
991 	uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev);  in ib_uverbs_create_comp_channel()
998 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_create_comp_channel()
1001 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_create_comp_channel()
1004 static int create_cq(struct uverbs_attr_bundle *attrs,  in create_cq()  argument
1015 	if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)  in create_cq()
1018 	obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,  in create_cq()
1024 		ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);  in create_cq()
1054 	ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);  in create_cq()
1060 	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);  in create_cq()
1063 	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);  in create_cq()
1067 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in create_cq()
1068 	return uverbs_response(attrs, &resp, sizeof(resp));  in create_cq()
1077 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in create_cq()
1081 static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_cq()  argument
1087 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_cq()
1097 	return create_cq(attrs, &cmd_ex);  in ib_uverbs_create_cq()
1100 static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_cq()  argument
1105 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_cq()
1115 	return create_cq(attrs, &cmd);  in ib_uverbs_ex_create_cq()
1118 static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_resize_cq()  argument
1125 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_resize_cq()
1129 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_resize_cq()
1133 	ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata);  in ib_uverbs_resize_cq()
1139 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_resize_cq()
1177 static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_poll_cq()  argument
1187 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_poll_cq()
1191 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_poll_cq()
1196 	header_ptr = attrs->ucore.outbuf;  in ib_uverbs_poll_cq()
1221 	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))  in ib_uverbs_poll_cq()
1222 		ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT);  in ib_uverbs_poll_cq()
1230 static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_req_notify_cq()  argument
1236 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_req_notify_cq()
1240 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_req_notify_cq()
1252 static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_destroy_cq()  argument
1260 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_destroy_cq()
1264 	uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_destroy_cq()
1275 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_destroy_cq()
1278 static int create_qp(struct uverbs_attr_bundle *attrs,  in create_qp()  argument
1312 	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,  in create_qp()
1323 					    cmd->rwq_ind_tbl_handle, attrs);  in create_qp()
1342 					  attrs);  in create_qp()
1362 							cmd->srq_handle, attrs);  in create_qp()
1373 						cmd->recv_cq_handle, attrs);  in create_qp()
1384 						cmd->send_cq_handle, attrs);  in create_qp()
1388 				       attrs);  in create_qp()
1438 	qp = ib_create_qp_user(device, pd, &attr, &attrs->driver_udata, obj,  in create_qp()
1447 	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);  in create_qp()
1471 	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);  in create_qp()
1480 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in create_qp()
1481 	return uverbs_response(attrs, &resp, sizeof(resp));  in create_qp()
1500 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in create_qp()
1504 static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_qp()  argument
1510 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_qp()
1529 	return create_qp(attrs, &cmd_ex);  in ib_uverbs_create_qp()
1532 static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_qp()  argument
1537 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_qp()
1547 	return create_qp(attrs, &cmd);  in ib_uverbs_ex_create_qp()
1550 static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_open_qp()  argument
1562 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_open_qp()
1566 	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,  in ib_uverbs_open_qp()
1571 	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs);  in ib_uverbs_open_qp()
1603 	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);  in ib_uverbs_open_qp()
1607 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_open_qp()
1612 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in ib_uverbs_open_qp()
1638 static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_query_qp()  argument
1647 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_query_qp()
1658 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_query_qp()
1706 	ret = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_query_qp()
1752 static int modify_qp(struct uverbs_attr_bundle *attrs,  in modify_qp()  argument
1764 			       attrs);  in modify_qp()
1906 				      &attrs->driver_udata);  in modify_qp()
1917 static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_modify_qp()  argument
1922 	ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base));  in ib_uverbs_modify_qp()
1929 	return modify_qp(attrs, &cmd);  in ib_uverbs_modify_qp()
1932 static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_modify_qp()  argument
1936 		.response_length = uverbs_response_length(attrs, sizeof(resp))  in ib_uverbs_ex_modify_qp()
1940 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_modify_qp()
1951 	ret = modify_qp(attrs, &cmd);  in ib_uverbs_ex_modify_qp()
1955 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_modify_qp()
1958 static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)  in ib_uverbs_destroy_qp()  argument
1966 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_destroy_qp()
1970 	uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_destroy_qp()
1980 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_destroy_qp()
1994 static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)  in ib_uverbs_post_send()  argument
2010 	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_post_send()
2028 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_post_send()
2066 						   user_wr->wr.ud.ah, attrs);  in ib_uverbs_post_send()
2166 	ret2 = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_post_send()
2286 static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)  in ib_uverbs_post_recv()  argument
2296 	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_post_recv()
2305 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_post_recv()
2324 	ret2 = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_post_recv()
2337 static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)  in ib_uverbs_post_srq_recv()  argument
2347 	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_post_srq_recv()
2356 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);  in ib_uverbs_post_srq_recv()
2375 	ret2 = uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_post_srq_recv()
2389 static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_ah()  argument
2400 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_ah()
2404 	uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev);  in ib_uverbs_create_ah()
2413 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_create_ah()
2437 	ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata);  in ib_uverbs_create_ah()
2447 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_create_ah()
2450 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_create_ah()
2455 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_create_ah()
2459 static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs)  in ib_uverbs_destroy_ah()  argument
2464 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_destroy_ah()
2468 	return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs);  in ib_uverbs_destroy_ah()
2471 static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)  in ib_uverbs_attach_mcast()  argument
2479 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_attach_mcast()
2483 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_attach_mcast()
2520 static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)  in ib_uverbs_detach_mcast()  argument
2529 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_detach_mcast()
2533 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_detach_mcast()
2639 static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,  in kern_spec_to_ib_spec_action()  argument
2668 							attrs);  in kern_spec_to_ib_spec_action()
2686 					  attrs);  in kern_spec_to_ib_spec_action()
2866 static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs,  in kern_spec_to_ib_spec()  argument
2875 		return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec,  in kern_spec_to_ib_spec()
2881 static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_wq()  argument
2893 	err = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_wq()
2900 	obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs,  in ib_uverbs_ex_create_wq()
2905 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);  in ib_uverbs_ex_create_wq()
2911 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_ex_create_wq()
2926 	wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);  in ib_uverbs_ex_create_wq()
2941 	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);  in ib_uverbs_ex_create_wq()
2948 	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);  in ib_uverbs_ex_create_wq()
2954 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in ib_uverbs_ex_create_wq()
2955 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_create_wq()
2963 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in ib_uverbs_ex_create_wq()
2968 static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_destroy_wq()  argument
2976 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_destroy_wq()
2983 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in ib_uverbs_ex_destroy_wq()
2984 	uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);  in ib_uverbs_ex_destroy_wq()
2993 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_destroy_wq()
2996 static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_modify_wq()  argument
3003 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_modify_wq()
3013 	wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);  in ib_uverbs_ex_modify_wq()
3041 					&attrs->driver_udata);  in ib_uverbs_ex_modify_wq()
3047 static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_rwq_ind_table()  argument
3063 	err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_rwq_ind_table()
3097 				       wqs_handles[num_read_wqs], attrs);  in ib_uverbs_ex_create_rwq_ind_table()
3107 	uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);  in ib_uverbs_ex_create_rwq_ind_table()
3130 					       &attrs->driver_udata);  in ib_uverbs_ex_create_rwq_ind_table()
3138 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_ex_create_rwq_ind_table()
3142 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in ib_uverbs_ex_create_rwq_ind_table()
3143 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_create_rwq_ind_table()
3148 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_ex_create_rwq_ind_table()
3161 static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_destroy_rwq_ind_table()  argument
3166 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_destroy_rwq_ind_table()
3174 				    cmd.ind_tbl_handle, attrs);  in ib_uverbs_ex_destroy_rwq_ind_table()
3177 static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_create_flow()  argument
3194 	err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));  in ib_uverbs_ex_create_flow()
3242 	uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev);  in ib_uverbs_ex_create_flow()
3253 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);  in ib_uverbs_ex_create_flow()
3290 				attrs, (struct ib_uverbs_flow_spec *)kern_spec,  in ib_uverbs_ex_create_flow()
3309 					      &attrs->driver_udata);  in ib_uverbs_ex_create_flow()
3324 	uobj_finalize_uobj_create(uobj, attrs);  in ib_uverbs_ex_create_flow()
3327 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_create_flow()
3337 	uobj_alloc_abort(uobj, attrs);  in ib_uverbs_ex_create_flow()
3344 static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_destroy_flow()  argument
3349 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_destroy_flow()
3356 	return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs);  in ib_uverbs_ex_destroy_flow()
3359 static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,  in __uverbs_create_xsrq()  argument
3372 	obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,  in __uverbs_create_xsrq()
3382 					  attrs);  in __uverbs_create_xsrq()
3400 						cmd->cq_handle, attrs);  in __uverbs_create_xsrq()
3407 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs);  in __uverbs_create_xsrq()
3430 	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);  in __uverbs_create_xsrq()
3445 	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);  in __uverbs_create_xsrq()
3450 	return uverbs_response(attrs, &resp, sizeof(resp));  in __uverbs_create_xsrq()
3466 	uobj_alloc_abort(&obj->uevent.uobject, attrs);  in __uverbs_create_xsrq()
3470 static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_srq()  argument
3476 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_srq()
3489 	return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata);  in ib_uverbs_create_srq()
3492 static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_create_xsrq()  argument
3497 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_create_xsrq()
3501 	return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata);  in ib_uverbs_create_xsrq()
3504 static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_modify_srq()  argument
3511 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_modify_srq()
3515 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);  in ib_uverbs_modify_srq()
3523 					  &attrs->driver_udata);  in ib_uverbs_modify_srq()
3531 static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_query_srq()  argument
3539 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_query_srq()
3543 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);  in ib_uverbs_query_srq()
3561 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_query_srq()
3564 static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_destroy_srq()  argument
3572 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_destroy_srq()
3576 	uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);  in ib_uverbs_destroy_srq()
3586 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_destroy_srq()
3589 static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_query_device()  argument
3598 	ucontext = ib_uverbs_get_ucontext(attrs);  in ib_uverbs_ex_query_device()
3603 	err = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_query_device()
3613 	err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata);  in ib_uverbs_ex_query_device()
3648 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));  in ib_uverbs_ex_query_device()
3650 	return uverbs_response(attrs, &resp, sizeof(resp));  in ib_uverbs_ex_query_device()
3653 static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)  in ib_uverbs_ex_modify_cq()  argument
3659 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));  in ib_uverbs_ex_modify_cq()
3669 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);  in ib_uverbs_ex_modify_cq()