84 	struct mlx5_core_srq *srq;  in mlx5_cmd_get_srq()  local
87 	srq = xa_load(&table->array, srqn);  in mlx5_cmd_get_srq()
88 	if (srq)  in mlx5_cmd_get_srq()
89 		refcount_inc(&srq->common.refcount);  in mlx5_cmd_get_srq()
92 	return srq;  in mlx5_cmd_get_srq()
95 static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in create_srq_cmd()  argument
126 		srq->srqn = MLX5_GET(create_srq_out, create_out, srqn);  in create_srq_cmd()
127 		srq->uid = in->uid;  in create_srq_cmd()
133 static int destroy_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)  in destroy_srq_cmd()  argument
138 	MLX5_SET(destroy_srq_in, in, srqn, srq->srqn);  in destroy_srq_cmd()
139 	MLX5_SET(destroy_srq_in, in, uid, srq->uid);  in destroy_srq_cmd()
144 static int arm_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in arm_srq_cmd()  argument
151 	MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);  in arm_srq_cmd()
153 	MLX5_SET(arm_rq_in, in, uid, srq->uid);  in arm_srq_cmd()
158 static int query_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in query_srq_cmd()  argument
171 	MLX5_SET(query_srq_in, in, srqn, srq->srqn);  in query_srq_cmd()
186 			      struct mlx5_core_srq *srq,  in create_xrc_srq_cmd()  argument
220 	srq->srqn = MLX5_GET(create_xrc_srq_out, create_out, xrc_srqn);  in create_xrc_srq_cmd()
221 	srq->uid = in->uid;  in create_xrc_srq_cmd()
228 			       struct mlx5_core_srq *srq)  in destroy_xrc_srq_cmd()  argument
233 	MLX5_SET(destroy_xrc_srq_in, in, xrc_srqn, srq->srqn);  in destroy_xrc_srq_cmd()
234 	MLX5_SET(destroy_xrc_srq_in, in, uid, srq->uid);  in destroy_xrc_srq_cmd()
239 static int arm_xrc_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in arm_xrc_srq_cmd()  argument
247 	MLX5_SET(arm_xrc_srq_in, in, xrc_srqn, srq->srqn);  in arm_xrc_srq_cmd()
249 	MLX5_SET(arm_xrc_srq_in, in, uid, srq->uid);  in arm_xrc_srq_cmd()
255 			     struct mlx5_core_srq *srq,  in query_xrc_srq_cmd()  argument
268 	MLX5_SET(query_xrc_srq_in, in, xrc_srqn, srq->srqn);  in query_xrc_srq_cmd()
285 static int create_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in create_rmp_cmd()  argument
318 		srq->srqn = MLX5_GET(create_rmp_out, create_out, rmpn);  in create_rmp_cmd()
319 		srq->uid = in->uid;  in create_rmp_cmd()
328 static int destroy_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)  in destroy_rmp_cmd()  argument
333 	MLX5_SET(destroy_rmp_in, in, rmpn, srq->srqn);  in destroy_rmp_cmd()
334 	MLX5_SET(destroy_rmp_in, in, uid, srq->uid);  in destroy_rmp_cmd()
338 static int arm_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in arm_rmp_cmd()  argument
365 	MLX5_SET(modify_rmp_in, in,	 rmpn,      srq->srqn);  in arm_rmp_cmd()
366 	MLX5_SET(modify_rmp_in, in, uid, srq->uid);  in arm_rmp_cmd()
380 static int query_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in query_rmp_cmd()  argument
401 	MLX5_SET(query_rmp_in, rmp_in, rmpn,   srq->srqn);  in query_rmp_cmd()
417 static int create_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in create_xrq_cmd()  argument
456 		srq->srqn = MLX5_GET(create_xrq_out, create_out, xrqn);  in create_xrq_cmd()
457 		srq->uid = in->uid;  in create_xrq_cmd()
463 static int destroy_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)  in destroy_xrq_cmd()  argument
468 	MLX5_SET(destroy_xrq_in, in, xrqn, srq->srqn);  in destroy_xrq_cmd()
469 	MLX5_SET(destroy_xrq_in, in, uid, srq->uid);  in destroy_xrq_cmd()
475 		       struct mlx5_core_srq *srq,  in arm_xrq_cmd()  argument
482 	MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);  in arm_xrq_cmd()
484 	MLX5_SET(arm_rq_in, in, uid, srq->uid);  in arm_xrq_cmd()
489 static int query_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in query_xrq_cmd()  argument
503 	MLX5_SET(query_xrq_in, in, xrqn, srq->srqn);  in query_xrq_cmd()
528 static int create_srq_split(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in create_srq_split()  argument
532 		return create_srq_cmd(dev, srq, in);  in create_srq_split()
533 	switch (srq->common.res) {  in create_srq_split()
535 		return create_xrc_srq_cmd(dev, srq, in);  in create_srq_split()
537 		return create_xrq_cmd(dev, srq, in);  in create_srq_split()
539 		return create_rmp_cmd(dev, srq, in);  in create_srq_split()
543 static int destroy_srq_split(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)  in destroy_srq_split()  argument
546 		return destroy_srq_cmd(dev, srq);  in destroy_srq_split()
547 	switch (srq->common.res) {  in destroy_srq_split()
549 		return destroy_xrc_srq_cmd(dev, srq);  in destroy_srq_split()
551 		return destroy_xrq_cmd(dev, srq);  in destroy_srq_split()
553 		return destroy_rmp_cmd(dev, srq);  in destroy_srq_split()
557 int mlx5_cmd_create_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in mlx5_cmd_create_srq()  argument
565 		srq->common.res = MLX5_RES_XSRQ;  in mlx5_cmd_create_srq()
568 		srq->common.res = MLX5_RES_XRQ;  in mlx5_cmd_create_srq()
571 		srq->common.res = MLX5_RES_SRQ;  in mlx5_cmd_create_srq()
574 	err = create_srq_split(dev, srq, in);  in mlx5_cmd_create_srq()
578 	refcount_set(&srq->common.refcount, 1);  in mlx5_cmd_create_srq()
579 	init_completion(&srq->common.free);  in mlx5_cmd_create_srq()
581 	err = xa_err(xa_store_irq(&table->array, srq->srqn, srq, GFP_KERNEL));  in mlx5_cmd_create_srq()
588 	destroy_srq_split(dev, srq);  in mlx5_cmd_create_srq()
593 int mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)  in mlx5_cmd_destroy_srq()  argument
600 	tmp = xa_cmpxchg_irq(&table->array, srq->srqn, srq, XA_ZERO_ENTRY, 0);  in mlx5_cmd_destroy_srq()
601 	if (WARN_ON(tmp != srq))  in mlx5_cmd_destroy_srq()
604 	err = destroy_srq_split(dev, srq);  in mlx5_cmd_destroy_srq()
611 		xa_cmpxchg_irq(&table->array, srq->srqn, XA_ZERO_ENTRY, srq, 0);  in mlx5_cmd_destroy_srq()
614 	xa_erase_irq(&table->array, srq->srqn);  in mlx5_cmd_destroy_srq()
616 	mlx5_core_res_put(&srq->common);  in mlx5_cmd_destroy_srq()
617 	wait_for_completion(&srq->common.free);  in mlx5_cmd_destroy_srq()
621 int mlx5_cmd_query_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in mlx5_cmd_query_srq()  argument
625 		return query_srq_cmd(dev, srq, out);  in mlx5_cmd_query_srq()
626 	switch (srq->common.res) {  in mlx5_cmd_query_srq()
628 		return query_xrc_srq_cmd(dev, srq, out);  in mlx5_cmd_query_srq()
630 		return query_xrq_cmd(dev, srq, out);  in mlx5_cmd_query_srq()
632 		return query_rmp_cmd(dev, srq, out);  in mlx5_cmd_query_srq()
636 int mlx5_cmd_arm_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,  in mlx5_cmd_arm_srq()  argument
640 		return arm_srq_cmd(dev, srq, lwm, is_srq);  in mlx5_cmd_arm_srq()
641 	switch (srq->common.res) {  in mlx5_cmd_arm_srq()
643 		return arm_xrc_srq_cmd(dev, srq, lwm);  in mlx5_cmd_arm_srq()
645 		return arm_xrq_cmd(dev, srq, lwm);  in mlx5_cmd_arm_srq()
647 		return arm_rmp_cmd(dev, srq, lwm);  in mlx5_cmd_arm_srq()
655 	struct mlx5_core_srq *srq;  in srq_event_notifier()  local
669 	srq = xa_load(&table->array, srqn);  in srq_event_notifier()
670 	if (srq)  in srq_event_notifier()
671 		refcount_inc(&srq->common.refcount);  in srq_event_notifier()
674 	if (!srq)  in srq_event_notifier()
677 	srq->event(srq, eqe->type);  in srq_event_notifier()
679 	mlx5_core_res_put(&srq->common);  in srq_event_notifier()