Lines Matching full:ndev
114 struct mlx5_vdpa_net *ndev; member
146 static void free_resources(struct mlx5_vdpa_net *ndev);
147 static void init_mvqs(struct mlx5_vdpa_net *ndev);
148 static int setup_driver(struct mlx5_vdpa_net *ndev);
149 static void teardown_driver(struct mlx5_vdpa_net *ndev);
237 static int create_tis(struct mlx5_vdpa_net *ndev) in create_tis() argument
239 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; in create_tis()
245 MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn); in create_tis()
246 err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn); in create_tis()
253 static void destroy_tis(struct mlx5_vdpa_net *ndev) in destroy_tis() argument
255 mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn); in destroy_tis()
261 static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent) in cq_frag_buf_alloc() argument
268 err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf, in cq_frag_buf_alloc()
269 ndev->mvdev.mdev->priv.numa_node); in cq_frag_buf_alloc()
281 static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size) in umem_frag_buf_alloc() argument
285 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf, in umem_frag_buf_alloc()
286 ndev->mvdev.mdev->priv.numa_node); in umem_frag_buf_alloc()
289 static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf) in cq_frag_buf_free() argument
291 mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf); in cq_frag_buf_free()
329 static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in, in qp_prepare() argument
337 MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid); in qp_prepare()
351 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); in qp_prepare()
353 MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index); in qp_prepare()
363 static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent) in rq_buf_alloc() argument
365 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, in rq_buf_alloc()
367 ndev->mvdev.mdev->priv.numa_node); in rq_buf_alloc()
370 static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) in rq_buf_free() argument
372 mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf); in rq_buf_free()
375 static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, in qp_create() argument
378 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; in qp_create()
387 err = rq_buf_alloc(ndev, vqp, mvq->num_ent); in qp_create()
391 err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db); in qp_create()
403 qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent); in qp_create()
407 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); in qp_create()
417 vqp->mqp.uid = ndev->mvdev.res.uid; in qp_create()
427 mlx5_db_free(ndev->mvdev.mdev, &vqp->db); in qp_create()
430 rq_buf_free(ndev, vqp); in qp_create()
435 static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) in qp_destroy() argument
441 MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid); in qp_destroy()
442 if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in)) in qp_destroy()
443 mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn); in qp_destroy()
445 mlx5_db_free(ndev->mvdev.mdev, &vqp->db); in qp_destroy()
446 rq_buf_free(ndev, vqp); in qp_destroy()
483 struct mlx5_vdpa_net *ndev = mvq->ndev; in mlx5_vdpa_cq_comp() local
484 void __iomem *uar_page = ndev->mvdev.res.uar->map; in mlx5_vdpa_cq_comp()
507 static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent) in cq_create() argument
509 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; in cq_create()
510 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; in cq_create()
511 void __iomem *uar_page = ndev->mvdev.res.uar->map; in cq_create()
529 err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent); in cq_create()
543 MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid); in cq_create()
559 MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index); in cq_create()
578 cq_frag_buf_free(ndev, &vcq->buf); in cq_create()
580 mlx5_db_free(ndev->mvdev.mdev, &vcq->db); in cq_create()
584 static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx) in cq_destroy() argument
586 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; in cq_destroy()
587 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; in cq_destroy()
591 mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn); in cq_destroy()
594 cq_frag_buf_free(ndev, &vcq->buf); in cq_destroy()
595 mlx5_db_free(ndev->mvdev.mdev, &vcq->db); in cq_destroy()
598 static void set_umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num, in set_umem_size() argument
601 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; in set_umem_size()
625 static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem) in umem_frag_buf_free() argument
627 mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf); in umem_frag_buf_free()
630 static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) in create_umem() argument
640 set_umem_size(ndev, mvq, num, &umem); in create_umem()
641 err = umem_frag_buf_alloc(ndev, umem, umem->size); in create_umem()
654 MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid); in create_umem()
662 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); in create_umem()
664 mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err); in create_umem()
676 umem_frag_buf_free(ndev, umem); in create_umem()
680 static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) in umem_destroy() argument
700 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) in umem_destroy()
703 umem_frag_buf_free(ndev, umem); in umem_destroy()
706 static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in umems_create() argument
712 err = create_umem(ndev, mvq, num); in umems_create()
720 umem_destroy(ndev, mvq, num); in umems_create()
725 static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in umems_destroy() argument
730 umem_destroy(ndev, mvq, num); in umems_destroy()
733 static int get_queue_type(struct mlx5_vdpa_net *ndev) in get_queue_type() argument
737 type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type); in get_queue_type()
761 static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in create_virtqueue() argument
771 err = umems_create(ndev, mvq); in create_virtqueue()
785 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); in create_virtqueue()
791 get_features_12_3(ndev->mvdev.actual_features)); in create_virtqueue()
793 MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev)); in create_virtqueue()
796 MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); in create_virtqueue()
803 !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); in create_virtqueue()
807 MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey.key); in create_virtqueue()
814 MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn); in create_virtqueue()
816 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); in create_virtqueue()
828 umems_destroy(ndev, mvq); in create_virtqueue()
832 static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in destroy_virtqueue() argument
840 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid); in destroy_virtqueue()
843 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) { in destroy_virtqueue()
844 mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); in destroy_virtqueue()
847 umems_destroy(ndev, mvq); in destroy_virtqueue()
860 static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out, in alloc_inout() argument
876 MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid); in alloc_inout()
888 MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid); in alloc_inout()
905 MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid); in alloc_inout()
923 MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid); in alloc_inout()
955 static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd) in modify_qp() argument
963 alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw)); in modify_qp()
967 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen); in modify_qp()
972 static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in connect_qps() argument
976 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP); in connect_qps()
980 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP); in connect_qps()
984 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP); in connect_qps()
988 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP); in connect_qps()
992 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP); in connect_qps()
996 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP); in connect_qps()
1000 return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP); in connect_qps()
1009 static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, in query_virtqueue() argument
1028 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); in query_virtqueue()
1029 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen); in query_virtqueue()
1046 static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) in modify_virtqueue() argument
1064 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); in modify_virtqueue()
1070 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); in modify_virtqueue()
1078 static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in setup_vq() argument
1087 mlx5_vdpa_warn(&ndev->mvdev, "attempt re init\n"); in setup_vq()
1091 err = cq_create(ndev, idx, mvq->num_ent); in setup_vq()
1095 err = qp_create(ndev, mvq, &mvq->fwqp); in setup_vq()
1099 err = qp_create(ndev, mvq, &mvq->vqqp); in setup_vq()
1103 err = connect_qps(ndev, mvq); in setup_vq()
1107 err = create_virtqueue(ndev, mvq); in setup_vq()
1112 err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); in setup_vq()
1114 mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", in setup_vq()
1124 qp_destroy(ndev, &mvq->vqqp); in setup_vq()
1126 qp_destroy(ndev, &mvq->fwqp); in setup_vq()
1128 cq_destroy(ndev, idx); in setup_vq()
1132 static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in suspend_vq() argument
1142 if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)) in suspend_vq()
1143 mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n"); in suspend_vq()
1145 if (query_virtqueue(ndev, mvq, &attr)) { in suspend_vq()
1146 mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n"); in suspend_vq()
1153 static void suspend_vqs(struct mlx5_vdpa_net *ndev) in suspend_vqs() argument
1158 suspend_vq(ndev, &ndev->vqs[i]); in suspend_vqs()
1161 static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in teardown_vq() argument
1166 suspend_vq(ndev, mvq); in teardown_vq()
1167 destroy_virtqueue(ndev, mvq); in teardown_vq()
1168 qp_destroy(ndev, &mvq->vqqp); in teardown_vq()
1169 qp_destroy(ndev, &mvq->fwqp); in teardown_vq()
1170 cq_destroy(ndev, mvq->index); in teardown_vq()
1174 static int create_rqt(struct mlx5_vdpa_net *ndev) in create_rqt() argument
1184 log_max_rqt = min_t(int, 1, MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size)); in create_rqt()
1193 MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid); in create_rqt()
1200 for (i = 0, j = 0; j < ndev->mvdev.max_vqs; j++) { in create_rqt()
1201 if (!ndev->vqs[j].initialized) in create_rqt()
1204 if (!vq_is_tx(ndev->vqs[j].index)) { in create_rqt()
1205 list[i] = cpu_to_be32(ndev->vqs[j].virtq_id); in create_rqt()
1210 err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn); in create_rqt()
1218 static void destroy_rqt(struct mlx5_vdpa_net *ndev) in destroy_rqt() argument
1220 mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn); in destroy_rqt()
1223 static int create_tir(struct mlx5_vdpa_net *ndev) in create_tir() argument
1243 MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid); in create_tir()
1257 MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn); in create_tir()
1258 MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn); in create_tir()
1260 err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn); in create_tir()
1265 static void destroy_tir(struct mlx5_vdpa_net *ndev) in destroy_tir() argument
1267 mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn); in destroy_tir()
1270 static int add_fwd_to_tir(struct mlx5_vdpa_net *ndev) in add_fwd_to_tir() argument
1282 ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS); in add_fwd_to_tir()
1284 mlx5_vdpa_warn(&ndev->mvdev, "get flow namespace\n"); in add_fwd_to_tir()
1288 ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); in add_fwd_to_tir()
1289 if (IS_ERR(ndev->rxft)) in add_fwd_to_tir()
1290 return PTR_ERR(ndev->rxft); in add_fwd_to_tir()
1292 ndev->rx_counter = mlx5_fc_create(ndev->mvdev.mdev, false); in add_fwd_to_tir()
1293 if (IS_ERR(ndev->rx_counter)) { in add_fwd_to_tir()
1294 err = PTR_ERR(ndev->rx_counter); in add_fwd_to_tir()
1300 dest[0].tir_num = ndev->res.tirn; in add_fwd_to_tir()
1302 dest[1].counter_id = mlx5_fc_id(ndev->rx_counter); in add_fwd_to_tir()
1303 ndev->rx_rule = mlx5_add_flow_rules(ndev->rxft, NULL, &flow_act, dest, 2); in add_fwd_to_tir()
1304 if (IS_ERR(ndev->rx_rule)) { in add_fwd_to_tir()
1305 err = PTR_ERR(ndev->rx_rule); in add_fwd_to_tir()
1306 ndev->rx_rule = NULL; in add_fwd_to_tir()
1313 mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter); in add_fwd_to_tir()
1315 mlx5_destroy_flow_table(ndev->rxft); in add_fwd_to_tir()
1319 static void remove_fwd_to_tir(struct mlx5_vdpa_net *ndev) in remove_fwd_to_tir() argument
1321 if (!ndev->rx_rule) in remove_fwd_to_tir()
1324 mlx5_del_flow_rules(ndev->rx_rule); in remove_fwd_to_tir()
1325 mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter); in remove_fwd_to_tir()
1326 mlx5_destroy_flow_table(ndev->rxft); in remove_fwd_to_tir()
1328 ndev->rx_rule = NULL; in remove_fwd_to_tir()
1334 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_kick_vq() local
1335 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; in mlx5_vdpa_kick_vq()
1340 iowrite16(idx, ndev->mvdev.res.kick_addr); in mlx5_vdpa_kick_vq()
1347 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_vq_address() local
1348 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; in mlx5_vdpa_set_vq_address()
1359 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_vq_num() local
1362 mvq = &ndev->vqs[idx]; in mlx5_vdpa_set_vq_num()
1369 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_vq_cb() local
1370 struct mlx5_vdpa_virtqueue *vq = &ndev->vqs[idx]; in mlx5_vdpa_set_vq_cb()
1378 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_vq_ready() local
1379 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; in mlx5_vdpa_set_vq_ready()
1382 suspend_vq(ndev, mvq); in mlx5_vdpa_set_vq_ready()
1390 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_get_vq_ready() local
1391 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; in mlx5_vdpa_get_vq_ready()
1400 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_vq_state() local
1401 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; in mlx5_vdpa_set_vq_state()
1416 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_get_vq_state() local
1417 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; in mlx5_vdpa_get_vq_state()
1434 err = query_virtqueue(ndev, mvq, &attr); in mlx5_vdpa_get_vq_state()
1473 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_get_features() local
1477 ndev->mvdev.mlx_features = mlx_to_vritio_features(dev_features); in mlx5_vdpa_get_features()
1479 ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_F_VERSION_1); in mlx5_vdpa_get_features()
1480 ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_F_ACCESS_PLATFORM); in mlx5_vdpa_get_features()
1481 print_features(mvdev, ndev->mvdev.mlx_features, false); in mlx5_vdpa_get_features()
1482 return ndev->mvdev.mlx_features; in mlx5_vdpa_get_features()
1493 static int setup_virtqueues(struct mlx5_vdpa_net *ndev) in setup_virtqueues() argument
1498 for (i = 0; i < 2 * mlx5_vdpa_max_qps(ndev->mvdev.max_vqs); i++) { in setup_virtqueues()
1499 err = setup_vq(ndev, &ndev->vqs[i]); in setup_virtqueues()
1508 teardown_vq(ndev, &ndev->vqs[i]); in setup_virtqueues()
1513 static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) in teardown_virtqueues() argument
1518 for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) { in teardown_virtqueues()
1519 mvq = &ndev->vqs[i]; in teardown_virtqueues()
1523 teardown_vq(ndev, mvq); in teardown_virtqueues()
1542 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_features() local
1551 ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; in mlx5_vdpa_set_features()
1552 ndev->config.mtu = cpu_to_mlx5vdpa16(mvdev, ndev->mtu); in mlx5_vdpa_set_features()
1553 ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); in mlx5_vdpa_set_features()
1582 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_get_status() local
1584 print_status(mvdev, ndev->mvdev.status, false); in mlx5_vdpa_get_status()
1585 return ndev->mvdev.status; in mlx5_vdpa_get_status()
1588 static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in save_channel_info() argument
1597 err = query_virtqueue(ndev, mvq, &attr); in save_channel_info()
1613 static int save_channels_info(struct mlx5_vdpa_net *ndev) in save_channels_info() argument
1617 for (i = 0; i < ndev->mvdev.max_vqs; i++) { in save_channels_info()
1618 memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri)); in save_channels_info()
1619 save_channel_info(ndev, &ndev->vqs[i]); in save_channels_info()
1624 static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev) in mlx5_clear_vqs() argument
1628 for (i = 0; i < ndev->mvdev.max_vqs; i++) in mlx5_clear_vqs()
1629 memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); in mlx5_clear_vqs()
1632 static void restore_channels_info(struct mlx5_vdpa_net *ndev) in restore_channels_info() argument
1638 mlx5_clear_vqs(ndev); in restore_channels_info()
1639 init_mvqs(ndev); in restore_channels_info()
1640 for (i = 0; i < ndev->mvdev.max_vqs; i++) { in restore_channels_info()
1641 mvq = &ndev->vqs[i]; in restore_channels_info()
1657 static int mlx5_vdpa_change_map(struct mlx5_vdpa_net *ndev, struct vhost_iotlb *iotlb) in mlx5_vdpa_change_map() argument
1661 suspend_vqs(ndev); in mlx5_vdpa_change_map()
1662 err = save_channels_info(ndev); in mlx5_vdpa_change_map()
1666 teardown_driver(ndev); in mlx5_vdpa_change_map()
1667 mlx5_vdpa_destroy_mr(&ndev->mvdev); in mlx5_vdpa_change_map()
1668 err = mlx5_vdpa_create_mr(&ndev->mvdev, iotlb); in mlx5_vdpa_change_map()
1672 if (!(ndev->mvdev.status & VIRTIO_CONFIG_S_DRIVER_OK)) in mlx5_vdpa_change_map()
1675 restore_channels_info(ndev); in mlx5_vdpa_change_map()
1676 err = setup_driver(ndev); in mlx5_vdpa_change_map()
1683 mlx5_vdpa_destroy_mr(&ndev->mvdev); in mlx5_vdpa_change_map()
1688 static int setup_driver(struct mlx5_vdpa_net *ndev) in setup_driver() argument
1692 mutex_lock(&ndev->reslock); in setup_driver()
1693 if (ndev->setup) { in setup_driver()
1694 mlx5_vdpa_warn(&ndev->mvdev, "setup driver called for already setup driver\n"); in setup_driver()
1698 err = setup_virtqueues(ndev); in setup_driver()
1700 mlx5_vdpa_warn(&ndev->mvdev, "setup_virtqueues\n"); in setup_driver()
1704 err = create_rqt(ndev); in setup_driver()
1706 mlx5_vdpa_warn(&ndev->mvdev, "create_rqt\n"); in setup_driver()
1710 err = create_tir(ndev); in setup_driver()
1712 mlx5_vdpa_warn(&ndev->mvdev, "create_tir\n"); in setup_driver()
1716 err = add_fwd_to_tir(ndev); in setup_driver()
1718 mlx5_vdpa_warn(&ndev->mvdev, "add_fwd_to_tir\n"); in setup_driver()
1721 ndev->setup = true; in setup_driver()
1722 mutex_unlock(&ndev->reslock); in setup_driver()
1727 destroy_tir(ndev); in setup_driver()
1729 destroy_rqt(ndev); in setup_driver()
1731 teardown_virtqueues(ndev); in setup_driver()
1733 mutex_unlock(&ndev->reslock); in setup_driver()
1737 static void teardown_driver(struct mlx5_vdpa_net *ndev) in teardown_driver() argument
1739 mutex_lock(&ndev->reslock); in teardown_driver()
1740 if (!ndev->setup) in teardown_driver()
1743 remove_fwd_to_tir(ndev); in teardown_driver()
1744 destroy_tir(ndev); in teardown_driver()
1745 destroy_rqt(ndev); in teardown_driver()
1746 teardown_virtqueues(ndev); in teardown_driver()
1747 ndev->setup = false; in teardown_driver()
1749 mutex_unlock(&ndev->reslock); in teardown_driver()
1752 static void clear_vqs_ready(struct mlx5_vdpa_net *ndev) in clear_vqs_ready() argument
1756 for (i = 0; i < ndev->mvdev.max_vqs; i++) in clear_vqs_ready()
1757 ndev->vqs[i].ready = false; in clear_vqs_ready()
1763 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_status() local
1769 teardown_driver(ndev); in mlx5_vdpa_set_status()
1770 clear_vqs_ready(ndev); in mlx5_vdpa_set_status()
1771 mlx5_vdpa_destroy_mr(&ndev->mvdev); in mlx5_vdpa_set_status()
1772 ndev->mvdev.status = 0; in mlx5_vdpa_set_status()
1773 ndev->mvdev.mlx_features = 0; in mlx5_vdpa_set_status()
1778 if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) { in mlx5_vdpa_set_status()
1780 err = setup_driver(ndev); in mlx5_vdpa_set_status()
1791 ndev->mvdev.status = status; in mlx5_vdpa_set_status()
1795 mlx5_vdpa_destroy_mr(&ndev->mvdev); in mlx5_vdpa_set_status()
1796 ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; in mlx5_vdpa_set_status()
1803 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_get_config() local
1806 memcpy(buf, (u8 *)&ndev->config + offset, len); in mlx5_vdpa_get_config()
1825 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_map() local
1836 return mlx5_vdpa_change_map(ndev, iotlb); in mlx5_vdpa_set_map()
1845 struct mlx5_vdpa_net *ndev; in mlx5_vdpa_free() local
1847 ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_free()
1849 free_resources(ndev); in mlx5_vdpa_free()
1850 if (!is_zero_ether_addr(ndev->config.mac)) { in mlx5_vdpa_free()
1852 mlx5_mpfs_del_mac(pfmdev, ndev->config.mac); in mlx5_vdpa_free()
1854 mlx5_vdpa_free_resources(&ndev->mvdev); in mlx5_vdpa_free()
1855 mutex_destroy(&ndev->reslock); in mlx5_vdpa_free()
1910 static int alloc_resources(struct mlx5_vdpa_net *ndev) in alloc_resources() argument
1912 struct mlx5_vdpa_net_resources *res = &ndev->res; in alloc_resources()
1916 mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n"); in alloc_resources()
1920 err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn); in alloc_resources()
1924 err = create_tis(ndev); in alloc_resources()
1933 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); in alloc_resources()
1937 static void free_resources(struct mlx5_vdpa_net *ndev) in free_resources() argument
1939 struct mlx5_vdpa_net_resources *res = &ndev->res; in free_resources()
1944 destroy_tis(ndev); in free_resources()
1945 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); in free_resources()
1949 static void init_mvqs(struct mlx5_vdpa_net *ndev) in init_mvqs() argument
1954 for (i = 0; i < 2 * mlx5_vdpa_max_qps(ndev->mvdev.max_vqs); ++i) { in init_mvqs()
1955 mvq = &ndev->vqs[i]; in init_mvqs()
1958 mvq->ndev = ndev; in init_mvqs()
1961 for (; i < ndev->mvdev.max_vqs; i++) { in init_mvqs()
1962 mvq = &ndev->vqs[i]; in init_mvqs()
1965 mvq->ndev = ndev; in init_mvqs()
1974 struct mlx5_vdpa_net *ndev; in mlx5_vdpa_add_dev() local
1982 ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops, in mlx5_vdpa_add_dev()
1984 if (IS_ERR(ndev)) in mlx5_vdpa_add_dev()
1985 return ndev; in mlx5_vdpa_add_dev()
1987 ndev->mvdev.max_vqs = max_vqs; in mlx5_vdpa_add_dev()
1988 mvdev = &ndev->mvdev; in mlx5_vdpa_add_dev()
1990 init_mvqs(ndev); in mlx5_vdpa_add_dev()
1991 mutex_init(&ndev->reslock); in mlx5_vdpa_add_dev()
1992 config = &ndev->config; in mlx5_vdpa_add_dev()
1993 err = query_mtu(mdev, &ndev->mtu); in mlx5_vdpa_add_dev()
2009 err = mlx5_vdpa_alloc_resources(&ndev->mvdev); in mlx5_vdpa_add_dev()
2013 err = alloc_resources(ndev); in mlx5_vdpa_add_dev()
2021 return ndev; in mlx5_vdpa_add_dev()
2024 free_resources(ndev); in mlx5_vdpa_add_dev()
2026 mlx5_vdpa_free_resources(&ndev->mvdev); in mlx5_vdpa_add_dev()
2031 mutex_destroy(&ndev->reslock); in mlx5_vdpa_add_dev()