1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020 Mellanox Technologies Ltd. */
3
4 #include <linux/module.h>
5 #include <linux/vdpa.h>
6 #include <linux/vringh.h>
7 #include <uapi/linux/virtio_net.h>
8 #include <uapi/linux/virtio_ids.h>
9 #include <linux/virtio_config.h>
10 #include <linux/auxiliary_bus.h>
11 #include <linux/mlx5/cq.h>
12 #include <linux/mlx5/qp.h>
13 #include <linux/mlx5/device.h>
14 #include <linux/mlx5/driver.h>
15 #include <linux/mlx5/vport.h>
16 #include <linux/mlx5/fs.h>
17 #include <linux/mlx5/mlx5_ifc_vdpa.h>
18 #include <linux/mlx5/mpfs.h>
19 #include "mlx5_vdpa.h"
20
21 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
22 MODULE_DESCRIPTION("Mellanox VDPA driver");
23 MODULE_LICENSE("Dual BSD/GPL");
24
25 #define to_mlx5_vdpa_ndev(__mvdev) \
26 container_of(__mvdev, struct mlx5_vdpa_net, mvdev)
27 #define to_mvdev(__vdev) container_of((__vdev), struct mlx5_vdpa_dev, vdev)
28
29 #define VALID_FEATURES_MASK \
30 (BIT_ULL(VIRTIO_NET_F_CSUM) | BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) | \
31 BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_MAC) | \
32 BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) | BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) | \
33 BIT_ULL(VIRTIO_NET_F_GUEST_ECN) | BIT_ULL(VIRTIO_NET_F_GUEST_UFO) | BIT_ULL(VIRTIO_NET_F_HOST_TSO4) | \
34 BIT_ULL(VIRTIO_NET_F_HOST_TSO6) | BIT_ULL(VIRTIO_NET_F_HOST_ECN) | BIT_ULL(VIRTIO_NET_F_HOST_UFO) | \
35 BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | BIT_ULL(VIRTIO_NET_F_STATUS) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ) | \
36 BIT_ULL(VIRTIO_NET_F_CTRL_RX) | BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) | \
37 BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) | BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE) | \
38 BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) | BIT_ULL(VIRTIO_NET_F_HASH_REPORT) | \
39 BIT_ULL(VIRTIO_NET_F_RSS) | BIT_ULL(VIRTIO_NET_F_RSC_EXT) | BIT_ULL(VIRTIO_NET_F_STANDBY) | \
40 BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX) | BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY) | \
41 BIT_ULL(VIRTIO_F_ANY_LAYOUT) | BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) | \
42 BIT_ULL(VIRTIO_F_RING_PACKED) | BIT_ULL(VIRTIO_F_ORDER_PLATFORM) | BIT_ULL(VIRTIO_F_SR_IOV))
43
44 #define VALID_STATUS_MASK \
45 (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK | \
46 VIRTIO_CONFIG_S_FEATURES_OK | VIRTIO_CONFIG_S_NEEDS_RESET | VIRTIO_CONFIG_S_FAILED)
47
48 #define MLX5_FEATURE(_mvdev, _feature) (!!((_mvdev)->actual_features & BIT_ULL(_feature)))
49
50 struct mlx5_vdpa_net_resources {
51 u32 tisn;
52 u32 tdn;
53 u32 tirn;
54 u32 rqtn;
55 bool valid;
56 };
57
58 struct mlx5_vdpa_cq_buf {
59 struct mlx5_frag_buf_ctrl fbc;
60 struct mlx5_frag_buf frag_buf;
61 int cqe_size;
62 int nent;
63 };
64
65 struct mlx5_vdpa_cq {
66 struct mlx5_core_cq mcq;
67 struct mlx5_vdpa_cq_buf buf;
68 struct mlx5_db db;
69 int cqe;
70 };
71
72 struct mlx5_vdpa_umem {
73 struct mlx5_frag_buf_ctrl fbc;
74 struct mlx5_frag_buf frag_buf;
75 int size;
76 u32 id;
77 };
78
79 struct mlx5_vdpa_qp {
80 struct mlx5_core_qp mqp;
81 struct mlx5_frag_buf frag_buf;
82 struct mlx5_db db;
83 u16 head;
84 bool fw;
85 };
86
87 struct mlx5_vq_restore_info {
88 u32 num_ent;
89 u64 desc_addr;
90 u64 device_addr;
91 u64 driver_addr;
92 u16 avail_index;
93 u16 used_index;
94 bool ready;
95 bool restore;
96 };
97
98 struct mlx5_vdpa_virtqueue {
99 bool ready;
100 u64 desc_addr;
101 u64 device_addr;
102 u64 driver_addr;
103 u32 num_ent;
104
105 /* Resources for implementing the notification channel from the device
106 * to the driver. fwqp is the firmware end of an RC connection; the
107 * other end is vqqp used by the driver. cq is is where completions are
108 * reported.
109 */
110 struct mlx5_vdpa_cq cq;
111 struct mlx5_vdpa_qp fwqp;
112 struct mlx5_vdpa_qp vqqp;
113
114 /* umem resources are required for the virtqueue operation. They're use
115 * is internal and they must be provided by the driver.
116 */
117 struct mlx5_vdpa_umem umem1;
118 struct mlx5_vdpa_umem umem2;
119 struct mlx5_vdpa_umem umem3;
120
121 bool initialized;
122 int index;
123 u32 virtq_id;
124 struct mlx5_vdpa_net *ndev;
125 u16 avail_idx;
126 u16 used_idx;
127 int fw_state;
128
129 /* keep last in the struct */
130 struct mlx5_vq_restore_info ri;
131 };
132
133 /* We will remove this limitation once mlx5_vdpa_alloc_resources()
134 * provides for driver space allocation
135 */
136 #define MLX5_MAX_SUPPORTED_VQS 16
137
is_index_valid(struct mlx5_vdpa_dev * mvdev,u16 idx)138 static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx)
139 {
140 if (unlikely(idx > mvdev->max_idx))
141 return false;
142
143 return true;
144 }
145
146 struct mlx5_vdpa_net {
147 struct mlx5_vdpa_dev mvdev;
148 struct mlx5_vdpa_net_resources res;
149 struct virtio_net_config config;
150 struct mlx5_vdpa_virtqueue vqs[MLX5_MAX_SUPPORTED_VQS];
151 struct vdpa_callback event_cbs[MLX5_MAX_SUPPORTED_VQS + 1];
152
153 /* Serialize vq resources creation and destruction. This is required
154 * since memory map might change and we need to destroy and create
155 * resources while driver in operational.
156 */
157 struct mutex reslock;
158 struct mlx5_flow_table *rxft;
159 struct mlx5_fc *rx_counter;
160 struct mlx5_flow_handle *rx_rule;
161 bool setup;
162 u16 mtu;
163 u32 cur_num_vqs;
164 struct notifier_block nb;
165 struct vdpa_callback config_cb;
166 struct mlx5_vdpa_wq_ent cvq_ent;
167 };
168
169 static void free_resources(struct mlx5_vdpa_net *ndev);
170 static void init_mvqs(struct mlx5_vdpa_net *ndev);
171 static int setup_driver(struct mlx5_vdpa_dev *mvdev);
172 static void teardown_driver(struct mlx5_vdpa_net *ndev);
173
174 static bool mlx5_vdpa_debug;
175
176 #define MLX5_CVQ_MAX_ENT 16
177
178 #define MLX5_LOG_VIO_FLAG(_feature) \
179 do { \
180 if (features & BIT_ULL(_feature)) \
181 mlx5_vdpa_info(mvdev, "%s\n", #_feature); \
182 } while (0)
183
184 #define MLX5_LOG_VIO_STAT(_status) \
185 do { \
186 if (status & (_status)) \
187 mlx5_vdpa_info(mvdev, "%s\n", #_status); \
188 } while (0)
189
190 /* TODO: cross-endian support */
mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev * mvdev)191 static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev)
192 {
193 return virtio_legacy_is_little_endian() ||
194 (mvdev->actual_features & BIT_ULL(VIRTIO_F_VERSION_1));
195 }
196
mlx5vdpa16_to_cpu(struct mlx5_vdpa_dev * mvdev,__virtio16 val)197 static u16 mlx5vdpa16_to_cpu(struct mlx5_vdpa_dev *mvdev, __virtio16 val)
198 {
199 return __virtio16_to_cpu(mlx5_vdpa_is_little_endian(mvdev), val);
200 }
201
cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev * mvdev,u16 val)202 static __virtio16 cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev *mvdev, u16 val)
203 {
204 return __cpu_to_virtio16(mlx5_vdpa_is_little_endian(mvdev), val);
205 }
206
mlx5_vdpa_max_qps(int max_vqs)207 static inline u32 mlx5_vdpa_max_qps(int max_vqs)
208 {
209 return max_vqs / 2;
210 }
211
ctrl_vq_idx(struct mlx5_vdpa_dev * mvdev)212 static u16 ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev)
213 {
214 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ)))
215 return 2;
216
217 return 2 * mlx5_vdpa_max_qps(mvdev->max_vqs);
218 }
219
is_ctrl_vq_idx(struct mlx5_vdpa_dev * mvdev,u16 idx)220 static bool is_ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev, u16 idx)
221 {
222 return idx == ctrl_vq_idx(mvdev);
223 }
224
print_status(struct mlx5_vdpa_dev * mvdev,u8 status,bool set)225 static void print_status(struct mlx5_vdpa_dev *mvdev, u8 status, bool set)
226 {
227 if (status & ~VALID_STATUS_MASK)
228 mlx5_vdpa_warn(mvdev, "Warning: there are invalid status bits 0x%x\n",
229 status & ~VALID_STATUS_MASK);
230
231 if (!mlx5_vdpa_debug)
232 return;
233
234 mlx5_vdpa_info(mvdev, "driver status %s", set ? "set" : "get");
235 if (set && !status) {
236 mlx5_vdpa_info(mvdev, "driver resets the device\n");
237 return;
238 }
239
240 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_ACKNOWLEDGE);
241 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER);
242 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER_OK);
243 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FEATURES_OK);
244 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_NEEDS_RESET);
245 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FAILED);
246 }
247
print_features(struct mlx5_vdpa_dev * mvdev,u64 features,bool set)248 static void print_features(struct mlx5_vdpa_dev *mvdev, u64 features, bool set)
249 {
250 if (features & ~VALID_FEATURES_MASK)
251 mlx5_vdpa_warn(mvdev, "There are invalid feature bits 0x%llx\n",
252 features & ~VALID_FEATURES_MASK);
253
254 if (!mlx5_vdpa_debug)
255 return;
256
257 mlx5_vdpa_info(mvdev, "driver %s feature bits:\n", set ? "sets" : "reads");
258 if (!features)
259 mlx5_vdpa_info(mvdev, "all feature bits are cleared\n");
260
261 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CSUM);
262 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_CSUM);
263 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS);
264 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MTU);
265 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MAC);
266 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO4);
267 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO6);
268 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ECN);
269 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_UFO);
270 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO4);
271 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO6);
272 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_ECN);
273 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_UFO);
274 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MRG_RXBUF);
275 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STATUS);
276 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VQ);
277 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX);
278 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VLAN);
279 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX_EXTRA);
280 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ANNOUNCE);
281 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MQ);
282 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_MAC_ADDR);
283 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HASH_REPORT);
284 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSS);
285 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSC_EXT);
286 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STANDBY);
287 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_SPEED_DUPLEX);
288 MLX5_LOG_VIO_FLAG(VIRTIO_F_NOTIFY_ON_EMPTY);
289 MLX5_LOG_VIO_FLAG(VIRTIO_F_ANY_LAYOUT);
290 MLX5_LOG_VIO_FLAG(VIRTIO_F_VERSION_1);
291 MLX5_LOG_VIO_FLAG(VIRTIO_F_ACCESS_PLATFORM);
292 MLX5_LOG_VIO_FLAG(VIRTIO_F_RING_PACKED);
293 MLX5_LOG_VIO_FLAG(VIRTIO_F_ORDER_PLATFORM);
294 MLX5_LOG_VIO_FLAG(VIRTIO_F_SR_IOV);
295 }
296
create_tis(struct mlx5_vdpa_net * ndev)297 static int create_tis(struct mlx5_vdpa_net *ndev)
298 {
299 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
300 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
301 void *tisc;
302 int err;
303
304 tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
305 MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn);
306 err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn);
307 if (err)
308 mlx5_vdpa_warn(mvdev, "create TIS (%d)\n", err);
309
310 return err;
311 }
312
destroy_tis(struct mlx5_vdpa_net * ndev)313 static void destroy_tis(struct mlx5_vdpa_net *ndev)
314 {
315 mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn);
316 }
317
318 #define MLX5_VDPA_CQE_SIZE 64
319 #define MLX5_VDPA_LOG_CQE_SIZE ilog2(MLX5_VDPA_CQE_SIZE)
320
cq_frag_buf_alloc(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_cq_buf * buf,int nent)321 static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent)
322 {
323 struct mlx5_frag_buf *frag_buf = &buf->frag_buf;
324 u8 log_wq_stride = MLX5_VDPA_LOG_CQE_SIZE;
325 u8 log_wq_sz = MLX5_VDPA_LOG_CQE_SIZE;
326 int err;
327
328 err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf,
329 ndev->mvdev.mdev->priv.numa_node);
330 if (err)
331 return err;
332
333 mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc);
334
335 buf->cqe_size = MLX5_VDPA_CQE_SIZE;
336 buf->nent = nent;
337
338 return 0;
339 }
340
umem_frag_buf_alloc(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_umem * umem,int size)341 static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size)
342 {
343 struct mlx5_frag_buf *frag_buf = &umem->frag_buf;
344
345 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf,
346 ndev->mvdev.mdev->priv.numa_node);
347 }
348
cq_frag_buf_free(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_cq_buf * buf)349 static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf)
350 {
351 mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf);
352 }
353
get_cqe(struct mlx5_vdpa_cq * vcq,int n)354 static void *get_cqe(struct mlx5_vdpa_cq *vcq, int n)
355 {
356 return mlx5_frag_buf_get_wqe(&vcq->buf.fbc, n);
357 }
358
cq_frag_buf_init(struct mlx5_vdpa_cq * vcq,struct mlx5_vdpa_cq_buf * buf)359 static void cq_frag_buf_init(struct mlx5_vdpa_cq *vcq, struct mlx5_vdpa_cq_buf *buf)
360 {
361 struct mlx5_cqe64 *cqe64;
362 void *cqe;
363 int i;
364
365 for (i = 0; i < buf->nent; i++) {
366 cqe = get_cqe(vcq, i);
367 cqe64 = cqe;
368 cqe64->op_own = MLX5_CQE_INVALID << 4;
369 }
370 }
371
get_sw_cqe(struct mlx5_vdpa_cq * cq,int n)372 static void *get_sw_cqe(struct mlx5_vdpa_cq *cq, int n)
373 {
374 struct mlx5_cqe64 *cqe64 = get_cqe(cq, n & (cq->cqe - 1));
375
376 if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) &&
377 !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & cq->cqe)))
378 return cqe64;
379
380 return NULL;
381 }
382
rx_post(struct mlx5_vdpa_qp * vqp,int n)383 static void rx_post(struct mlx5_vdpa_qp *vqp, int n)
384 {
385 vqp->head += n;
386 vqp->db.db[0] = cpu_to_be32(vqp->head);
387 }
388
qp_prepare(struct mlx5_vdpa_net * ndev,bool fw,void * in,struct mlx5_vdpa_virtqueue * mvq,u32 num_ent)389 static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in,
390 struct mlx5_vdpa_virtqueue *mvq, u32 num_ent)
391 {
392 struct mlx5_vdpa_qp *vqp;
393 __be64 *pas;
394 void *qpc;
395
396 vqp = fw ? &mvq->fwqp : &mvq->vqqp;
397 MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid);
398 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
399 if (vqp->fw) {
400 /* Firmware QP is allocated by the driver for the firmware's
401 * use so we can skip part of the params as they will be chosen by firmware
402 */
403 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
404 MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ);
405 MLX5_SET(qpc, qpc, no_sq, 1);
406 return;
407 }
408
409 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
410 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
411 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn);
412 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES);
413 MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index);
414 MLX5_SET(qpc, qpc, log_page_size, vqp->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
415 MLX5_SET(qpc, qpc, no_sq, 1);
416 MLX5_SET(qpc, qpc, cqn_rcv, mvq->cq.mcq.cqn);
417 MLX5_SET(qpc, qpc, log_rq_size, ilog2(num_ent));
418 MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ);
419 pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas);
420 mlx5_fill_page_frag_array(&vqp->frag_buf, pas);
421 }
422
rq_buf_alloc(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_qp * vqp,u32 num_ent)423 static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent)
424 {
425 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev,
426 num_ent * sizeof(struct mlx5_wqe_data_seg), &vqp->frag_buf,
427 ndev->mvdev.mdev->priv.numa_node);
428 }
429
rq_buf_free(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_qp * vqp)430 static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp)
431 {
432 mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf);
433 }
434
qp_create(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq,struct mlx5_vdpa_qp * vqp)435 static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq,
436 struct mlx5_vdpa_qp *vqp)
437 {
438 struct mlx5_core_dev *mdev = ndev->mvdev.mdev;
439 int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
440 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
441 void *qpc;
442 void *in;
443 int err;
444
445 if (!vqp->fw) {
446 vqp = &mvq->vqqp;
447 err = rq_buf_alloc(ndev, vqp, mvq->num_ent);
448 if (err)
449 return err;
450
451 err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db);
452 if (err)
453 goto err_db;
454 inlen += vqp->frag_buf.npages * sizeof(__be64);
455 }
456
457 in = kzalloc(inlen, GFP_KERNEL);
458 if (!in) {
459 err = -ENOMEM;
460 goto err_kzalloc;
461 }
462
463 qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent);
464 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
465 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
466 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
467 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn);
468 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES);
469 if (!vqp->fw)
470 MLX5_SET64(qpc, qpc, dbr_addr, vqp->db.dma);
471 MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
472 err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out));
473 kfree(in);
474 if (err)
475 goto err_kzalloc;
476
477 vqp->mqp.uid = ndev->mvdev.res.uid;
478 vqp->mqp.qpn = MLX5_GET(create_qp_out, out, qpn);
479
480 if (!vqp->fw)
481 rx_post(vqp, mvq->num_ent);
482
483 return 0;
484
485 err_kzalloc:
486 if (!vqp->fw)
487 mlx5_db_free(ndev->mvdev.mdev, &vqp->db);
488 err_db:
489 if (!vqp->fw)
490 rq_buf_free(ndev, vqp);
491
492 return err;
493 }
494
qp_destroy(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_qp * vqp)495 static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp)
496 {
497 u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {};
498
499 MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);
500 MLX5_SET(destroy_qp_in, in, qpn, vqp->mqp.qpn);
501 MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid);
502 if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in))
503 mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn);
504 if (!vqp->fw) {
505 mlx5_db_free(ndev->mvdev.mdev, &vqp->db);
506 rq_buf_free(ndev, vqp);
507 }
508 }
509
next_cqe_sw(struct mlx5_vdpa_cq * cq)510 static void *next_cqe_sw(struct mlx5_vdpa_cq *cq)
511 {
512 return get_sw_cqe(cq, cq->mcq.cons_index);
513 }
514
mlx5_vdpa_poll_one(struct mlx5_vdpa_cq * vcq)515 static int mlx5_vdpa_poll_one(struct mlx5_vdpa_cq *vcq)
516 {
517 struct mlx5_cqe64 *cqe64;
518
519 cqe64 = next_cqe_sw(vcq);
520 if (!cqe64)
521 return -EAGAIN;
522
523 vcq->mcq.cons_index++;
524 return 0;
525 }
526
mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue * mvq,int num)527 static void mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue *mvq, int num)
528 {
529 struct mlx5_vdpa_net *ndev = mvq->ndev;
530 struct vdpa_callback *event_cb;
531
532 event_cb = &ndev->event_cbs[mvq->index];
533 mlx5_cq_set_ci(&mvq->cq.mcq);
534
535 /* make sure CQ cosumer update is visible to the hardware before updating
536 * RX doorbell record.
537 */
538 dma_wmb();
539 rx_post(&mvq->vqqp, num);
540 if (event_cb->callback)
541 event_cb->callback(event_cb->private);
542 }
543
mlx5_vdpa_cq_comp(struct mlx5_core_cq * mcq,struct mlx5_eqe * eqe)544 static void mlx5_vdpa_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
545 {
546 struct mlx5_vdpa_virtqueue *mvq = container_of(mcq, struct mlx5_vdpa_virtqueue, cq.mcq);
547 struct mlx5_vdpa_net *ndev = mvq->ndev;
548 void __iomem *uar_page = ndev->mvdev.res.uar->map;
549 int num = 0;
550
551 while (!mlx5_vdpa_poll_one(&mvq->cq)) {
552 num++;
553 if (num > mvq->num_ent / 2) {
554 /* If completions keep coming while we poll, we want to
555 * let the hardware know that we consumed them by
556 * updating the doorbell record. We also let vdpa core
557 * know about this so it passes it on the virtio driver
558 * on the guest.
559 */
560 mlx5_vdpa_handle_completions(mvq, num);
561 num = 0;
562 }
563 }
564
565 if (num)
566 mlx5_vdpa_handle_completions(mvq, num);
567
568 mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index);
569 }
570
cq_create(struct mlx5_vdpa_net * ndev,u16 idx,u32 num_ent)571 static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent)
572 {
573 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx];
574 struct mlx5_core_dev *mdev = ndev->mvdev.mdev;
575 void __iomem *uar_page = ndev->mvdev.res.uar->map;
576 u32 out[MLX5_ST_SZ_DW(create_cq_out)];
577 struct mlx5_vdpa_cq *vcq = &mvq->cq;
578 __be64 *pas;
579 int inlen;
580 void *cqc;
581 void *in;
582 int err;
583 int eqn;
584
585 err = mlx5_db_alloc(mdev, &vcq->db);
586 if (err)
587 return err;
588
589 vcq->mcq.set_ci_db = vcq->db.db;
590 vcq->mcq.arm_db = vcq->db.db + 1;
591 vcq->mcq.cqe_sz = 64;
592
593 err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent);
594 if (err)
595 goto err_db;
596
597 cq_frag_buf_init(vcq, &vcq->buf);
598
599 inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
600 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * vcq->buf.frag_buf.npages;
601 in = kzalloc(inlen, GFP_KERNEL);
602 if (!in) {
603 err = -ENOMEM;
604 goto err_vzalloc;
605 }
606
607 MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid);
608 pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas);
609 mlx5_fill_page_frag_array(&vcq->buf.frag_buf, pas);
610
611 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
612 MLX5_SET(cqc, cqc, log_page_size, vcq->buf.frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
613
614 /* Use vector 0 by default. Consider adding code to choose least used
615 * vector.
616 */
617 err = mlx5_vector2eqn(mdev, 0, &eqn);
618 if (err)
619 goto err_vec;
620
621 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
622 MLX5_SET(cqc, cqc, log_cq_size, ilog2(num_ent));
623 MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index);
624 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn);
625 MLX5_SET64(cqc, cqc, dbr_addr, vcq->db.dma);
626
627 err = mlx5_core_create_cq(mdev, &vcq->mcq, in, inlen, out, sizeof(out));
628 if (err)
629 goto err_vec;
630
631 vcq->mcq.comp = mlx5_vdpa_cq_comp;
632 vcq->cqe = num_ent;
633 vcq->mcq.set_ci_db = vcq->db.db;
634 vcq->mcq.arm_db = vcq->db.db + 1;
635 mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index);
636 kfree(in);
637 return 0;
638
639 err_vec:
640 kfree(in);
641 err_vzalloc:
642 cq_frag_buf_free(ndev, &vcq->buf);
643 err_db:
644 mlx5_db_free(ndev->mvdev.mdev, &vcq->db);
645 return err;
646 }
647
cq_destroy(struct mlx5_vdpa_net * ndev,u16 idx)648 static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx)
649 {
650 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx];
651 struct mlx5_core_dev *mdev = ndev->mvdev.mdev;
652 struct mlx5_vdpa_cq *vcq = &mvq->cq;
653
654 if (mlx5_core_destroy_cq(mdev, &vcq->mcq)) {
655 mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn);
656 return;
657 }
658 cq_frag_buf_free(ndev, &vcq->buf);
659 mlx5_db_free(ndev->mvdev.mdev, &vcq->db);
660 }
661
set_umem_size(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq,int num,struct mlx5_vdpa_umem ** umemp)662 static void set_umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num,
663 struct mlx5_vdpa_umem **umemp)
664 {
665 struct mlx5_core_dev *mdev = ndev->mvdev.mdev;
666 int p_a;
667 int p_b;
668
669 switch (num) {
670 case 1:
671 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_a);
672 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_b);
673 *umemp = &mvq->umem1;
674 break;
675 case 2:
676 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_a);
677 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_b);
678 *umemp = &mvq->umem2;
679 break;
680 case 3:
681 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_a);
682 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_b);
683 *umemp = &mvq->umem3;
684 break;
685 }
686 (*umemp)->size = p_a * mvq->num_ent + p_b;
687 }
688
umem_frag_buf_free(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_umem * umem)689 static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem)
690 {
691 mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf);
692 }
693
create_umem(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq,int num)694 static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num)
695 {
696 int inlen;
697 u32 out[MLX5_ST_SZ_DW(create_umem_out)] = {};
698 void *um;
699 void *in;
700 int err;
701 __be64 *pas;
702 struct mlx5_vdpa_umem *umem;
703
704 set_umem_size(ndev, mvq, num, &umem);
705 err = umem_frag_buf_alloc(ndev, umem, umem->size);
706 if (err)
707 return err;
708
709 inlen = MLX5_ST_SZ_BYTES(create_umem_in) + MLX5_ST_SZ_BYTES(mtt) * umem->frag_buf.npages;
710
711 in = kzalloc(inlen, GFP_KERNEL);
712 if (!in) {
713 err = -ENOMEM;
714 goto err_in;
715 }
716
717 MLX5_SET(create_umem_in, in, opcode, MLX5_CMD_OP_CREATE_UMEM);
718 MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid);
719 um = MLX5_ADDR_OF(create_umem_in, in, umem);
720 MLX5_SET(umem, um, log_page_size, umem->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
721 MLX5_SET64(umem, um, num_of_mtt, umem->frag_buf.npages);
722
723 pas = (__be64 *)MLX5_ADDR_OF(umem, um, mtt[0]);
724 mlx5_fill_page_frag_array_perm(&umem->frag_buf, pas, MLX5_MTT_PERM_RW);
725
726 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out));
727 if (err) {
728 mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err);
729 goto err_cmd;
730 }
731
732 kfree(in);
733 umem->id = MLX5_GET(create_umem_out, out, umem_id);
734
735 return 0;
736
737 err_cmd:
738 kfree(in);
739 err_in:
740 umem_frag_buf_free(ndev, umem);
741 return err;
742 }
743
umem_destroy(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq,int num)744 static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num)
745 {
746 u32 in[MLX5_ST_SZ_DW(destroy_umem_in)] = {};
747 u32 out[MLX5_ST_SZ_DW(destroy_umem_out)] = {};
748 struct mlx5_vdpa_umem *umem;
749
750 switch (num) {
751 case 1:
752 umem = &mvq->umem1;
753 break;
754 case 2:
755 umem = &mvq->umem2;
756 break;
757 case 3:
758 umem = &mvq->umem3;
759 break;
760 }
761
762 MLX5_SET(destroy_umem_in, in, opcode, MLX5_CMD_OP_DESTROY_UMEM);
763 MLX5_SET(destroy_umem_in, in, umem_id, umem->id);
764 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)))
765 return;
766
767 umem_frag_buf_free(ndev, umem);
768 }
769
umems_create(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq)770 static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
771 {
772 int num;
773 int err;
774
775 for (num = 1; num <= 3; num++) {
776 err = create_umem(ndev, mvq, num);
777 if (err)
778 goto err_umem;
779 }
780 return 0;
781
782 err_umem:
783 for (num--; num > 0; num--)
784 umem_destroy(ndev, mvq, num);
785
786 return err;
787 }
788
umems_destroy(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq)789 static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
790 {
791 int num;
792
793 for (num = 3; num > 0; num--)
794 umem_destroy(ndev, mvq, num);
795 }
796
get_queue_type(struct mlx5_vdpa_net * ndev)797 static int get_queue_type(struct mlx5_vdpa_net *ndev)
798 {
799 u32 type_mask;
800
801 type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type);
802
803 /* prefer split queue */
804 if (type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT)
805 return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_SPLIT;
806
807 WARN_ON(!(type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_PACKED));
808
809 return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_PACKED;
810 }
811
vq_is_tx(u16 idx)812 static bool vq_is_tx(u16 idx)
813 {
814 return idx % 2;
815 }
816
get_features_12_3(u64 features)817 static u16 get_features_12_3(u64 features)
818 {
819 return (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << 9) |
820 (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << 8) |
821 (!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << 7) |
822 (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_CSUM)) << 6);
823 }
824
create_virtqueue(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq)825 static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
826 {
827 int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in);
828 u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {};
829 void *obj_context;
830 void *cmd_hdr;
831 void *vq_ctx;
832 void *in;
833 int err;
834
835 err = umems_create(ndev, mvq);
836 if (err)
837 return err;
838
839 in = kzalloc(inlen, GFP_KERNEL);
840 if (!in) {
841 err = -ENOMEM;
842 goto err_alloc;
843 }
844
845 cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr);
846
847 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
848 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q);
849 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid);
850
851 obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context);
852 MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx);
853 MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx);
854 MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3,
855 get_features_12_3(ndev->mvdev.actual_features));
856 vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context);
857 MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev));
858
859 if (vq_is_tx(mvq->index))
860 MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn);
861
862 MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE);
863 MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index);
864 MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn);
865 MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent);
866 MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0,
867 !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1)));
868 MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr);
869 MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr);
870 MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr);
871 MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey.key);
872 MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id);
873 MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size);
874 MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id);
875 MLX5_SET(virtio_q, vq_ctx, umem_2_size, mvq->umem2.size);
876 MLX5_SET(virtio_q, vq_ctx, umem_3_id, mvq->umem3.id);
877 MLX5_SET(virtio_q, vq_ctx, umem_3_size, mvq->umem3.size);
878 MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn);
879
880 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out));
881 if (err)
882 goto err_cmd;
883
884 kfree(in);
885 mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
886
887 return 0;
888
889 err_cmd:
890 kfree(in);
891 err_alloc:
892 umems_destroy(ndev, mvq);
893 return err;
894 }
895
destroy_virtqueue(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq)896 static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
897 {
898 u32 in[MLX5_ST_SZ_DW(destroy_virtio_net_q_in)] = {};
899 u32 out[MLX5_ST_SZ_DW(destroy_virtio_net_q_out)] = {};
900
901 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.opcode,
902 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
903 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_id, mvq->virtq_id);
904 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid);
905 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_type,
906 MLX5_OBJ_TYPE_VIRTIO_NET_Q);
907 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) {
908 mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id);
909 return;
910 }
911 umems_destroy(ndev, mvq);
912 }
913
get_rqpn(struct mlx5_vdpa_virtqueue * mvq,bool fw)914 static u32 get_rqpn(struct mlx5_vdpa_virtqueue *mvq, bool fw)
915 {
916 return fw ? mvq->vqqp.mqp.qpn : mvq->fwqp.mqp.qpn;
917 }
918
get_qpn(struct mlx5_vdpa_virtqueue * mvq,bool fw)919 static u32 get_qpn(struct mlx5_vdpa_virtqueue *mvq, bool fw)
920 {
921 return fw ? mvq->fwqp.mqp.qpn : mvq->vqqp.mqp.qpn;
922 }
923
alloc_inout(struct mlx5_vdpa_net * ndev,int cmd,void ** in,int * inlen,void ** out,int * outlen,u32 qpn,u32 rqpn)924 static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out,
925 int *outlen, u32 qpn, u32 rqpn)
926 {
927 void *qpc;
928 void *pp;
929
930 switch (cmd) {
931 case MLX5_CMD_OP_2RST_QP:
932 *inlen = MLX5_ST_SZ_BYTES(qp_2rst_in);
933 *outlen = MLX5_ST_SZ_BYTES(qp_2rst_out);
934 *in = kzalloc(*inlen, GFP_KERNEL);
935 *out = kzalloc(*outlen, GFP_KERNEL);
936 if (!*in || !*out)
937 goto outerr;
938
939 MLX5_SET(qp_2rst_in, *in, opcode, cmd);
940 MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid);
941 MLX5_SET(qp_2rst_in, *in, qpn, qpn);
942 break;
943 case MLX5_CMD_OP_RST2INIT_QP:
944 *inlen = MLX5_ST_SZ_BYTES(rst2init_qp_in);
945 *outlen = MLX5_ST_SZ_BYTES(rst2init_qp_out);
946 *in = kzalloc(*inlen, GFP_KERNEL);
947 *out = kzalloc(MLX5_ST_SZ_BYTES(rst2init_qp_out), GFP_KERNEL);
948 if (!*in || !*out)
949 goto outerr;
950
951 MLX5_SET(rst2init_qp_in, *in, opcode, cmd);
952 MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid);
953 MLX5_SET(rst2init_qp_in, *in, qpn, qpn);
954 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc);
955 MLX5_SET(qpc, qpc, remote_qpn, rqpn);
956 MLX5_SET(qpc, qpc, rwe, 1);
957 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
958 MLX5_SET(ads, pp, vhca_port_num, 1);
959 break;
960 case MLX5_CMD_OP_INIT2RTR_QP:
961 *inlen = MLX5_ST_SZ_BYTES(init2rtr_qp_in);
962 *outlen = MLX5_ST_SZ_BYTES(init2rtr_qp_out);
963 *in = kzalloc(*inlen, GFP_KERNEL);
964 *out = kzalloc(MLX5_ST_SZ_BYTES(init2rtr_qp_out), GFP_KERNEL);
965 if (!*in || !*out)
966 goto outerr;
967
968 MLX5_SET(init2rtr_qp_in, *in, opcode, cmd);
969 MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid);
970 MLX5_SET(init2rtr_qp_in, *in, qpn, qpn);
971 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc);
972 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES);
973 MLX5_SET(qpc, qpc, log_msg_max, 30);
974 MLX5_SET(qpc, qpc, remote_qpn, rqpn);
975 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
976 MLX5_SET(ads, pp, fl, 1);
977 break;
978 case MLX5_CMD_OP_RTR2RTS_QP:
979 *inlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_in);
980 *outlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_out);
981 *in = kzalloc(*inlen, GFP_KERNEL);
982 *out = kzalloc(MLX5_ST_SZ_BYTES(rtr2rts_qp_out), GFP_KERNEL);
983 if (!*in || !*out)
984 goto outerr;
985
986 MLX5_SET(rtr2rts_qp_in, *in, opcode, cmd);
987 MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid);
988 MLX5_SET(rtr2rts_qp_in, *in, qpn, qpn);
989 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc);
990 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
991 MLX5_SET(ads, pp, ack_timeout, 14);
992 MLX5_SET(qpc, qpc, retry_count, 7);
993 MLX5_SET(qpc, qpc, rnr_retry, 7);
994 break;
995 default:
996 goto outerr_nullify;
997 }
998
999 return;
1000
1001 outerr:
1002 kfree(*in);
1003 kfree(*out);
1004 outerr_nullify:
1005 *in = NULL;
1006 *out = NULL;
1007 }
1008
free_inout(void * in,void * out)1009 static void free_inout(void *in, void *out)
1010 {
1011 kfree(in);
1012 kfree(out);
1013 }
1014
1015 /* Two QPs are used by each virtqueue. One is used by the driver and one by
1016 * firmware. The fw argument indicates whether the subjected QP is the one used
1017 * by firmware.
1018 */
modify_qp(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq,bool fw,int cmd)1019 static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd)
1020 {
1021 int outlen;
1022 int inlen;
1023 void *out;
1024 void *in;
1025 int err;
1026
1027 alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw));
1028 if (!in || !out)
1029 return -ENOMEM;
1030
1031 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen);
1032 free_inout(in, out);
1033 return err;
1034 }
1035
connect_qps(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq)1036 static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
1037 {
1038 int err;
1039
1040 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP);
1041 if (err)
1042 return err;
1043
1044 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP);
1045 if (err)
1046 return err;
1047
1048 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP);
1049 if (err)
1050 return err;
1051
1052 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP);
1053 if (err)
1054 return err;
1055
1056 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP);
1057 if (err)
1058 return err;
1059
1060 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP);
1061 if (err)
1062 return err;
1063
1064 return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP);
1065 }
1066
1067 struct mlx5_virtq_attr {
1068 u8 state;
1069 u16 available_index;
1070 u16 used_index;
1071 };
1072
query_virtqueue(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq,struct mlx5_virtq_attr * attr)1073 static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq,
1074 struct mlx5_virtq_attr *attr)
1075 {
1076 int outlen = MLX5_ST_SZ_BYTES(query_virtio_net_q_out);
1077 u32 in[MLX5_ST_SZ_DW(query_virtio_net_q_in)] = {};
1078 void *out;
1079 void *obj_context;
1080 void *cmd_hdr;
1081 int err;
1082
1083 out = kzalloc(outlen, GFP_KERNEL);
1084 if (!out)
1085 return -ENOMEM;
1086
1087 cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, in, general_obj_in_cmd_hdr);
1088
1089 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
1090 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q);
1091 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id);
1092 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid);
1093 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen);
1094 if (err)
1095 goto err_cmd;
1096
1097 obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, out, obj_context);
1098 memset(attr, 0, sizeof(*attr));
1099 attr->state = MLX5_GET(virtio_net_q_object, obj_context, state);
1100 attr->available_index = MLX5_GET(virtio_net_q_object, obj_context, hw_available_index);
1101 attr->used_index = MLX5_GET(virtio_net_q_object, obj_context, hw_used_index);
1102 kfree(out);
1103 return 0;
1104
1105 err_cmd:
1106 kfree(out);
1107 return err;
1108 }
1109
modify_virtqueue(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq,int state)1110 static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state)
1111 {
1112 int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in);
1113 u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {};
1114 void *obj_context;
1115 void *cmd_hdr;
1116 void *in;
1117 int err;
1118
1119 in = kzalloc(inlen, GFP_KERNEL);
1120 if (!in)
1121 return -ENOMEM;
1122
1123 cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, in, general_obj_in_cmd_hdr);
1124
1125 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
1126 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q);
1127 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id);
1128 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid);
1129
1130 obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, in, obj_context);
1131 MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select,
1132 MLX5_VIRTQ_MODIFY_MASK_STATE);
1133 MLX5_SET(virtio_net_q_object, obj_context, state, state);
1134 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out));
1135 kfree(in);
1136 if (!err)
1137 mvq->fw_state = state;
1138
1139 return err;
1140 }
1141
setup_vq(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq)1142 static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
1143 {
1144 u16 idx = mvq->index;
1145 int err;
1146
1147 if (!mvq->num_ent)
1148 return 0;
1149
1150 if (mvq->initialized)
1151 return 0;
1152
1153 err = cq_create(ndev, idx, mvq->num_ent);
1154 if (err)
1155 return err;
1156
1157 err = qp_create(ndev, mvq, &mvq->fwqp);
1158 if (err)
1159 goto err_fwqp;
1160
1161 err = qp_create(ndev, mvq, &mvq->vqqp);
1162 if (err)
1163 goto err_vqqp;
1164
1165 err = connect_qps(ndev, mvq);
1166 if (err)
1167 goto err_connect;
1168
1169 err = create_virtqueue(ndev, mvq);
1170 if (err)
1171 goto err_connect;
1172
1173 if (mvq->ready) {
1174 err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY);
1175 if (err) {
1176 mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n",
1177 idx, err);
1178 goto err_connect;
1179 }
1180 }
1181
1182 mvq->initialized = true;
1183 return 0;
1184
1185 err_connect:
1186 qp_destroy(ndev, &mvq->vqqp);
1187 err_vqqp:
1188 qp_destroy(ndev, &mvq->fwqp);
1189 err_fwqp:
1190 cq_destroy(ndev, idx);
1191 return err;
1192 }
1193
suspend_vq(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq)1194 static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
1195 {
1196 struct mlx5_virtq_attr attr;
1197
1198 if (!mvq->initialized)
1199 return;
1200
1201 if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY)
1202 return;
1203
1204 if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND))
1205 mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n");
1206
1207 if (query_virtqueue(ndev, mvq, &attr)) {
1208 mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n");
1209 return;
1210 }
1211 mvq->avail_idx = attr.available_index;
1212 mvq->used_idx = attr.used_index;
1213 }
1214
suspend_vqs(struct mlx5_vdpa_net * ndev)1215 static void suspend_vqs(struct mlx5_vdpa_net *ndev)
1216 {
1217 int i;
1218
1219 for (i = 0; i < MLX5_MAX_SUPPORTED_VQS; i++)
1220 suspend_vq(ndev, &ndev->vqs[i]);
1221 }
1222
teardown_vq(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq)1223 static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
1224 {
1225 if (!mvq->initialized)
1226 return;
1227
1228 suspend_vq(ndev, mvq);
1229 destroy_virtqueue(ndev, mvq);
1230 qp_destroy(ndev, &mvq->vqqp);
1231 qp_destroy(ndev, &mvq->fwqp);
1232 cq_destroy(ndev, mvq->index);
1233 mvq->initialized = false;
1234 }
1235
create_rqt(struct mlx5_vdpa_net * ndev)1236 static int create_rqt(struct mlx5_vdpa_net *ndev)
1237 {
1238 __be32 *list;
1239 int max_rqt;
1240 void *rqtc;
1241 int inlen;
1242 void *in;
1243 int i, j;
1244 int err;
1245
1246 max_rqt = min_t(int, MLX5_MAX_SUPPORTED_VQS / 2,
1247 1 << MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size));
1248 if (max_rqt < 1)
1249 return -EOPNOTSUPP;
1250
1251 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + max_rqt * MLX5_ST_SZ_BYTES(rq_num);
1252 in = kzalloc(inlen, GFP_KERNEL);
1253 if (!in)
1254 return -ENOMEM;
1255
1256 MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid);
1257 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
1258
1259 MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q);
1260 MLX5_SET(rqtc, rqtc, rqt_max_size, max_rqt);
1261 list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]);
1262 for (i = 0, j = 0; j < max_rqt; j++) {
1263 if (!ndev->vqs[j].initialized)
1264 continue;
1265
1266 if (!vq_is_tx(ndev->vqs[j].index)) {
1267 list[i] = cpu_to_be32(ndev->vqs[j].virtq_id);
1268 i++;
1269 }
1270 }
1271 MLX5_SET(rqtc, rqtc, rqt_actual_size, i);
1272
1273 err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn);
1274 kfree(in);
1275 if (err)
1276 return err;
1277
1278 return 0;
1279 }
1280
1281 #define MLX5_MODIFY_RQT_NUM_RQS ((u64)1)
1282
modify_rqt(struct mlx5_vdpa_net * ndev,int num)1283 static int modify_rqt(struct mlx5_vdpa_net *ndev, int num)
1284 {
1285 __be32 *list;
1286 int max_rqt;
1287 void *rqtc;
1288 int inlen;
1289 void *in;
1290 int i, j;
1291 int err;
1292
1293 max_rqt = min_t(int, ndev->cur_num_vqs / 2,
1294 1 << MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size));
1295 if (max_rqt < 1)
1296 return -EOPNOTSUPP;
1297
1298 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + max_rqt * MLX5_ST_SZ_BYTES(rq_num);
1299 in = kzalloc(inlen, GFP_KERNEL);
1300 if (!in)
1301 return -ENOMEM;
1302
1303 MLX5_SET(modify_rqt_in, in, uid, ndev->mvdev.res.uid);
1304 MLX5_SET64(modify_rqt_in, in, bitmask, MLX5_MODIFY_RQT_NUM_RQS);
1305 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
1306 MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q);
1307
1308 list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]);
1309 for (i = 0, j = 0; j < num; j++) {
1310 if (!ndev->vqs[j].initialized)
1311 continue;
1312
1313 if (!vq_is_tx(ndev->vqs[j].index)) {
1314 list[i] = cpu_to_be32(ndev->vqs[j].virtq_id);
1315 i++;
1316 }
1317 }
1318 MLX5_SET(rqtc, rqtc, rqt_actual_size, i);
1319 err = mlx5_vdpa_modify_rqt(&ndev->mvdev, in, inlen, ndev->res.rqtn);
1320 kfree(in);
1321 if (err)
1322 return err;
1323
1324 return 0;
1325 }
1326
destroy_rqt(struct mlx5_vdpa_net * ndev)1327 static void destroy_rqt(struct mlx5_vdpa_net *ndev)
1328 {
1329 mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn);
1330 }
1331
create_tir(struct mlx5_vdpa_net * ndev)1332 static int create_tir(struct mlx5_vdpa_net *ndev)
1333 {
1334 #define HASH_IP_L4PORTS \
1335 (MLX5_HASH_FIELD_SEL_SRC_IP | MLX5_HASH_FIELD_SEL_DST_IP | MLX5_HASH_FIELD_SEL_L4_SPORT | \
1336 MLX5_HASH_FIELD_SEL_L4_DPORT)
1337 static const u8 rx_hash_toeplitz_key[] = { 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7,
1338 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94,
1339 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1,
1340 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59,
1341 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a };
1342 void *rss_key;
1343 void *outer;
1344 void *tirc;
1345 void *in;
1346 int err;
1347
1348 in = kzalloc(MLX5_ST_SZ_BYTES(create_tir_in), GFP_KERNEL);
1349 if (!in)
1350 return -ENOMEM;
1351
1352 MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid);
1353 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1354 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
1355
1356 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
1357 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);
1358 rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
1359 memcpy(rss_key, rx_hash_toeplitz_key, sizeof(rx_hash_toeplitz_key));
1360
1361 outer = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1362 MLX5_SET(rx_hash_field_select, outer, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4);
1363 MLX5_SET(rx_hash_field_select, outer, l4_prot_type, MLX5_L4_PROT_TYPE_TCP);
1364 MLX5_SET(rx_hash_field_select, outer, selected_fields, HASH_IP_L4PORTS);
1365
1366 MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn);
1367 MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn);
1368
1369 err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn);
1370 kfree(in);
1371 return err;
1372 }
1373
destroy_tir(struct mlx5_vdpa_net * ndev)1374 static void destroy_tir(struct mlx5_vdpa_net *ndev)
1375 {
1376 mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn);
1377 }
1378
add_fwd_to_tir(struct mlx5_vdpa_net * ndev)1379 static int add_fwd_to_tir(struct mlx5_vdpa_net *ndev)
1380 {
1381 struct mlx5_flow_destination dest[2] = {};
1382 struct mlx5_flow_table_attr ft_attr = {};
1383 struct mlx5_flow_act flow_act = {};
1384 struct mlx5_flow_namespace *ns;
1385 int err;
1386
1387 /* for now, one entry, match all, forward to tir */
1388 ft_attr.max_fte = 1;
1389 ft_attr.autogroup.max_num_groups = 1;
1390
1391 ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS);
1392 if (!ns) {
1393 mlx5_vdpa_warn(&ndev->mvdev, "get flow namespace\n");
1394 return -EOPNOTSUPP;
1395 }
1396
1397 ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
1398 if (IS_ERR(ndev->rxft))
1399 return PTR_ERR(ndev->rxft);
1400
1401 ndev->rx_counter = mlx5_fc_create(ndev->mvdev.mdev, false);
1402 if (IS_ERR(ndev->rx_counter)) {
1403 err = PTR_ERR(ndev->rx_counter);
1404 goto err_fc;
1405 }
1406
1407 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_COUNT;
1408 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_TIR;
1409 dest[0].tir_num = ndev->res.tirn;
1410 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1411 dest[1].counter_id = mlx5_fc_id(ndev->rx_counter);
1412 ndev->rx_rule = mlx5_add_flow_rules(ndev->rxft, NULL, &flow_act, dest, 2);
1413 if (IS_ERR(ndev->rx_rule)) {
1414 err = PTR_ERR(ndev->rx_rule);
1415 ndev->rx_rule = NULL;
1416 goto err_rule;
1417 }
1418
1419 return 0;
1420
1421 err_rule:
1422 mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter);
1423 err_fc:
1424 mlx5_destroy_flow_table(ndev->rxft);
1425 return err;
1426 }
1427
remove_fwd_to_tir(struct mlx5_vdpa_net * ndev)1428 static void remove_fwd_to_tir(struct mlx5_vdpa_net *ndev)
1429 {
1430 if (!ndev->rx_rule)
1431 return;
1432
1433 mlx5_del_flow_rules(ndev->rx_rule);
1434 mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter);
1435 mlx5_destroy_flow_table(ndev->rxft);
1436
1437 ndev->rx_rule = NULL;
1438 }
1439
handle_ctrl_mac(struct mlx5_vdpa_dev * mvdev,u8 cmd)1440 static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd)
1441 {
1442 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1443 struct mlx5_control_vq *cvq = &mvdev->cvq;
1444 virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1445 struct mlx5_core_dev *pfmdev;
1446 size_t read;
1447 u8 mac[ETH_ALEN];
1448
1449 pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev));
1450 switch (cmd) {
1451 case VIRTIO_NET_CTRL_MAC_ADDR_SET:
1452 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)mac, ETH_ALEN);
1453 if (read != ETH_ALEN)
1454 break;
1455
1456 if (!memcmp(ndev->config.mac, mac, 6)) {
1457 status = VIRTIO_NET_OK;
1458 break;
1459 }
1460
1461 if (!is_zero_ether_addr(ndev->config.mac)) {
1462 if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) {
1463 mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n",
1464 ndev->config.mac);
1465 break;
1466 }
1467 }
1468
1469 if (mlx5_mpfs_add_mac(pfmdev, mac)) {
1470 mlx5_vdpa_warn(mvdev, "failed to insert new MAC %pM into MPFS table\n",
1471 mac);
1472 break;
1473 }
1474
1475 memcpy(ndev->config.mac, mac, ETH_ALEN);
1476 status = VIRTIO_NET_OK;
1477 break;
1478
1479 default:
1480 break;
1481 }
1482
1483 return status;
1484 }
1485
change_num_qps(struct mlx5_vdpa_dev * mvdev,int newqps)1486 static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps)
1487 {
1488 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1489 int cur_qps = ndev->cur_num_vqs / 2;
1490 int err;
1491 int i;
1492
1493 if (cur_qps > newqps) {
1494 err = modify_rqt(ndev, 2 * newqps);
1495 if (err)
1496 return err;
1497
1498 for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--)
1499 teardown_vq(ndev, &ndev->vqs[i]);
1500
1501 ndev->cur_num_vqs = 2 * newqps;
1502 } else {
1503 ndev->cur_num_vqs = 2 * newqps;
1504 for (i = cur_qps * 2; i < 2 * newqps; i++) {
1505 err = setup_vq(ndev, &ndev->vqs[i]);
1506 if (err)
1507 goto clean_added;
1508 }
1509 err = modify_rqt(ndev, 2 * newqps);
1510 if (err)
1511 goto clean_added;
1512 }
1513 return 0;
1514
1515 clean_added:
1516 for (--i; i >= 2 * cur_qps; --i)
1517 teardown_vq(ndev, &ndev->vqs[i]);
1518
1519 ndev->cur_num_vqs = 2 * cur_qps;
1520
1521 return err;
1522 }
1523
handle_ctrl_mq(struct mlx5_vdpa_dev * mvdev,u8 cmd)1524 static virtio_net_ctrl_ack handle_ctrl_mq(struct mlx5_vdpa_dev *mvdev, u8 cmd)
1525 {
1526 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1527 virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1528 struct mlx5_control_vq *cvq = &mvdev->cvq;
1529 struct virtio_net_ctrl_mq mq;
1530 size_t read;
1531 u16 newqps;
1532
1533 switch (cmd) {
1534 case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET:
1535 /* This mq feature check aligns with pre-existing userspace
1536 * implementation.
1537 *
1538 * Without it, an untrusted driver could fake a multiqueue config
1539 * request down to a non-mq device that may cause kernel to
1540 * panic due to uninitialized resources for extra vqs. Even with
1541 * a well behaving guest driver, it is not expected to allow
1542 * changing the number of vqs on a non-mq device.
1543 */
1544 if (!MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ))
1545 break;
1546
1547 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)&mq, sizeof(mq));
1548 if (read != sizeof(mq))
1549 break;
1550
1551 newqps = mlx5vdpa16_to_cpu(mvdev, mq.virtqueue_pairs);
1552 if (newqps < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
1553 newqps > mlx5_vdpa_max_qps(mvdev->max_vqs))
1554 break;
1555
1556 if (ndev->cur_num_vqs == 2 * newqps) {
1557 status = VIRTIO_NET_OK;
1558 break;
1559 }
1560
1561 if (newqps & (newqps - 1))
1562 break;
1563
1564 if (!change_num_qps(mvdev, newqps))
1565 status = VIRTIO_NET_OK;
1566
1567 break;
1568 default:
1569 break;
1570 }
1571
1572 return status;
1573 }
1574
mlx5_cvq_kick_handler(struct work_struct * work)1575 static void mlx5_cvq_kick_handler(struct work_struct *work)
1576 {
1577 virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1578 struct virtio_net_ctrl_hdr ctrl;
1579 struct mlx5_vdpa_wq_ent *wqent;
1580 struct mlx5_vdpa_dev *mvdev;
1581 struct mlx5_control_vq *cvq;
1582 struct mlx5_vdpa_net *ndev;
1583 size_t read, write;
1584 int err;
1585
1586 wqent = container_of(work, struct mlx5_vdpa_wq_ent, work);
1587 mvdev = wqent->mvdev;
1588 ndev = to_mlx5_vdpa_ndev(mvdev);
1589 cvq = &mvdev->cvq;
1590 if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)))
1591 return;
1592
1593 if (!cvq->ready)
1594 return;
1595
1596 while (true) {
1597 err = vringh_getdesc_iotlb(&cvq->vring, &cvq->riov, &cvq->wiov, &cvq->head,
1598 GFP_ATOMIC);
1599 if (err <= 0)
1600 break;
1601
1602 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &ctrl, sizeof(ctrl));
1603 if (read != sizeof(ctrl))
1604 break;
1605
1606 switch (ctrl.class) {
1607 case VIRTIO_NET_CTRL_MAC:
1608 status = handle_ctrl_mac(mvdev, ctrl.cmd);
1609 break;
1610 case VIRTIO_NET_CTRL_MQ:
1611 status = handle_ctrl_mq(mvdev, ctrl.cmd);
1612 break;
1613
1614 default:
1615 break;
1616 }
1617
1618 /* Make sure data is written before advancing index */
1619 smp_wmb();
1620
1621 write = vringh_iov_push_iotlb(&cvq->vring, &cvq->wiov, &status, sizeof(status));
1622 vringh_complete_iotlb(&cvq->vring, cvq->head, write);
1623 vringh_kiov_cleanup(&cvq->riov);
1624 vringh_kiov_cleanup(&cvq->wiov);
1625
1626 if (vringh_need_notify_iotlb(&cvq->vring))
1627 vringh_notify(&cvq->vring);
1628
1629 queue_work(mvdev->wq, &wqent->work);
1630 break;
1631 }
1632 }
1633
mlx5_vdpa_kick_vq(struct vdpa_device * vdev,u16 idx)1634 static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx)
1635 {
1636 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1637 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1638 struct mlx5_vdpa_virtqueue *mvq;
1639
1640 if (!is_index_valid(mvdev, idx))
1641 return;
1642
1643 if (unlikely(is_ctrl_vq_idx(mvdev, idx))) {
1644 if (!mvdev->wq || !mvdev->cvq.ready)
1645 return;
1646
1647 queue_work(mvdev->wq, &ndev->cvq_ent.work);
1648 return;
1649 }
1650
1651 mvq = &ndev->vqs[idx];
1652 if (unlikely(!mvq->ready))
1653 return;
1654
1655 iowrite16(idx, ndev->mvdev.res.kick_addr);
1656 }
1657
mlx5_vdpa_set_vq_address(struct vdpa_device * vdev,u16 idx,u64 desc_area,u64 driver_area,u64 device_area)1658 static int mlx5_vdpa_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area,
1659 u64 driver_area, u64 device_area)
1660 {
1661 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1662 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1663 struct mlx5_vdpa_virtqueue *mvq;
1664
1665 if (!is_index_valid(mvdev, idx))
1666 return -EINVAL;
1667
1668 if (is_ctrl_vq_idx(mvdev, idx)) {
1669 mvdev->cvq.desc_addr = desc_area;
1670 mvdev->cvq.device_addr = device_area;
1671 mvdev->cvq.driver_addr = driver_area;
1672 return 0;
1673 }
1674
1675 mvq = &ndev->vqs[idx];
1676 mvq->desc_addr = desc_area;
1677 mvq->device_addr = device_area;
1678 mvq->driver_addr = driver_area;
1679 return 0;
1680 }
1681
mlx5_vdpa_set_vq_num(struct vdpa_device * vdev,u16 idx,u32 num)1682 static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num)
1683 {
1684 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1685 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1686 struct mlx5_vdpa_virtqueue *mvq;
1687
1688 if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx))
1689 return;
1690
1691 mvq = &ndev->vqs[idx];
1692 mvq->num_ent = num;
1693 }
1694
mlx5_vdpa_set_vq_cb(struct vdpa_device * vdev,u16 idx,struct vdpa_callback * cb)1695 static void mlx5_vdpa_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb)
1696 {
1697 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1698 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1699
1700 ndev->event_cbs[idx] = *cb;
1701 if (is_ctrl_vq_idx(mvdev, idx))
1702 mvdev->cvq.event_cb = *cb;
1703 }
1704
mlx5_cvq_notify(struct vringh * vring)1705 static void mlx5_cvq_notify(struct vringh *vring)
1706 {
1707 struct mlx5_control_vq *cvq = container_of(vring, struct mlx5_control_vq, vring);
1708
1709 if (!cvq->event_cb.callback)
1710 return;
1711
1712 cvq->event_cb.callback(cvq->event_cb.private);
1713 }
1714
set_cvq_ready(struct mlx5_vdpa_dev * mvdev,bool ready)1715 static void set_cvq_ready(struct mlx5_vdpa_dev *mvdev, bool ready)
1716 {
1717 struct mlx5_control_vq *cvq = &mvdev->cvq;
1718
1719 cvq->ready = ready;
1720 if (!ready)
1721 return;
1722
1723 cvq->vring.notify = mlx5_cvq_notify;
1724 }
1725
mlx5_vdpa_set_vq_ready(struct vdpa_device * vdev,u16 idx,bool ready)1726 static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready)
1727 {
1728 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1729 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1730 struct mlx5_vdpa_virtqueue *mvq;
1731
1732 if (!mvdev->actual_features)
1733 return;
1734
1735 if (!is_index_valid(mvdev, idx))
1736 return;
1737
1738 if (is_ctrl_vq_idx(mvdev, idx)) {
1739 set_cvq_ready(mvdev, ready);
1740 return;
1741 }
1742
1743 mvq = &ndev->vqs[idx];
1744 if (!ready)
1745 suspend_vq(ndev, mvq);
1746
1747 mvq->ready = ready;
1748 }
1749
mlx5_vdpa_get_vq_ready(struct vdpa_device * vdev,u16 idx)1750 static bool mlx5_vdpa_get_vq_ready(struct vdpa_device *vdev, u16 idx)
1751 {
1752 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1753 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1754
1755 if (!is_index_valid(mvdev, idx))
1756 return false;
1757
1758 if (is_ctrl_vq_idx(mvdev, idx))
1759 return mvdev->cvq.ready;
1760
1761 return ndev->vqs[idx].ready;
1762 }
1763
mlx5_vdpa_set_vq_state(struct vdpa_device * vdev,u16 idx,const struct vdpa_vq_state * state)1764 static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx,
1765 const struct vdpa_vq_state *state)
1766 {
1767 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1768 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1769 struct mlx5_vdpa_virtqueue *mvq;
1770
1771 if (!is_index_valid(mvdev, idx))
1772 return -EINVAL;
1773
1774 if (is_ctrl_vq_idx(mvdev, idx)) {
1775 mvdev->cvq.vring.last_avail_idx = state->split.avail_index;
1776 return 0;
1777 }
1778
1779 mvq = &ndev->vqs[idx];
1780 if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) {
1781 mlx5_vdpa_warn(mvdev, "can't modify available index\n");
1782 return -EINVAL;
1783 }
1784
1785 mvq->used_idx = state->split.avail_index;
1786 mvq->avail_idx = state->split.avail_index;
1787 return 0;
1788 }
1789
mlx5_vdpa_get_vq_state(struct vdpa_device * vdev,u16 idx,struct vdpa_vq_state * state)1790 static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state)
1791 {
1792 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1793 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1794 struct mlx5_vdpa_virtqueue *mvq;
1795 struct mlx5_virtq_attr attr;
1796 int err;
1797
1798 if (!is_index_valid(mvdev, idx))
1799 return -EINVAL;
1800
1801 if (is_ctrl_vq_idx(mvdev, idx)) {
1802 state->split.avail_index = mvdev->cvq.vring.last_avail_idx;
1803 return 0;
1804 }
1805
1806 mvq = &ndev->vqs[idx];
1807 /* If the virtq object was destroyed, use the value saved at
1808 * the last minute of suspend_vq. This caters for userspace
1809 * that cares about emulating the index after vq is stopped.
1810 */
1811 if (!mvq->initialized) {
1812 /* Firmware returns a wrong value for the available index.
1813 * Since both values should be identical, we take the value of
1814 * used_idx which is reported correctly.
1815 */
1816 state->split.avail_index = mvq->used_idx;
1817 return 0;
1818 }
1819
1820 err = query_virtqueue(ndev, mvq, &attr);
1821 if (err) {
1822 mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n");
1823 return err;
1824 }
1825 state->split.avail_index = attr.used_index;
1826 return 0;
1827 }
1828
mlx5_vdpa_get_vq_align(struct vdpa_device * vdev)1829 static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev)
1830 {
1831 return PAGE_SIZE;
1832 }
1833
1834 enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9,
1835 MLX5_VIRTIO_NET_F_CSUM = 1 << 10,
1836 MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11,
1837 MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12,
1838 };
1839
mlx_to_vritio_features(u16 dev_features)1840 static u64 mlx_to_vritio_features(u16 dev_features)
1841 {
1842 u64 result = 0;
1843
1844 if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM)
1845 result |= BIT_ULL(VIRTIO_NET_F_GUEST_CSUM);
1846 if (dev_features & MLX5_VIRTIO_NET_F_CSUM)
1847 result |= BIT_ULL(VIRTIO_NET_F_CSUM);
1848 if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6)
1849 result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO6);
1850 if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4)
1851 result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO4);
1852
1853 return result;
1854 }
1855
mlx5_vdpa_get_features(struct vdpa_device * vdev)1856 static u64 mlx5_vdpa_get_features(struct vdpa_device *vdev)
1857 {
1858 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1859 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1860 u16 dev_features;
1861
1862 dev_features = MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, device_features_bits_mask);
1863 ndev->mvdev.mlx_features |= mlx_to_vritio_features(dev_features);
1864 if (MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, virtio_version_1_0))
1865 ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_F_VERSION_1);
1866 ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_F_ACCESS_PLATFORM);
1867 ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VQ);
1868 ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR);
1869 ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_MQ);
1870 ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_STATUS);
1871
1872 print_features(mvdev, ndev->mvdev.mlx_features, false);
1873 return ndev->mvdev.mlx_features;
1874 }
1875
verify_driver_features(struct mlx5_vdpa_dev * mvdev,u64 features)1876 static int verify_driver_features(struct mlx5_vdpa_dev *mvdev, u64 features)
1877 {
1878 /* Minimum features to expect */
1879 if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)))
1880 return -EOPNOTSUPP;
1881
1882 /* Double check features combination sent down by the driver.
1883 * Fail invalid features due to absence of the depended feature.
1884 *
1885 * Per VIRTIO v1.1 specification, section 5.1.3.1 Feature bit
1886 * requirements: "VIRTIO_NET_F_MQ Requires VIRTIO_NET_F_CTRL_VQ".
1887 * By failing the invalid features sent down by untrusted drivers,
1888 * we're assured the assumption made upon is_index_valid() and
1889 * is_ctrl_vq_idx() will not be compromised.
1890 */
1891 if ((features & (BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) ==
1892 BIT_ULL(VIRTIO_NET_F_MQ))
1893 return -EINVAL;
1894
1895 return 0;
1896 }
1897
setup_virtqueues(struct mlx5_vdpa_dev * mvdev)1898 static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev)
1899 {
1900 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1901 int err;
1902 int i;
1903
1904 for (i = 0; i < 2 * mlx5_vdpa_max_qps(mvdev->max_vqs); i++) {
1905 err = setup_vq(ndev, &ndev->vqs[i]);
1906 if (err)
1907 goto err_vq;
1908 }
1909
1910 return 0;
1911
1912 err_vq:
1913 for (--i; i >= 0; i--)
1914 teardown_vq(ndev, &ndev->vqs[i]);
1915
1916 return err;
1917 }
1918
teardown_virtqueues(struct mlx5_vdpa_net * ndev)1919 static void teardown_virtqueues(struct mlx5_vdpa_net *ndev)
1920 {
1921 struct mlx5_vdpa_virtqueue *mvq;
1922 int i;
1923
1924 for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) {
1925 mvq = &ndev->vqs[i];
1926 if (!mvq->initialized)
1927 continue;
1928
1929 teardown_vq(ndev, mvq);
1930 }
1931 }
1932
update_cvq_info(struct mlx5_vdpa_dev * mvdev)1933 static void update_cvq_info(struct mlx5_vdpa_dev *mvdev)
1934 {
1935 if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_CTRL_VQ)) {
1936 if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) {
1937 /* MQ supported. CVQ index is right above the last data virtqueue's */
1938 mvdev->max_idx = mvdev->max_vqs;
1939 } else {
1940 /* Only CVQ supportted. data virtqueues occupy indices 0 and 1.
1941 * CVQ gets index 2
1942 */
1943 mvdev->max_idx = 2;
1944 }
1945 } else {
1946 /* Two data virtqueues only: one for rx and one for tx */
1947 mvdev->max_idx = 1;
1948 }
1949 }
1950
mlx5_vdpa_set_features(struct vdpa_device * vdev,u64 features)1951 static int mlx5_vdpa_set_features(struct vdpa_device *vdev, u64 features)
1952 {
1953 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1954 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1955 int err;
1956
1957 print_features(mvdev, features, true);
1958
1959 err = verify_driver_features(mvdev, features);
1960 if (err)
1961 return err;
1962
1963 ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features;
1964 ndev->config.mtu = cpu_to_mlx5vdpa16(mvdev, ndev->mtu);
1965 ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP);
1966 update_cvq_info(mvdev);
1967 return err;
1968 }
1969
mlx5_vdpa_set_config_cb(struct vdpa_device * vdev,struct vdpa_callback * cb)1970 static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb)
1971 {
1972 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1973 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1974
1975 ndev->config_cb = *cb;
1976 }
1977
1978 #define MLX5_VDPA_MAX_VQ_ENTRIES 256
mlx5_vdpa_get_vq_num_max(struct vdpa_device * vdev)1979 static u16 mlx5_vdpa_get_vq_num_max(struct vdpa_device *vdev)
1980 {
1981 return MLX5_VDPA_MAX_VQ_ENTRIES;
1982 }
1983
mlx5_vdpa_get_device_id(struct vdpa_device * vdev)1984 static u32 mlx5_vdpa_get_device_id(struct vdpa_device *vdev)
1985 {
1986 return VIRTIO_ID_NET;
1987 }
1988
mlx5_vdpa_get_vendor_id(struct vdpa_device * vdev)1989 static u32 mlx5_vdpa_get_vendor_id(struct vdpa_device *vdev)
1990 {
1991 return PCI_VENDOR_ID_MELLANOX;
1992 }
1993
mlx5_vdpa_get_status(struct vdpa_device * vdev)1994 static u8 mlx5_vdpa_get_status(struct vdpa_device *vdev)
1995 {
1996 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1997 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1998
1999 print_status(mvdev, ndev->mvdev.status, false);
2000 return ndev->mvdev.status;
2001 }
2002
save_channel_info(struct mlx5_vdpa_net * ndev,struct mlx5_vdpa_virtqueue * mvq)2003 static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
2004 {
2005 struct mlx5_vq_restore_info *ri = &mvq->ri;
2006 struct mlx5_virtq_attr attr = {};
2007 int err;
2008
2009 if (mvq->initialized) {
2010 err = query_virtqueue(ndev, mvq, &attr);
2011 if (err)
2012 return err;
2013 }
2014
2015 ri->avail_index = attr.available_index;
2016 ri->used_index = attr.used_index;
2017 ri->ready = mvq->ready;
2018 ri->num_ent = mvq->num_ent;
2019 ri->desc_addr = mvq->desc_addr;
2020 ri->device_addr = mvq->device_addr;
2021 ri->driver_addr = mvq->driver_addr;
2022 ri->restore = true;
2023 return 0;
2024 }
2025
save_channels_info(struct mlx5_vdpa_net * ndev)2026 static int save_channels_info(struct mlx5_vdpa_net *ndev)
2027 {
2028 int i;
2029
2030 for (i = 0; i < ndev->mvdev.max_vqs; i++) {
2031 memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri));
2032 save_channel_info(ndev, &ndev->vqs[i]);
2033 }
2034 return 0;
2035 }
2036
mlx5_clear_vqs(struct mlx5_vdpa_net * ndev)2037 static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev)
2038 {
2039 int i;
2040
2041 for (i = 0; i < ndev->mvdev.max_vqs; i++)
2042 memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri));
2043 }
2044
restore_channels_info(struct mlx5_vdpa_net * ndev)2045 static void restore_channels_info(struct mlx5_vdpa_net *ndev)
2046 {
2047 struct mlx5_vdpa_virtqueue *mvq;
2048 struct mlx5_vq_restore_info *ri;
2049 int i;
2050
2051 mlx5_clear_vqs(ndev);
2052 init_mvqs(ndev);
2053 for (i = 0; i < ndev->mvdev.max_vqs; i++) {
2054 mvq = &ndev->vqs[i];
2055 ri = &mvq->ri;
2056 if (!ri->restore)
2057 continue;
2058
2059 mvq->avail_idx = ri->avail_index;
2060 mvq->used_idx = ri->used_index;
2061 mvq->ready = ri->ready;
2062 mvq->num_ent = ri->num_ent;
2063 mvq->desc_addr = ri->desc_addr;
2064 mvq->device_addr = ri->device_addr;
2065 mvq->driver_addr = ri->driver_addr;
2066 }
2067 }
2068
mlx5_vdpa_change_map(struct mlx5_vdpa_dev * mvdev,struct vhost_iotlb * iotlb)2069 static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb)
2070 {
2071 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
2072 int err;
2073
2074 suspend_vqs(ndev);
2075 err = save_channels_info(ndev);
2076 if (err)
2077 goto err_mr;
2078
2079 teardown_driver(ndev);
2080 mlx5_vdpa_destroy_mr(mvdev);
2081 err = mlx5_vdpa_create_mr(mvdev, iotlb);
2082 if (err)
2083 goto err_mr;
2084
2085 if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
2086 return 0;
2087
2088 restore_channels_info(ndev);
2089 err = setup_driver(mvdev);
2090 if (err)
2091 goto err_setup;
2092
2093 return 0;
2094
2095 err_setup:
2096 mlx5_vdpa_destroy_mr(mvdev);
2097 err_mr:
2098 return err;
2099 }
2100
setup_driver(struct mlx5_vdpa_dev * mvdev)2101 static int setup_driver(struct mlx5_vdpa_dev *mvdev)
2102 {
2103 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
2104 int err;
2105
2106 mutex_lock(&ndev->reslock);
2107 if (ndev->setup) {
2108 mlx5_vdpa_warn(mvdev, "setup driver called for already setup driver\n");
2109 err = 0;
2110 goto out;
2111 }
2112 err = setup_virtqueues(mvdev);
2113 if (err) {
2114 mlx5_vdpa_warn(mvdev, "setup_virtqueues\n");
2115 goto out;
2116 }
2117
2118 err = create_rqt(ndev);
2119 if (err) {
2120 mlx5_vdpa_warn(mvdev, "create_rqt\n");
2121 goto err_rqt;
2122 }
2123
2124 err = create_tir(ndev);
2125 if (err) {
2126 mlx5_vdpa_warn(mvdev, "create_tir\n");
2127 goto err_tir;
2128 }
2129
2130 err = add_fwd_to_tir(ndev);
2131 if (err) {
2132 mlx5_vdpa_warn(mvdev, "add_fwd_to_tir\n");
2133 goto err_fwd;
2134 }
2135 ndev->setup = true;
2136 mutex_unlock(&ndev->reslock);
2137
2138 return 0;
2139
2140 err_fwd:
2141 destroy_tir(ndev);
2142 err_tir:
2143 destroy_rqt(ndev);
2144 err_rqt:
2145 teardown_virtqueues(ndev);
2146 out:
2147 mutex_unlock(&ndev->reslock);
2148 return err;
2149 }
2150
teardown_driver(struct mlx5_vdpa_net * ndev)2151 static void teardown_driver(struct mlx5_vdpa_net *ndev)
2152 {
2153 mutex_lock(&ndev->reslock);
2154 if (!ndev->setup)
2155 goto out;
2156
2157 remove_fwd_to_tir(ndev);
2158 destroy_tir(ndev);
2159 destroy_rqt(ndev);
2160 teardown_virtqueues(ndev);
2161 ndev->setup = false;
2162 out:
2163 mutex_unlock(&ndev->reslock);
2164 }
2165
clear_vqs_ready(struct mlx5_vdpa_net * ndev)2166 static void clear_vqs_ready(struct mlx5_vdpa_net *ndev)
2167 {
2168 int i;
2169
2170 for (i = 0; i < ndev->mvdev.max_vqs; i++)
2171 ndev->vqs[i].ready = false;
2172
2173 ndev->mvdev.cvq.ready = false;
2174 }
2175
setup_cvq_vring(struct mlx5_vdpa_dev * mvdev)2176 static int setup_cvq_vring(struct mlx5_vdpa_dev *mvdev)
2177 {
2178 struct mlx5_control_vq *cvq = &mvdev->cvq;
2179 int err = 0;
2180
2181 if (mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)) {
2182 u16 idx = cvq->vring.last_avail_idx;
2183
2184 err = vringh_init_iotlb(&cvq->vring, mvdev->actual_features,
2185 MLX5_CVQ_MAX_ENT, false,
2186 (struct vring_desc *)(uintptr_t)cvq->desc_addr,
2187 (struct vring_avail *)(uintptr_t)cvq->driver_addr,
2188 (struct vring_used *)(uintptr_t)cvq->device_addr);
2189
2190 if (!err)
2191 cvq->vring.last_avail_idx = cvq->vring.last_used_idx = idx;
2192 }
2193 return err;
2194 }
2195
mlx5_vdpa_set_status(struct vdpa_device * vdev,u8 status)2196 static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status)
2197 {
2198 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
2199 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
2200 int err;
2201
2202 print_status(mvdev, status, true);
2203
2204 if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) {
2205 if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
2206 err = setup_cvq_vring(mvdev);
2207 if (err) {
2208 mlx5_vdpa_warn(mvdev, "failed to setup control VQ vring\n");
2209 goto err_setup;
2210 }
2211 err = setup_driver(mvdev);
2212 if (err) {
2213 mlx5_vdpa_warn(mvdev, "failed to setup driver\n");
2214 goto err_setup;
2215 }
2216 } else {
2217 mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n");
2218 return;
2219 }
2220 }
2221
2222 ndev->mvdev.status = status;
2223 return;
2224
2225 err_setup:
2226 mlx5_vdpa_destroy_mr(&ndev->mvdev);
2227 ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED;
2228 }
2229
mlx5_vdpa_reset(struct vdpa_device * vdev)2230 static int mlx5_vdpa_reset(struct vdpa_device *vdev)
2231 {
2232 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
2233 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
2234
2235 print_status(mvdev, 0, true);
2236 mlx5_vdpa_info(mvdev, "performing device reset\n");
2237 teardown_driver(ndev);
2238 clear_vqs_ready(ndev);
2239 mlx5_vdpa_destroy_mr(&ndev->mvdev);
2240 ndev->mvdev.status = 0;
2241 memset(ndev->event_cbs, 0, sizeof(ndev->event_cbs));
2242 ndev->mvdev.actual_features = 0;
2243 ++mvdev->generation;
2244 if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) {
2245 if (mlx5_vdpa_create_mr(mvdev, NULL))
2246 mlx5_vdpa_warn(mvdev, "create MR failed\n");
2247 }
2248
2249 return 0;
2250 }
2251
mlx5_vdpa_get_config_size(struct vdpa_device * vdev)2252 static size_t mlx5_vdpa_get_config_size(struct vdpa_device *vdev)
2253 {
2254 return sizeof(struct virtio_net_config);
2255 }
2256
mlx5_vdpa_get_config(struct vdpa_device * vdev,unsigned int offset,void * buf,unsigned int len)2257 static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf,
2258 unsigned int len)
2259 {
2260 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
2261 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
2262
2263 if (offset + len <= sizeof(struct virtio_net_config))
2264 memcpy(buf, (u8 *)&ndev->config + offset, len);
2265 }
2266
mlx5_vdpa_set_config(struct vdpa_device * vdev,unsigned int offset,const void * buf,unsigned int len)2267 static void mlx5_vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, const void *buf,
2268 unsigned int len)
2269 {
2270 /* not supported */
2271 }
2272
mlx5_vdpa_get_generation(struct vdpa_device * vdev)2273 static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev)
2274 {
2275 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
2276
2277 return mvdev->generation;
2278 }
2279
mlx5_vdpa_set_map(struct vdpa_device * vdev,struct vhost_iotlb * iotlb)2280 static int mlx5_vdpa_set_map(struct vdpa_device *vdev, struct vhost_iotlb *iotlb)
2281 {
2282 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
2283 bool change_map;
2284 int err;
2285
2286 err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map);
2287 if (err) {
2288 mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err);
2289 return err;
2290 }
2291
2292 if (change_map)
2293 return mlx5_vdpa_change_map(mvdev, iotlb);
2294
2295 return 0;
2296 }
2297
mlx5_vdpa_free(struct vdpa_device * vdev)2298 static void mlx5_vdpa_free(struct vdpa_device *vdev)
2299 {
2300 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
2301 struct mlx5_core_dev *pfmdev;
2302 struct mlx5_vdpa_net *ndev;
2303
2304 ndev = to_mlx5_vdpa_ndev(mvdev);
2305
2306 free_resources(ndev);
2307 mlx5_vdpa_destroy_mr(mvdev);
2308 if (!is_zero_ether_addr(ndev->config.mac)) {
2309 pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev));
2310 mlx5_mpfs_del_mac(pfmdev, ndev->config.mac);
2311 }
2312 mlx5_vdpa_free_resources(&ndev->mvdev);
2313 mutex_destroy(&ndev->reslock);
2314 }
2315
mlx5_get_vq_notification(struct vdpa_device * vdev,u16 idx)2316 static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device *vdev, u16 idx)
2317 {
2318 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
2319 struct vdpa_notification_area ret = {};
2320 struct mlx5_vdpa_net *ndev;
2321 phys_addr_t addr;
2322
2323 if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx))
2324 return ret;
2325
2326 /* If SF BAR size is smaller than PAGE_SIZE, do not use direct
2327 * notification to avoid the risk of mapping pages that contain BAR of more
2328 * than one SF
2329 */
2330 if (MLX5_CAP_GEN(mvdev->mdev, log_min_sf_size) + 12 < PAGE_SHIFT)
2331 return ret;
2332
2333 ndev = to_mlx5_vdpa_ndev(mvdev);
2334 addr = (phys_addr_t)ndev->mvdev.res.phys_kick_addr;
2335 ret.addr = addr;
2336 ret.size = PAGE_SIZE;
2337 return ret;
2338 }
2339
mlx5_get_vq_irq(struct vdpa_device * vdv,u16 idx)2340 static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx)
2341 {
2342 return -EOPNOTSUPP;
2343 }
2344
2345 static const struct vdpa_config_ops mlx5_vdpa_ops = {
2346 .set_vq_address = mlx5_vdpa_set_vq_address,
2347 .set_vq_num = mlx5_vdpa_set_vq_num,
2348 .kick_vq = mlx5_vdpa_kick_vq,
2349 .set_vq_cb = mlx5_vdpa_set_vq_cb,
2350 .set_vq_ready = mlx5_vdpa_set_vq_ready,
2351 .get_vq_ready = mlx5_vdpa_get_vq_ready,
2352 .set_vq_state = mlx5_vdpa_set_vq_state,
2353 .get_vq_state = mlx5_vdpa_get_vq_state,
2354 .get_vq_notification = mlx5_get_vq_notification,
2355 .get_vq_irq = mlx5_get_vq_irq,
2356 .get_vq_align = mlx5_vdpa_get_vq_align,
2357 .get_features = mlx5_vdpa_get_features,
2358 .set_features = mlx5_vdpa_set_features,
2359 .set_config_cb = mlx5_vdpa_set_config_cb,
2360 .get_vq_num_max = mlx5_vdpa_get_vq_num_max,
2361 .get_device_id = mlx5_vdpa_get_device_id,
2362 .get_vendor_id = mlx5_vdpa_get_vendor_id,
2363 .get_status = mlx5_vdpa_get_status,
2364 .set_status = mlx5_vdpa_set_status,
2365 .reset = mlx5_vdpa_reset,
2366 .get_config_size = mlx5_vdpa_get_config_size,
2367 .get_config = mlx5_vdpa_get_config,
2368 .set_config = mlx5_vdpa_set_config,
2369 .get_generation = mlx5_vdpa_get_generation,
2370 .set_map = mlx5_vdpa_set_map,
2371 .free = mlx5_vdpa_free,
2372 };
2373
query_mtu(struct mlx5_core_dev * mdev,u16 * mtu)2374 static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu)
2375 {
2376 u16 hw_mtu;
2377 int err;
2378
2379 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
2380 if (err)
2381 return err;
2382
2383 *mtu = hw_mtu - MLX5V_ETH_HARD_MTU;
2384 return 0;
2385 }
2386
alloc_resources(struct mlx5_vdpa_net * ndev)2387 static int alloc_resources(struct mlx5_vdpa_net *ndev)
2388 {
2389 struct mlx5_vdpa_net_resources *res = &ndev->res;
2390 int err;
2391
2392 if (res->valid) {
2393 mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n");
2394 return -EEXIST;
2395 }
2396
2397 err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn);
2398 if (err)
2399 return err;
2400
2401 err = create_tis(ndev);
2402 if (err)
2403 goto err_tis;
2404
2405 res->valid = true;
2406
2407 return 0;
2408
2409 err_tis:
2410 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn);
2411 return err;
2412 }
2413
free_resources(struct mlx5_vdpa_net * ndev)2414 static void free_resources(struct mlx5_vdpa_net *ndev)
2415 {
2416 struct mlx5_vdpa_net_resources *res = &ndev->res;
2417
2418 if (!res->valid)
2419 return;
2420
2421 destroy_tis(ndev);
2422 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn);
2423 res->valid = false;
2424 }
2425
init_mvqs(struct mlx5_vdpa_net * ndev)2426 static void init_mvqs(struct mlx5_vdpa_net *ndev)
2427 {
2428 struct mlx5_vdpa_virtqueue *mvq;
2429 int i;
2430
2431 for (i = 0; i < 2 * mlx5_vdpa_max_qps(ndev->mvdev.max_vqs); ++i) {
2432 mvq = &ndev->vqs[i];
2433 memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri));
2434 mvq->index = i;
2435 mvq->ndev = ndev;
2436 mvq->fwqp.fw = true;
2437 }
2438 for (; i < ndev->mvdev.max_vqs; i++) {
2439 mvq = &ndev->vqs[i];
2440 memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri));
2441 mvq->index = i;
2442 mvq->ndev = ndev;
2443 }
2444 }
2445
2446 struct mlx5_vdpa_mgmtdev {
2447 struct vdpa_mgmt_dev mgtdev;
2448 struct mlx5_adev *madev;
2449 struct mlx5_vdpa_net *ndev;
2450 };
2451
query_vport_state(struct mlx5_core_dev * mdev,u8 opmod,u16 vport)2452 static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport)
2453 {
2454 u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {};
2455 u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {};
2456 int err;
2457
2458 MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE);
2459 MLX5_SET(query_vport_state_in, in, op_mod, opmod);
2460 MLX5_SET(query_vport_state_in, in, vport_number, vport);
2461 if (vport)
2462 MLX5_SET(query_vport_state_in, in, other_vport, 1);
2463
2464 err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out);
2465 if (err)
2466 return 0;
2467
2468 return MLX5_GET(query_vport_state_out, out, state);
2469 }
2470
get_link_state(struct mlx5_vdpa_dev * mvdev)2471 static bool get_link_state(struct mlx5_vdpa_dev *mvdev)
2472 {
2473 if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) ==
2474 VPORT_STATE_UP)
2475 return true;
2476
2477 return false;
2478 }
2479
update_carrier(struct work_struct * work)2480 static void update_carrier(struct work_struct *work)
2481 {
2482 struct mlx5_vdpa_wq_ent *wqent;
2483 struct mlx5_vdpa_dev *mvdev;
2484 struct mlx5_vdpa_net *ndev;
2485
2486 wqent = container_of(work, struct mlx5_vdpa_wq_ent, work);
2487 mvdev = wqent->mvdev;
2488 ndev = to_mlx5_vdpa_ndev(mvdev);
2489 if (get_link_state(mvdev))
2490 ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP);
2491 else
2492 ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP);
2493
2494 if (ndev->config_cb.callback)
2495 ndev->config_cb.callback(ndev->config_cb.private);
2496
2497 kfree(wqent);
2498 }
2499
event_handler(struct notifier_block * nb,unsigned long event,void * param)2500 static int event_handler(struct notifier_block *nb, unsigned long event, void *param)
2501 {
2502 struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb);
2503 struct mlx5_eqe *eqe = param;
2504 int ret = NOTIFY_DONE;
2505 struct mlx5_vdpa_wq_ent *wqent;
2506
2507 if (event == MLX5_EVENT_TYPE_PORT_CHANGE) {
2508 switch (eqe->sub_type) {
2509 case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
2510 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
2511 wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);
2512 if (!wqent)
2513 return NOTIFY_DONE;
2514
2515 wqent->mvdev = &ndev->mvdev;
2516 INIT_WORK(&wqent->work, update_carrier);
2517 queue_work(ndev->mvdev.wq, &wqent->work);
2518 ret = NOTIFY_OK;
2519 break;
2520 default:
2521 return NOTIFY_DONE;
2522 }
2523 return ret;
2524 }
2525 return ret;
2526 }
2527
mlx5_vdpa_dev_add(struct vdpa_mgmt_dev * v_mdev,const char * name)2528 static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name)
2529 {
2530 struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev);
2531 struct virtio_net_config *config;
2532 struct mlx5_core_dev *pfmdev;
2533 struct mlx5_vdpa_dev *mvdev;
2534 struct mlx5_vdpa_net *ndev;
2535 struct mlx5_core_dev *mdev;
2536 u32 max_vqs;
2537 int err;
2538
2539 if (mgtdev->ndev)
2540 return -ENOSPC;
2541
2542 mdev = mgtdev->madev->mdev;
2543 if (!(MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_queue_type) &
2544 MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT)) {
2545 dev_warn(mdev->device, "missing support for split virtqueues\n");
2546 return -EOPNOTSUPP;
2547 }
2548
2549 /* we save one virtqueue for control virtqueue should we require it */
2550 max_vqs = MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues);
2551 max_vqs = min_t(u32, max_vqs, MLX5_MAX_SUPPORTED_VQS);
2552
2553 ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops,
2554 name, false);
2555 if (IS_ERR(ndev))
2556 return PTR_ERR(ndev);
2557
2558 ndev->mvdev.max_vqs = max_vqs;
2559 mvdev = &ndev->mvdev;
2560 mvdev->mdev = mdev;
2561 init_mvqs(ndev);
2562 mutex_init(&ndev->reslock);
2563 config = &ndev->config;
2564 err = query_mtu(mdev, &ndev->mtu);
2565 if (err)
2566 goto err_mtu;
2567
2568 err = mlx5_query_nic_vport_mac_address(mdev, 0, 0, config->mac);
2569 if (err)
2570 goto err_mtu;
2571
2572 if (get_link_state(mvdev))
2573 ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP);
2574 else
2575 ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP);
2576
2577 if (!is_zero_ether_addr(config->mac)) {
2578 pfmdev = pci_get_drvdata(pci_physfn(mdev->pdev));
2579 err = mlx5_mpfs_add_mac(pfmdev, config->mac);
2580 if (err)
2581 goto err_mtu;
2582
2583 ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_MAC);
2584 }
2585
2586 config->max_virtqueue_pairs = cpu_to_mlx5vdpa16(mvdev, mlx5_vdpa_max_qps(max_vqs));
2587 mvdev->vdev.dma_dev = &mdev->pdev->dev;
2588 err = mlx5_vdpa_alloc_resources(&ndev->mvdev);
2589 if (err)
2590 goto err_mpfs;
2591
2592 if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) {
2593 err = mlx5_vdpa_create_mr(mvdev, NULL);
2594 if (err)
2595 goto err_res;
2596 }
2597
2598 err = alloc_resources(ndev);
2599 if (err)
2600 goto err_mr;
2601
2602 ndev->cvq_ent.mvdev = mvdev;
2603 INIT_WORK(&ndev->cvq_ent.work, mlx5_cvq_kick_handler);
2604 mvdev->wq = create_singlethread_workqueue("mlx5_vdpa_wq");
2605 if (!mvdev->wq) {
2606 err = -ENOMEM;
2607 goto err_res2;
2608 }
2609
2610 ndev->nb.notifier_call = event_handler;
2611 mlx5_notifier_register(mdev, &ndev->nb);
2612 ndev->cur_num_vqs = 2 * mlx5_vdpa_max_qps(max_vqs);
2613 mvdev->vdev.mdev = &mgtdev->mgtdev;
2614 err = _vdpa_register_device(&mvdev->vdev, ndev->cur_num_vqs + 1);
2615 if (err)
2616 goto err_reg;
2617
2618 mgtdev->ndev = ndev;
2619 return 0;
2620
2621 err_reg:
2622 destroy_workqueue(mvdev->wq);
2623 err_res2:
2624 free_resources(ndev);
2625 err_mr:
2626 mlx5_vdpa_destroy_mr(mvdev);
2627 err_res:
2628 mlx5_vdpa_free_resources(&ndev->mvdev);
2629 err_mpfs:
2630 if (!is_zero_ether_addr(config->mac))
2631 mlx5_mpfs_del_mac(pfmdev, config->mac);
2632 err_mtu:
2633 mutex_destroy(&ndev->reslock);
2634 put_device(&mvdev->vdev.dev);
2635 return err;
2636 }
2637
mlx5_vdpa_dev_del(struct vdpa_mgmt_dev * v_mdev,struct vdpa_device * dev)2638 static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *dev)
2639 {
2640 struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev);
2641 struct mlx5_vdpa_dev *mvdev = to_mvdev(dev);
2642 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
2643 struct workqueue_struct *wq;
2644
2645 mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
2646 wq = mvdev->wq;
2647 mvdev->wq = NULL;
2648 destroy_workqueue(wq);
2649 _vdpa_unregister_device(dev);
2650 mgtdev->ndev = NULL;
2651 }
2652
2653 static const struct vdpa_mgmtdev_ops mdev_ops = {
2654 .dev_add = mlx5_vdpa_dev_add,
2655 .dev_del = mlx5_vdpa_dev_del,
2656 };
2657
2658 static struct virtio_device_id id_table[] = {
2659 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
2660 { 0 },
2661 };
2662
mlx5v_probe(struct auxiliary_device * adev,const struct auxiliary_device_id * id)2663 static int mlx5v_probe(struct auxiliary_device *adev,
2664 const struct auxiliary_device_id *id)
2665
2666 {
2667 struct mlx5_adev *madev = container_of(adev, struct mlx5_adev, adev);
2668 struct mlx5_core_dev *mdev = madev->mdev;
2669 struct mlx5_vdpa_mgmtdev *mgtdev;
2670 int err;
2671
2672 mgtdev = kzalloc(sizeof(*mgtdev), GFP_KERNEL);
2673 if (!mgtdev)
2674 return -ENOMEM;
2675
2676 mgtdev->mgtdev.ops = &mdev_ops;
2677 mgtdev->mgtdev.device = mdev->device;
2678 mgtdev->mgtdev.id_table = id_table;
2679 mgtdev->madev = madev;
2680
2681 err = vdpa_mgmtdev_register(&mgtdev->mgtdev);
2682 if (err)
2683 goto reg_err;
2684
2685 dev_set_drvdata(&adev->dev, mgtdev);
2686
2687 return 0;
2688
2689 reg_err:
2690 kfree(mgtdev);
2691 return err;
2692 }
2693
mlx5v_remove(struct auxiliary_device * adev)2694 static void mlx5v_remove(struct auxiliary_device *adev)
2695 {
2696 struct mlx5_vdpa_mgmtdev *mgtdev;
2697
2698 mgtdev = dev_get_drvdata(&adev->dev);
2699 vdpa_mgmtdev_unregister(&mgtdev->mgtdev);
2700 kfree(mgtdev);
2701 }
2702
2703 static const struct auxiliary_device_id mlx5v_id_table[] = {
2704 { .name = MLX5_ADEV_NAME ".vnet", },
2705 {},
2706 };
2707
2708 MODULE_DEVICE_TABLE(auxiliary, mlx5v_id_table);
2709
2710 static struct auxiliary_driver mlx5v_driver = {
2711 .name = "vnet",
2712 .probe = mlx5v_probe,
2713 .remove = mlx5v_remove,
2714 .id_table = mlx5v_id_table,
2715 };
2716
2717 module_auxiliary_driver(mlx5v_driver);
2718