1 /*
2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #ifndef __MLX5_EN_H__
33 #define __MLX5_EN_H__
34
35 #include <linux/if_vlan.h>
36 #include <linux/etherdevice.h>
37 #include <linux/timecounter.h>
38 #include <linux/net_tstamp.h>
39 #include <linux/crash_dump.h>
40 #include <linux/mlx5/driver.h>
41 #include <linux/mlx5/qp.h>
42 #include <linux/mlx5/cq.h>
43 #include <linux/mlx5/port.h>
44 #include <linux/mlx5/vport.h>
45 #include <linux/mlx5/transobj.h>
46 #include <linux/mlx5/fs.h>
47 #include <linux/rhashtable.h>
48 #include <net/udp_tunnel.h>
49 #include <net/switchdev.h>
50 #include <net/xdp.h>
51 #include <linux/dim.h>
52 #include <linux/bits.h>
53 #include "wq.h"
54 #include "mlx5_core.h"
55 #include "en_stats.h"
56 #include "en/dcbnl.h"
57 #include "en/fs.h"
58 #include "lib/hv_vhca.h"
59
60 extern const struct net_device_ops mlx5e_netdev_ops;
61 struct page_pool;
62
63 #define MLX5E_METADATA_ETHER_TYPE (0x8CE4)
64 #define MLX5E_METADATA_ETHER_LEN 8
65
66 #define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v)
67
68 #define MLX5E_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN)
69
70 #define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu))
71 #define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu))
72
73 #define MLX5E_MAX_NUM_TC 8
74
75 #define MLX5_RX_HEADROOM NET_SKB_PAD
76 #define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \
77 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
78
79 #define MLX5E_RX_MAX_HEAD (256)
80
81 #define MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev) \
82 (6 + MLX5_CAP_GEN(mdev, cache_line_128byte)) /* HW restriction */
83 #define MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, req) \
84 max_t(u32, MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev), req)
85 #define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) \
86 MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, order_base_2(MLX5E_RX_MAX_HEAD))
87
88 #define MLX5_MPWRQ_LOG_WQE_SZ 18
89 #define MLX5_MPWRQ_WQE_PAGE_ORDER (MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT > 0 ? \
90 MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT : 0)
91 #define MLX5_MPWRQ_PAGES_PER_WQE BIT(MLX5_MPWRQ_WQE_PAGE_ORDER)
92
93 #define MLX5_ALIGN_MTTS(mtts) (ALIGN(mtts, 8))
94 #define MLX5_ALIGNED_MTTS_OCTW(mtts) ((mtts) / 2)
95 #define MLX5_MTT_OCTW(mtts) (MLX5_ALIGNED_MTTS_OCTW(MLX5_ALIGN_MTTS(mtts)))
96 /* Add another page to MLX5E_REQUIRED_WQE_MTTS as a buffer between
97 * WQEs, This page will absorb write overflow by the hardware, when
98 * receiving packets larger than MTU. These oversize packets are
99 * dropped by the driver at a later stage.
100 */
101 #define MLX5E_REQUIRED_WQE_MTTS (MLX5_ALIGN_MTTS(MLX5_MPWRQ_PAGES_PER_WQE + 1))
102 #define MLX5E_REQUIRED_MTTS(wqes) (wqes * MLX5E_REQUIRED_WQE_MTTS)
103 #define MLX5E_MAX_RQ_NUM_MTTS \
104 (ALIGN_DOWN(U16_MAX, 4) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */
105 #define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024))
106 #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW \
107 (ilog2(MLX5E_MAX_RQ_NUM_MTTS / MLX5E_REQUIRED_WQE_MTTS))
108 #define MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW \
109 (MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW + \
110 (MLX5_MPWRQ_LOG_WQE_SZ - MLX5E_ORDER2_MAX_PACKET_MTU))
111
112 #define MLX5E_MIN_SKB_FRAG_SZ (MLX5_SKB_FRAG_SZ(MLX5_RX_HEADROOM))
113 #define MLX5E_LOG_MAX_RX_WQE_BULK \
114 (ilog2(PAGE_SIZE / roundup_pow_of_two(MLX5E_MIN_SKB_FRAG_SZ)))
115
116 #define MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE 0x6
117 #define MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE 0xa
118 #define MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE 0xd
119
120 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE (1 + MLX5E_LOG_MAX_RX_WQE_BULK)
121 #define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE 0xa
122 #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE min_t(u8, 0xd, \
123 MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW)
124
125 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW 0x2
126
127 #define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024)
128 #define MLX5E_DEFAULT_LRO_TIMEOUT 32
129 #define MLX5E_LRO_TIMEOUT_ARR_SIZE 4
130
131 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC 0x10
132 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE 0x3
133 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS 0x20
134 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC 0x10
135 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE 0x10
136 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS 0x20
137 #define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES 0x80
138 #define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES_MPW 0x2
139
140 #define MLX5E_LOG_INDIR_RQT_SIZE 0x7
141 #define MLX5E_INDIR_RQT_SIZE BIT(MLX5E_LOG_INDIR_RQT_SIZE)
142 #define MLX5E_MIN_NUM_CHANNELS 0x1
143 #define MLX5E_MAX_NUM_CHANNELS MLX5E_INDIR_RQT_SIZE
144 #define MLX5E_MAX_NUM_SQS (MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC)
145 #define MLX5E_TX_CQ_POLL_BUDGET 128
146 #define MLX5E_TX_XSK_POLL_BUDGET 64
147 #define MLX5E_SQ_RECOVER_MIN_INTERVAL 500 /* msecs */
148
149 #define MLX5E_UMR_WQE_INLINE_SZ \
150 (sizeof(struct mlx5e_umr_wqe) + \
151 ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(struct mlx5_mtt), \
152 MLX5_UMR_MTT_ALIGNMENT))
153 #define MLX5E_UMR_WQEBBS \
154 (DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_BB))
155
156 #define MLX5E_MSG_LEVEL NETIF_MSG_LINK
157
158 #define mlx5e_dbg(mlevel, priv, format, ...) \
159 do { \
160 if (NETIF_MSG_##mlevel & (priv)->msglevel) \
161 netdev_warn(priv->netdev, format, \
162 ##__VA_ARGS__); \
163 } while (0)
164
165 enum mlx5e_rq_group {
166 MLX5E_RQ_GROUP_REGULAR,
167 MLX5E_RQ_GROUP_XSK,
168 #define MLX5E_NUM_RQ_GROUPS(g) (1 + MLX5E_RQ_GROUP_##g)
169 };
170
mlx5e_get_num_lag_ports(struct mlx5_core_dev * mdev)171 static inline u8 mlx5e_get_num_lag_ports(struct mlx5_core_dev *mdev)
172 {
173 if (mlx5_lag_is_lacp_owner(mdev))
174 return 1;
175
176 return clamp_t(u8, MLX5_CAP_GEN(mdev, num_lag_ports), 1, MLX5_MAX_PORTS);
177 }
178
mlx5_min_rx_wqes(int wq_type,u32 wq_size)179 static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size)
180 {
181 switch (wq_type) {
182 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
183 return min_t(u16, MLX5E_PARAMS_DEFAULT_MIN_RX_WQES_MPW,
184 wq_size / 2);
185 default:
186 return min_t(u16, MLX5E_PARAMS_DEFAULT_MIN_RX_WQES,
187 wq_size / 2);
188 }
189 }
190
191 /* Use this function to get max num channels (rxqs/txqs) only to create netdev */
mlx5e_get_max_num_channels(struct mlx5_core_dev * mdev)192 static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
193 {
194 return is_kdump_kernel() ?
195 MLX5E_MIN_NUM_CHANNELS :
196 min_t(int, mlx5_comp_vectors_count(mdev), MLX5E_MAX_NUM_CHANNELS);
197 }
198
199 struct mlx5e_tx_wqe {
200 struct mlx5_wqe_ctrl_seg ctrl;
201 struct mlx5_wqe_eth_seg eth;
202 struct mlx5_wqe_data_seg data[0];
203 };
204
205 struct mlx5e_rx_wqe_ll {
206 struct mlx5_wqe_srq_next_seg next;
207 struct mlx5_wqe_data_seg data[];
208 };
209
210 struct mlx5e_rx_wqe_cyc {
211 struct mlx5_wqe_data_seg data[0];
212 };
213
214 struct mlx5e_umr_wqe {
215 struct mlx5_wqe_ctrl_seg ctrl;
216 struct mlx5_wqe_umr_ctrl_seg uctrl;
217 struct mlx5_mkey_seg mkc;
218 struct mlx5_mtt inline_mtts[0];
219 };
220
221 extern const char mlx5e_self_tests[][ETH_GSTRING_LEN];
222
223 enum mlx5e_priv_flag {
224 MLX5E_PFLAG_RX_CQE_BASED_MODER,
225 MLX5E_PFLAG_TX_CQE_BASED_MODER,
226 MLX5E_PFLAG_RX_CQE_COMPRESS,
227 MLX5E_PFLAG_RX_STRIDING_RQ,
228 MLX5E_PFLAG_RX_NO_CSUM_COMPLETE,
229 MLX5E_PFLAG_XDP_TX_MPWQE,
230 MLX5E_PFLAG_SKB_TX_MPWQE,
231 MLX5E_NUM_PFLAGS, /* Keep last */
232 };
233
234 #define MLX5E_SET_PFLAG(params, pflag, enable) \
235 do { \
236 if (enable) \
237 (params)->pflags |= BIT(pflag); \
238 else \
239 (params)->pflags &= ~(BIT(pflag)); \
240 } while (0)
241
242 #define MLX5E_GET_PFLAG(params, pflag) (!!((params)->pflags & (BIT(pflag))))
243
244 struct mlx5e_params {
245 u8 log_sq_size;
246 u8 rq_wq_type;
247 u8 log_rq_mtu_frames;
248 u16 num_channels;
249 u8 num_tc;
250 bool rx_cqe_compress_def;
251 bool tunneled_offload_en;
252 struct dim_cq_moder rx_cq_moderation;
253 struct dim_cq_moder tx_cq_moderation;
254 bool lro_en;
255 u8 tx_min_inline_mode;
256 bool vlan_strip_disable;
257 bool scatter_fcs_en;
258 bool rx_dim_enabled;
259 bool tx_dim_enabled;
260 u32 lro_timeout;
261 u32 pflags;
262 struct bpf_prog *xdp_prog;
263 struct mlx5e_xsk *xsk;
264 unsigned int sw_mtu;
265 int hard_mtu;
266 };
267
268 enum {
269 MLX5E_RQ_STATE_ENABLED,
270 MLX5E_RQ_STATE_RECOVERING,
271 MLX5E_RQ_STATE_AM,
272 MLX5E_RQ_STATE_NO_CSUM_COMPLETE,
273 MLX5E_RQ_STATE_CSUM_FULL, /* cqe_csum_full hw bit is set */
274 MLX5E_RQ_STATE_FPGA_TLS, /* FPGA TLS enabled */
275 MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX /* set when mini_cqe_resp_stride_index cap is used */
276 };
277
278 struct mlx5e_cq {
279 /* data path - accessed per cqe */
280 struct mlx5_cqwq wq;
281
282 /* data path - accessed per napi poll */
283 u16 event_ctr;
284 struct napi_struct *napi;
285 struct mlx5_core_cq mcq;
286 struct mlx5e_channel *channel;
287
288 /* control */
289 struct mlx5_core_dev *mdev;
290 struct mlx5_wq_ctrl wq_ctrl;
291 } ____cacheline_aligned_in_smp;
292
293 struct mlx5e_cq_decomp {
294 /* cqe decompression */
295 struct mlx5_cqe64 title;
296 struct mlx5_mini_cqe8 mini_arr[MLX5_MINI_CQE_ARRAY_SIZE];
297 u8 mini_arr_idx;
298 u16 left;
299 u16 wqe_counter;
300 } ____cacheline_aligned_in_smp;
301
302 enum mlx5e_dma_map_type {
303 MLX5E_DMA_MAP_SINGLE,
304 MLX5E_DMA_MAP_PAGE
305 };
306
307 struct mlx5e_sq_dma {
308 dma_addr_t addr;
309 u32 size;
310 enum mlx5e_dma_map_type type;
311 };
312
313 enum {
314 MLX5E_SQ_STATE_ENABLED,
315 MLX5E_SQ_STATE_MPWQE,
316 MLX5E_SQ_STATE_RECOVERING,
317 MLX5E_SQ_STATE_IPSEC,
318 MLX5E_SQ_STATE_AM,
319 MLX5E_SQ_STATE_TLS,
320 MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE,
321 MLX5E_SQ_STATE_PENDING_XSK_TX,
322 };
323
324 struct mlx5e_tx_mpwqe {
325 /* Current MPWQE session */
326 struct mlx5e_tx_wqe *wqe;
327 u32 bytes_count;
328 u8 ds_count;
329 u8 pkt_count;
330 u8 inline_on;
331 };
332
333 struct mlx5e_txqsq {
334 /* data path */
335
336 /* dirtied @completion */
337 u16 cc;
338 u16 skb_fifo_cc;
339 u32 dma_fifo_cc;
340 struct dim dim; /* Adaptive Moderation */
341
342 /* dirtied @xmit */
343 u16 pc ____cacheline_aligned_in_smp;
344 u16 skb_fifo_pc;
345 u32 dma_fifo_pc;
346 struct mlx5e_tx_mpwqe mpwqe;
347
348 struct mlx5e_cq cq;
349
350 /* read only */
351 struct mlx5_wq_cyc wq;
352 u32 dma_fifo_mask;
353 u16 skb_fifo_mask;
354 struct mlx5e_sq_stats *stats;
355 struct {
356 struct mlx5e_sq_dma *dma_fifo;
357 struct sk_buff **skb_fifo;
358 struct mlx5e_tx_wqe_info *wqe_info;
359 } db;
360 void __iomem *uar_map;
361 struct netdev_queue *txq;
362 u32 sqn;
363 u16 stop_room;
364 u8 min_inline_mode;
365 struct device *pdev;
366 __be32 mkey_be;
367 unsigned long state;
368 unsigned int hw_mtu;
369 struct hwtstamp_config *tstamp;
370 struct mlx5_clock *clock;
371
372 /* control path */
373 struct mlx5_wq_ctrl wq_ctrl;
374 struct mlx5e_channel *channel;
375 int ch_ix;
376 int txq_ix;
377 u32 rate_limit;
378 struct work_struct recover_work;
379 } ____cacheline_aligned_in_smp;
380
381 struct mlx5e_dma_info {
382 dma_addr_t addr;
383 union {
384 struct page *page;
385 struct xdp_buff *xsk;
386 };
387 };
388
389 /* XDP packets can be transmitted in different ways. On completion, we need to
390 * distinguish between them to clean up things in a proper way.
391 */
392 enum mlx5e_xdp_xmit_mode {
393 /* An xdp_frame was transmitted due to either XDP_REDIRECT from another
394 * device or XDP_TX from an XSK RQ. The frame has to be unmapped and
395 * returned.
396 */
397 MLX5E_XDP_XMIT_MODE_FRAME,
398
399 /* The xdp_frame was created in place as a result of XDP_TX from a
400 * regular RQ. No DMA remapping happened, and the page belongs to us.
401 */
402 MLX5E_XDP_XMIT_MODE_PAGE,
403
404 /* No xdp_frame was created at all, the transmit happened from a UMEM
405 * page. The UMEM Completion Ring producer pointer has to be increased.
406 */
407 MLX5E_XDP_XMIT_MODE_XSK,
408 };
409
410 struct mlx5e_xdp_info {
411 enum mlx5e_xdp_xmit_mode mode;
412 union {
413 struct {
414 struct xdp_frame *xdpf;
415 dma_addr_t dma_addr;
416 } frame;
417 struct {
418 struct mlx5e_rq *rq;
419 struct mlx5e_dma_info di;
420 } page;
421 };
422 };
423
424 struct mlx5e_xmit_data {
425 dma_addr_t dma_addr;
426 void *data;
427 u32 len;
428 };
429
430 struct mlx5e_xdp_info_fifo {
431 struct mlx5e_xdp_info *xi;
432 u32 *cc;
433 u32 *pc;
434 u32 mask;
435 };
436
437 struct mlx5e_xdpsq;
438 typedef int (*mlx5e_fp_xmit_xdp_frame_check)(struct mlx5e_xdpsq *);
439 typedef bool (*mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq *,
440 struct mlx5e_xmit_data *,
441 struct mlx5e_xdp_info *,
442 int);
443
444 struct mlx5e_xdpsq {
445 /* data path */
446
447 /* dirtied @completion */
448 u32 xdpi_fifo_cc;
449 u16 cc;
450
451 /* dirtied @xmit */
452 u32 xdpi_fifo_pc ____cacheline_aligned_in_smp;
453 u16 pc;
454 struct mlx5_wqe_ctrl_seg *doorbell_cseg;
455 struct mlx5e_tx_mpwqe mpwqe;
456
457 struct mlx5e_cq cq;
458
459 /* read only */
460 struct xsk_buff_pool *xsk_pool;
461 struct mlx5_wq_cyc wq;
462 struct mlx5e_xdpsq_stats *stats;
463 mlx5e_fp_xmit_xdp_frame_check xmit_xdp_frame_check;
464 mlx5e_fp_xmit_xdp_frame xmit_xdp_frame;
465 struct {
466 struct mlx5e_xdp_wqe_info *wqe_info;
467 struct mlx5e_xdp_info_fifo xdpi_fifo;
468 } db;
469 void __iomem *uar_map;
470 u32 sqn;
471 struct device *pdev;
472 __be32 mkey_be;
473 u8 min_inline_mode;
474 unsigned long state;
475 unsigned int hw_mtu;
476
477 /* control path */
478 struct mlx5_wq_ctrl wq_ctrl;
479 struct mlx5e_channel *channel;
480 } ____cacheline_aligned_in_smp;
481
482 struct mlx5e_icosq {
483 /* data path */
484 u16 cc;
485 u16 pc;
486
487 struct mlx5_wqe_ctrl_seg *doorbell_cseg;
488 struct mlx5e_cq cq;
489
490 /* write@xmit, read@completion */
491 struct {
492 struct mlx5e_icosq_wqe_info *wqe_info;
493 } db;
494
495 /* read only */
496 struct mlx5_wq_cyc wq;
497 void __iomem *uar_map;
498 u32 sqn;
499 unsigned long state;
500
501 /* control path */
502 struct mlx5_wq_ctrl wq_ctrl;
503 struct mlx5e_channel *channel;
504
505 struct work_struct recover_work;
506 } ____cacheline_aligned_in_smp;
507
508 struct mlx5e_wqe_frag_info {
509 struct mlx5e_dma_info *di;
510 u32 offset;
511 bool last_in_page;
512 };
513
514 struct mlx5e_umr_dma_info {
515 struct mlx5e_dma_info dma_info[MLX5_MPWRQ_PAGES_PER_WQE];
516 };
517
518 struct mlx5e_mpw_info {
519 struct mlx5e_umr_dma_info umr;
520 u16 consumed_strides;
521 DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
522 };
523
524 #define MLX5E_MAX_RX_FRAGS 4
525
526 /* a single cache unit is capable to serve one napi call (for non-striding rq)
527 * or a MPWQE (for striding rq).
528 */
529 #define MLX5E_CACHE_UNIT (MLX5_MPWRQ_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \
530 MLX5_MPWRQ_PAGES_PER_WQE : NAPI_POLL_WEIGHT)
531 #define MLX5E_CACHE_SIZE (4 * roundup_pow_of_two(MLX5E_CACHE_UNIT))
532 struct mlx5e_page_cache {
533 u32 head;
534 u32 tail;
535 struct mlx5e_dma_info page_cache[MLX5E_CACHE_SIZE];
536 };
537
538 struct mlx5e_rq;
539 typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*);
540 typedef struct sk_buff *
541 (*mlx5e_fp_skb_from_cqe_mpwrq)(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
542 u16 cqe_bcnt, u32 head_offset, u32 page_idx);
543 typedef struct sk_buff *
544 (*mlx5e_fp_skb_from_cqe)(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
545 struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt);
546 typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq);
547 typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);
548
549 int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk);
550
551 enum mlx5e_rq_flag {
552 MLX5E_RQ_FLAG_XDP_XMIT,
553 MLX5E_RQ_FLAG_XDP_REDIRECT,
554 };
555
556 struct mlx5e_rq_frag_info {
557 int frag_size;
558 int frag_stride;
559 };
560
561 struct mlx5e_rq_frags_info {
562 struct mlx5e_rq_frag_info arr[MLX5E_MAX_RX_FRAGS];
563 u8 num_frags;
564 u8 log_num_frags;
565 u8 wqe_bulk;
566 };
567
568 struct mlx5e_rq {
569 /* data path */
570 union {
571 struct {
572 struct mlx5_wq_cyc wq;
573 struct mlx5e_wqe_frag_info *frags;
574 struct mlx5e_dma_info *di;
575 struct mlx5e_rq_frags_info info;
576 mlx5e_fp_skb_from_cqe skb_from_cqe;
577 } wqe;
578 struct {
579 struct mlx5_wq_ll wq;
580 struct mlx5e_umr_wqe umr_wqe;
581 struct mlx5e_mpw_info *info;
582 mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq;
583 u16 num_strides;
584 u16 actual_wq_head;
585 u8 log_stride_sz;
586 u8 umr_in_progress;
587 u8 umr_last_bulk;
588 u8 umr_completed;
589 } mpwqe;
590 };
591 struct {
592 u16 headroom;
593 u32 frame0_sz;
594 u8 map_dir; /* dma map direction */
595 } buff;
596
597 struct mlx5e_channel *channel;
598 struct device *pdev;
599 struct net_device *netdev;
600 struct mlx5e_rq_stats *stats;
601 struct mlx5e_cq cq;
602 struct mlx5e_cq_decomp cqd;
603 struct mlx5e_page_cache page_cache;
604 struct hwtstamp_config *tstamp;
605 struct mlx5_clock *clock;
606
607 mlx5e_fp_handle_rx_cqe handle_rx_cqe;
608 mlx5e_fp_post_rx_wqes post_wqes;
609 mlx5e_fp_dealloc_wqe dealloc_wqe;
610
611 unsigned long state;
612 int ix;
613 unsigned int hw_mtu;
614
615 struct dim dim; /* Dynamic Interrupt Moderation */
616
617 /* XDP */
618 struct bpf_prog __rcu *xdp_prog;
619 struct mlx5e_xdpsq *xdpsq;
620 DECLARE_BITMAP(flags, 8);
621 struct page_pool *page_pool;
622
623 /* AF_XDP zero-copy */
624 struct xsk_buff_pool *xsk_pool;
625
626 struct work_struct recover_work;
627
628 /* control */
629 struct mlx5_wq_ctrl wq_ctrl;
630 __be32 mkey_be;
631 u8 wq_type;
632 u32 rqn;
633 struct mlx5_core_dev *mdev;
634 struct mlx5_core_mkey umr_mkey;
635 struct mlx5e_dma_info wqe_overflow;
636
637 /* XDP read-mostly */
638 struct xdp_rxq_info xdp_rxq;
639 } ____cacheline_aligned_in_smp;
640
641 enum mlx5e_channel_state {
642 MLX5E_CHANNEL_STATE_XSK,
643 MLX5E_CHANNEL_NUM_STATES
644 };
645
646 struct mlx5e_channel {
647 /* data path */
648 struct mlx5e_rq rq;
649 struct mlx5e_xdpsq rq_xdpsq;
650 struct mlx5e_txqsq sq[MLX5E_MAX_NUM_TC];
651 struct mlx5e_icosq icosq; /* internal control operations */
652 bool xdp;
653 struct napi_struct napi;
654 struct device *pdev;
655 struct net_device *netdev;
656 __be32 mkey_be;
657 u8 num_tc;
658 u8 lag_port;
659
660 /* XDP_REDIRECT */
661 struct mlx5e_xdpsq xdpsq;
662
663 /* AF_XDP zero-copy */
664 struct mlx5e_rq xskrq;
665 struct mlx5e_xdpsq xsksq;
666
667 /* Async ICOSQ */
668 struct mlx5e_icosq async_icosq;
669 /* async_icosq can be accessed from any CPU - the spinlock protects it. */
670 spinlock_t async_icosq_lock;
671
672 /* data path - accessed per napi poll */
673 struct irq_desc *irq_desc;
674 struct mlx5e_ch_stats *stats;
675
676 /* control */
677 struct mlx5e_priv *priv;
678 struct mlx5_core_dev *mdev;
679 struct hwtstamp_config *tstamp;
680 DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES);
681 int ix;
682 int cpu;
683 };
684
685 struct mlx5e_channels {
686 struct mlx5e_channel **c;
687 unsigned int num;
688 struct mlx5e_params params;
689 };
690
691 struct mlx5e_channel_stats {
692 struct mlx5e_ch_stats ch;
693 struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC];
694 struct mlx5e_rq_stats rq;
695 struct mlx5e_rq_stats xskrq;
696 struct mlx5e_xdpsq_stats rq_xdpsq;
697 struct mlx5e_xdpsq_stats xdpsq;
698 struct mlx5e_xdpsq_stats xsksq;
699 } ____cacheline_aligned_in_smp;
700
701 enum {
702 MLX5E_STATE_OPENED,
703 MLX5E_STATE_DESTROYING,
704 MLX5E_STATE_XDP_TX_ENABLED,
705 MLX5E_STATE_XDP_ACTIVE,
706 };
707
708 struct mlx5e_rqt {
709 u32 rqtn;
710 bool enabled;
711 };
712
713 struct mlx5e_tir {
714 u32 tirn;
715 struct mlx5e_rqt rqt;
716 struct list_head list;
717 };
718
719 enum {
720 MLX5E_TC_PRIO = 0,
721 MLX5E_NIC_PRIO
722 };
723
724 struct mlx5e_rss_params {
725 u32 indirection_rqt[MLX5E_INDIR_RQT_SIZE];
726 u32 rx_hash_fields[MLX5E_NUM_INDIR_TIRS];
727 u8 toeplitz_hash_key[40];
728 u8 hfunc;
729 };
730
731 struct mlx5e_modify_sq_param {
732 int curr_state;
733 int next_state;
734 int rl_update;
735 int rl_index;
736 };
737
738 #if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE)
739 struct mlx5e_hv_vhca_stats_agent {
740 struct mlx5_hv_vhca_agent *agent;
741 struct delayed_work work;
742 u16 delay;
743 void *buf;
744 };
745 #endif
746
747 struct mlx5e_xsk {
748 /* XSK buffer pools are stored separately from channels,
749 * because we don't want to lose them when channels are
750 * recreated. The kernel also stores buffer pool, but it doesn't
751 * distinguish between zero-copy and non-zero-copy UMEMs, so
752 * rely on our mechanism.
753 */
754 struct xsk_buff_pool **pools;
755 u16 refcnt;
756 bool ever_used;
757 };
758
759 /* Temporary storage for variables that are allocated when struct mlx5e_priv is
760 * initialized, and used where we can't allocate them because that functions
761 * must not fail. Use with care and make sure the same variable is not used
762 * simultaneously by multiple users.
763 */
764 struct mlx5e_scratchpad {
765 cpumask_var_t cpumask;
766 };
767
768 struct mlx5e_priv {
769 /* priv data path fields - start */
770 struct mlx5e_txqsq *txq2sq[MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC];
771 int channel_tc2realtxq[MLX5E_MAX_NUM_CHANNELS][MLX5E_MAX_NUM_TC];
772 #ifdef CONFIG_MLX5_CORE_EN_DCB
773 struct mlx5e_dcbx_dp dcbx_dp;
774 #endif
775 /* priv data path fields - end */
776
777 u32 msglevel;
778 unsigned long state;
779 struct mutex state_lock; /* Protects Interface state */
780 struct mlx5e_rq drop_rq;
781
782 struct mlx5e_channels channels;
783 u32 tisn[MLX5_MAX_PORTS][MLX5E_MAX_NUM_TC];
784 struct mlx5e_rqt indir_rqt;
785 struct mlx5e_tir indir_tir[MLX5E_NUM_INDIR_TIRS];
786 struct mlx5e_tir inner_indir_tir[MLX5E_NUM_INDIR_TIRS];
787 struct mlx5e_tir direct_tir[MLX5E_MAX_NUM_CHANNELS];
788 struct mlx5e_tir xsk_tir[MLX5E_MAX_NUM_CHANNELS];
789 struct mlx5e_rss_params rss_params;
790 u32 tx_rates[MLX5E_MAX_NUM_SQS];
791
792 struct mlx5e_flow_steering fs;
793
794 struct workqueue_struct *wq;
795 struct work_struct update_carrier_work;
796 struct work_struct set_rx_mode_work;
797 struct work_struct tx_timeout_work;
798 struct work_struct update_stats_work;
799 struct work_struct monitor_counters_work;
800 struct mlx5_nb monitor_counters_nb;
801
802 struct mlx5_core_dev *mdev;
803 struct net_device *netdev;
804 struct mlx5e_stats stats;
805 struct mlx5e_channel_stats channel_stats[MLX5E_MAX_NUM_CHANNELS];
806 u16 max_nch;
807 u8 max_opened_tc;
808 struct hwtstamp_config tstamp;
809 u16 q_counter;
810 u16 drop_rq_q_counter;
811 struct notifier_block events_nb;
812
813 struct udp_tunnel_nic_info nic_info;
814 #ifdef CONFIG_MLX5_CORE_EN_DCB
815 struct mlx5e_dcbx dcbx;
816 #endif
817
818 const struct mlx5e_profile *profile;
819 void *ppriv;
820 #ifdef CONFIG_MLX5_EN_IPSEC
821 struct mlx5e_ipsec *ipsec;
822 #endif
823 #ifdef CONFIG_MLX5_EN_TLS
824 struct mlx5e_tls *tls;
825 #endif
826 struct devlink_health_reporter *tx_reporter;
827 struct devlink_health_reporter *rx_reporter;
828 struct devlink_port dl_port;
829 struct mlx5e_xsk xsk;
830 #if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE)
831 struct mlx5e_hv_vhca_stats_agent stats_agent;
832 #endif
833 struct mlx5e_scratchpad scratchpad;
834 };
835
836 struct mlx5e_rx_handlers {
837 mlx5e_fp_handle_rx_cqe handle_rx_cqe;
838 mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe;
839 };
840
841 extern const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic;
842
843 struct mlx5e_profile {
844 int (*init)(struct mlx5_core_dev *mdev,
845 struct net_device *netdev,
846 const struct mlx5e_profile *profile, void *ppriv);
847 void (*cleanup)(struct mlx5e_priv *priv);
848 int (*init_rx)(struct mlx5e_priv *priv);
849 void (*cleanup_rx)(struct mlx5e_priv *priv);
850 int (*init_tx)(struct mlx5e_priv *priv);
851 void (*cleanup_tx)(struct mlx5e_priv *priv);
852 void (*enable)(struct mlx5e_priv *priv);
853 void (*disable)(struct mlx5e_priv *priv);
854 int (*update_rx)(struct mlx5e_priv *priv);
855 void (*update_stats)(struct mlx5e_priv *priv);
856 void (*update_carrier)(struct mlx5e_priv *priv);
857 unsigned int (*stats_grps_num)(struct mlx5e_priv *priv);
858 mlx5e_stats_grp_t *stats_grps;
859 const struct mlx5e_rx_handlers *rx_handlers;
860 int max_tc;
861 u8 rq_groups;
862 };
863
864 void mlx5e_build_ptys2ethtool_map(void);
865
866 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev);
867 bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,
868 struct mlx5e_params *params);
869
870 void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats);
871 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s);
872
873 void mlx5e_init_l2_addr(struct mlx5e_priv *priv);
874 int mlx5e_self_test_num(struct mlx5e_priv *priv);
875 void mlx5e_self_test(struct net_device *ndev, struct ethtool_test *etest,
876 u64 *buf);
877 void mlx5e_set_rx_mode_work(struct work_struct *work);
878
879 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr);
880 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr);
881 int mlx5e_modify_rx_cqe_compression_locked(struct mlx5e_priv *priv, bool val);
882
883 int mlx5e_vlan_rx_add_vid(struct net_device *dev, __always_unused __be16 proto,
884 u16 vid);
885 int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __always_unused __be16 proto,
886 u16 vid);
887 void mlx5e_timestamp_init(struct mlx5e_priv *priv);
888
889 struct mlx5e_redirect_rqt_param {
890 bool is_rss;
891 union {
892 u32 rqn; /* Direct RQN (Non-RSS) */
893 struct {
894 u8 hfunc;
895 struct mlx5e_channels *channels;
896 } rss; /* RSS data */
897 };
898 };
899
900 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz,
901 struct mlx5e_redirect_rqt_param rrp);
902 void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_rss_params *rss_params,
903 const struct mlx5e_tirc_config *ttconfig,
904 void *tirc, bool inner);
905 void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in);
906 struct mlx5e_tirc_config mlx5e_tirc_get_default_config(enum mlx5e_traffic_types tt);
907
908 struct mlx5e_xsk_param;
909
910 struct mlx5e_rq_param;
911 int mlx5e_open_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
912 struct mlx5e_rq_param *param, struct mlx5e_xsk_param *xsk,
913 struct xsk_buff_pool *xsk_pool, struct mlx5e_rq *rq);
914 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time);
915 void mlx5e_deactivate_rq(struct mlx5e_rq *rq);
916 void mlx5e_close_rq(struct mlx5e_rq *rq);
917
918 struct mlx5e_sq_param;
919 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
920 struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
921 struct mlx5e_xdpsq *sq, bool is_redirect);
922 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq);
923
924 struct mlx5e_cq_param;
925 int mlx5e_open_cq(struct mlx5e_channel *c, struct dim_cq_moder moder,
926 struct mlx5e_cq_param *param, struct mlx5e_cq *cq);
927 void mlx5e_close_cq(struct mlx5e_cq *cq);
928
929 int mlx5e_open_locked(struct net_device *netdev);
930 int mlx5e_close_locked(struct net_device *netdev);
931
932 int mlx5e_open_channels(struct mlx5e_priv *priv,
933 struct mlx5e_channels *chs);
934 void mlx5e_close_channels(struct mlx5e_channels *chs);
935
936 /* Function pointer to be used to modify HW or kernel settings while
937 * switching channels
938 */
939 typedef int (*mlx5e_fp_preactivate)(struct mlx5e_priv *priv, void *context);
940 #define MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(fn) \
941 int fn##_ctx(struct mlx5e_priv *priv, void *context) \
942 { \
943 return fn(priv); \
944 }
945 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv);
946 int mlx5e_safe_switch_channels(struct mlx5e_priv *priv,
947 struct mlx5e_channels *new_chs,
948 mlx5e_fp_preactivate preactivate,
949 void *context);
950 int mlx5e_num_channels_changed(struct mlx5e_priv *priv);
951 int mlx5e_num_channels_changed_ctx(struct mlx5e_priv *priv, void *context);
952 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv);
953 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv);
954
955 void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
956 int num_channels);
957
958 void mlx5e_reset_tx_moderation(struct mlx5e_params *params, u8 cq_period_mode);
959 void mlx5e_reset_rx_moderation(struct mlx5e_params *params, u8 cq_period_mode);
960 void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode);
961 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode);
962
963 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
964 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
965 struct mlx5e_params *params);
966 int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state);
967 void mlx5e_activate_rq(struct mlx5e_rq *rq);
968 void mlx5e_deactivate_rq(struct mlx5e_rq *rq);
969 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq);
970 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq);
971
972 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
973 struct mlx5e_modify_sq_param *p);
974 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq);
975 void mlx5e_tx_disable_queue(struct netdev_queue *txq);
976
mlx5_tx_swp_supported(struct mlx5_core_dev * mdev)977 static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev)
978 {
979 return MLX5_CAP_ETH(mdev, swp) &&
980 MLX5_CAP_ETH(mdev, swp_csum) && MLX5_CAP_ETH(mdev, swp_lso);
981 }
982
983 extern const struct ethtool_ops mlx5e_ethtool_ops;
984
985 int mlx5e_create_tir(struct mlx5_core_dev *mdev, struct mlx5e_tir *tir,
986 u32 *in);
987 void mlx5e_destroy_tir(struct mlx5_core_dev *mdev,
988 struct mlx5e_tir *tir);
989 int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev);
990 void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev);
991 int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb,
992 bool enable_mc_lb);
993 void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc);
994
995 /* common netdev helpers */
996 void mlx5e_create_q_counters(struct mlx5e_priv *priv);
997 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv);
998 int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
999 struct mlx5e_rq *drop_rq);
1000 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq);
1001
1002 int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv);
1003
1004 int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc);
1005 void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv);
1006
1007 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs);
1008 void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs);
1009 int mlx5e_create_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs);
1010 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs);
1011 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt);
1012
1013 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn);
1014 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn);
1015
1016 int mlx5e_create_tises(struct mlx5e_priv *priv);
1017 void mlx5e_destroy_tises(struct mlx5e_priv *priv);
1018 int mlx5e_update_nic_rx(struct mlx5e_priv *priv);
1019 void mlx5e_update_carrier(struct mlx5e_priv *priv);
1020 int mlx5e_close(struct net_device *netdev);
1021 int mlx5e_open(struct net_device *netdev);
1022
1023 void mlx5e_queue_update_stats(struct mlx5e_priv *priv);
1024 int mlx5e_bits_invert(unsigned long a, int size);
1025
1026 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv);
1027 int mlx5e_set_dev_port_mtu_ctx(struct mlx5e_priv *priv, void *context);
1028 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
1029 mlx5e_fp_preactivate preactivate);
1030 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv);
1031
1032 /* ethtool helpers */
1033 void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv,
1034 struct ethtool_drvinfo *drvinfo);
1035 void mlx5e_ethtool_get_strings(struct mlx5e_priv *priv,
1036 uint32_t stringset, uint8_t *data);
1037 int mlx5e_ethtool_get_sset_count(struct mlx5e_priv *priv, int sset);
1038 void mlx5e_ethtool_get_ethtool_stats(struct mlx5e_priv *priv,
1039 struct ethtool_stats *stats, u64 *data);
1040 void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv,
1041 struct ethtool_ringparam *param);
1042 int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv,
1043 struct ethtool_ringparam *param);
1044 void mlx5e_ethtool_get_channels(struct mlx5e_priv *priv,
1045 struct ethtool_channels *ch);
1046 int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv,
1047 struct ethtool_channels *ch);
1048 int mlx5e_ethtool_get_coalesce(struct mlx5e_priv *priv,
1049 struct ethtool_coalesce *coal);
1050 int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv,
1051 struct ethtool_coalesce *coal);
1052 int mlx5e_ethtool_get_link_ksettings(struct mlx5e_priv *priv,
1053 struct ethtool_link_ksettings *link_ksettings);
1054 int mlx5e_ethtool_set_link_ksettings(struct mlx5e_priv *priv,
1055 const struct ethtool_link_ksettings *link_ksettings);
1056 int mlx5e_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, u8 *hfunc);
1057 int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key,
1058 const u8 hfunc);
1059 int mlx5e_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1060 u32 *rule_locs);
1061 int mlx5e_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd);
1062 u32 mlx5e_ethtool_get_rxfh_key_size(struct mlx5e_priv *priv);
1063 u32 mlx5e_ethtool_get_rxfh_indir_size(struct mlx5e_priv *priv);
1064 int mlx5e_ethtool_get_ts_info(struct mlx5e_priv *priv,
1065 struct ethtool_ts_info *info);
1066 int mlx5e_ethtool_flash_device(struct mlx5e_priv *priv,
1067 struct ethtool_flash *flash);
1068 void mlx5e_ethtool_get_pauseparam(struct mlx5e_priv *priv,
1069 struct ethtool_pauseparam *pauseparam);
1070 int mlx5e_ethtool_set_pauseparam(struct mlx5e_priv *priv,
1071 struct ethtool_pauseparam *pauseparam);
1072
1073 /* mlx5e generic netdev management API */
1074 int mlx5e_netdev_init(struct net_device *netdev,
1075 struct mlx5e_priv *priv,
1076 struct mlx5_core_dev *mdev,
1077 const struct mlx5e_profile *profile,
1078 void *ppriv);
1079 void mlx5e_netdev_cleanup(struct net_device *netdev, struct mlx5e_priv *priv);
1080 struct net_device*
1081 mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile,
1082 int nch, void *ppriv);
1083 int mlx5e_attach_netdev(struct mlx5e_priv *priv);
1084 void mlx5e_detach_netdev(struct mlx5e_priv *priv);
1085 void mlx5e_destroy_netdev(struct mlx5e_priv *priv);
1086 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv);
1087 void mlx5e_build_nic_params(struct mlx5e_priv *priv,
1088 struct mlx5e_xsk *xsk,
1089 struct mlx5e_rss_params *rss_params,
1090 struct mlx5e_params *params,
1091 u16 mtu);
1092 void mlx5e_build_rq_params(struct mlx5_core_dev *mdev,
1093 struct mlx5e_params *params);
1094 void mlx5e_build_rss_params(struct mlx5e_rss_params *rss_params,
1095 u16 num_channels);
1096 void mlx5e_rx_dim_work(struct work_struct *work);
1097 void mlx5e_tx_dim_work(struct work_struct *work);
1098
1099 netdev_features_t mlx5e_features_check(struct sk_buff *skb,
1100 struct net_device *netdev,
1101 netdev_features_t features);
1102 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features);
1103 #ifdef CONFIG_MLX5_ESWITCH
1104 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac);
1105 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, int max_tx_rate);
1106 int mlx5e_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi);
1107 int mlx5e_get_vf_stats(struct net_device *dev, int vf, struct ifla_vf_stats *vf_stats);
1108 #endif
1109 #endif /* __MLX5_EN_H__ */
1110