1 /*
2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #ifndef __MLX5_EN_H__
33 #define __MLX5_EN_H__
34
35 #include <linux/if_vlan.h>
36 #include <linux/etherdevice.h>
37 #include <linux/timecounter.h>
38 #include <linux/net_tstamp.h>
39 #include <linux/crash_dump.h>
40 #include <linux/mlx5/driver.h>
41 #include <linux/mlx5/qp.h>
42 #include <linux/mlx5/cq.h>
43 #include <linux/mlx5/port.h>
44 #include <linux/mlx5/vport.h>
45 #include <linux/mlx5/transobj.h>
46 #include <linux/mlx5/fs.h>
47 #include <linux/rhashtable.h>
48 #include <net/udp_tunnel.h>
49 #include <net/switchdev.h>
50 #include <net/xdp.h>
51 #include <linux/dim.h>
52 #include <linux/bits.h>
53 #include "wq.h"
54 #include "mlx5_core.h"
55 #include "en_stats.h"
56 #include "en/dcbnl.h"
57 #include "en/fs.h"
58 #include "en/qos.h"
59 #include "lib/hv_vhca.h"
60 #include "lib/clock.h"
61 #include "en/rx_res.h"
62 #include "en/selq.h"
63 #include "lib/sd.h"
64
65 extern const struct net_device_ops mlx5e_netdev_ops;
66 struct page_pool;
67
68 #define MLX5E_METADATA_ETHER_TYPE (0x8CE4)
69 #define MLX5E_METADATA_ETHER_LEN 8
70
71 #define MLX5E_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN)
72
73 #define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu))
74 #define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu))
75
76 #define MLX5E_MAX_NUM_MQPRIO_CH_TC TC_QOPT_MAX_QUEUE
77
78 #define MLX5_RX_HEADROOM NET_SKB_PAD
79 #define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \
80 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
81
82 #define MLX5E_RX_MAX_HEAD (256)
83 #define MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE (8)
84 #define MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE (9)
85 #define MLX5E_SHAMPO_WQ_HEADER_PER_PAGE (PAGE_SIZE >> MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE)
86 #define MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE (64)
87 #define MLX5E_SHAMPO_WQ_RESRV_SIZE (64 * 1024)
88 #define MLX5E_SHAMPO_WQ_BASE_RESRV_SIZE (4096)
89
90 #define MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev) \
91 (6 + MLX5_CAP_GEN(mdev, cache_line_128byte)) /* HW restriction */
92 #define MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, req) \
93 max_t(u32, MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev), req)
94 #define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) \
95 MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, order_base_2(MLX5E_RX_MAX_HEAD))
96
97 /* Keep in sync with mlx5e_mpwrq_log_wqe_sz.
98 * These are theoretical maximums, which can be further restricted by
99 * capabilities. These values are used for static resource allocations and
100 * sanity checks.
101 * MLX5_SEND_WQE_MAX_SIZE is a bit bigger than the maximum cacheline-aligned WQE
102 * size actually used at runtime, but it's not a problem when calculating static
103 * array sizes.
104 */
105 #define MLX5_UMR_MAX_FLEX_SPACE \
106 (ALIGN_DOWN(MLX5_SEND_WQE_MAX_SIZE - sizeof(struct mlx5e_umr_wqe), \
107 MLX5_UMR_FLEX_ALIGNMENT))
108 #define MLX5_MPWRQ_MAX_PAGES_PER_WQE \
109 rounddown_pow_of_two(MLX5_UMR_MAX_FLEX_SPACE / sizeof(struct mlx5_mtt))
110
111 #define MLX5E_MAX_RQ_NUM_MTTS \
112 (ALIGN_DOWN(U16_MAX, 4) * 2) /* Fits into u16 and aligned by WQEBB. */
113 #define MLX5E_MAX_RQ_NUM_KSMS (U16_MAX - 1) /* So that num_ksms fits into u16. */
114 #define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024))
115
116 #define MLX5E_MIN_SKB_FRAG_SZ (MLX5_SKB_FRAG_SZ(MLX5_RX_HEADROOM))
117 #define MLX5E_LOG_MAX_RX_WQE_BULK \
118 (ilog2(PAGE_SIZE / roundup_pow_of_two(MLX5E_MIN_SKB_FRAG_SZ)))
119
120 #define MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE 0x6
121 #define MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE 0xa
122 #define MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE 0xd
123
124 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE (1 + MLX5E_LOG_MAX_RX_WQE_BULK)
125 #define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE 0xa
126 #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE 0xd
127
128 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW 0x2
129
130 #define MLX5E_DEFAULT_LRO_TIMEOUT 32
131 #define MLX5E_DEFAULT_SHAMPO_TIMEOUT 1024
132
133 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC 0x10
134 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE 0x3
135 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS 0x20
136 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC 0x10
137 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE 0x10
138 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS 0x20
139 #define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES 0x80
140 #define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES_MPW 0x2
141
142 #define MLX5E_MIN_NUM_CHANNELS 0x1
143 #define MLX5E_MAX_NUM_CHANNELS 256
144 #define MLX5E_TX_CQ_POLL_BUDGET 128
145 #define MLX5E_TX_XSK_POLL_BUDGET 64
146 #define MLX5E_SQ_RECOVER_MIN_INTERVAL 500 /* msecs */
147
148 #define mlx5e_state_dereference(priv, p) \
149 rcu_dereference_protected((p), lockdep_is_held(&(priv)->state_lock))
150
151 enum mlx5e_devcom_events {
152 MPV_DEVCOM_MASTER_UP,
153 MPV_DEVCOM_MASTER_DOWN,
154 MPV_DEVCOM_IPSEC_MASTER_UP,
155 MPV_DEVCOM_IPSEC_MASTER_DOWN,
156 };
157
mlx5e_get_num_lag_ports(struct mlx5_core_dev * mdev)158 static inline u8 mlx5e_get_num_lag_ports(struct mlx5_core_dev *mdev)
159 {
160 if (mlx5_lag_is_lacp_owner(mdev))
161 return 1;
162
163 return clamp_t(u8, MLX5_CAP_GEN(mdev, num_lag_ports), 1, MLX5_MAX_PORTS);
164 }
165
mlx5_min_rx_wqes(int wq_type,u32 wq_size)166 static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size)
167 {
168 switch (wq_type) {
169 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
170 return min_t(u16, MLX5E_PARAMS_DEFAULT_MIN_RX_WQES_MPW,
171 wq_size / 2);
172 default:
173 return min_t(u16, MLX5E_PARAMS_DEFAULT_MIN_RX_WQES,
174 wq_size / 2);
175 }
176 }
177
178 /* Use this function to get max num channels (rxqs/txqs) only to create netdev */
mlx5e_get_max_num_channels(struct mlx5_core_dev * mdev)179 static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
180 {
181 return is_kdump_kernel() ?
182 MLX5E_MIN_NUM_CHANNELS :
183 min3(mlx5_comp_vectors_max(mdev), (u32)MLX5E_MAX_NUM_CHANNELS,
184 (u32)(1 << MLX5_CAP_GEN(mdev, log_max_rqt_size)));
185 }
186
187 /* The maximum WQE size can be retrieved by max_wqe_sz_sq in
188 * bytes units. Driver hardens the limitation to 1KB (16
189 * WQEBBs), unless firmware capability is stricter.
190 */
mlx5e_get_max_sq_wqebbs(struct mlx5_core_dev * mdev)191 static inline u8 mlx5e_get_max_sq_wqebbs(struct mlx5_core_dev *mdev)
192 {
193 BUILD_BUG_ON(MLX5_SEND_WQE_MAX_WQEBBS > U8_MAX);
194
195 return (u8)min_t(u16, MLX5_SEND_WQE_MAX_WQEBBS,
196 MLX5_CAP_GEN(mdev, max_wqe_sz_sq) / MLX5_SEND_WQE_BB);
197 }
198
mlx5e_get_max_sq_aligned_wqebbs(struct mlx5_core_dev * mdev)199 static inline u8 mlx5e_get_max_sq_aligned_wqebbs(struct mlx5_core_dev *mdev)
200 {
201 /* The return value will be multiplied by MLX5_SEND_WQEBB_NUM_DS.
202 * Since max_sq_wqebbs may be up to MLX5_SEND_WQE_MAX_WQEBBS == 16,
203 * see mlx5e_get_max_sq_wqebbs(), the multiplication (16 * 4 == 64)
204 * overflows the 6-bit DS field of Ctrl Segment. Use a bound lower
205 * than MLX5_SEND_WQE_MAX_WQEBBS to let a full-session WQE be
206 * cache-aligned.
207 */
208 u8 wqebbs = mlx5e_get_max_sq_wqebbs(mdev);
209
210 wqebbs = min_t(u8, wqebbs, MLX5_SEND_WQE_MAX_WQEBBS - 1);
211 #if L1_CACHE_BYTES >= 128
212 wqebbs = ALIGN_DOWN(wqebbs, 2);
213 #endif
214 return wqebbs;
215 }
216
217 struct mlx5e_tx_wqe {
218 struct mlx5_wqe_ctrl_seg ctrl;
219 struct mlx5_wqe_eth_seg eth;
220 struct mlx5_wqe_data_seg data[];
221 };
222
223 struct mlx5e_rx_wqe_ll {
224 struct mlx5_wqe_srq_next_seg next;
225 struct mlx5_wqe_data_seg data[];
226 };
227
228 struct mlx5e_rx_wqe_cyc {
229 DECLARE_FLEX_ARRAY(struct mlx5_wqe_data_seg, data);
230 };
231
232 struct mlx5e_umr_wqe {
233 struct mlx5_wqe_ctrl_seg ctrl;
234 struct mlx5_wqe_umr_ctrl_seg uctrl;
235 struct mlx5_mkey_seg mkc;
236 union {
237 DECLARE_FLEX_ARRAY(struct mlx5_mtt, inline_mtts);
238 DECLARE_FLEX_ARRAY(struct mlx5_klm, inline_klms);
239 DECLARE_FLEX_ARRAY(struct mlx5_ksm, inline_ksms);
240 };
241 };
242
243 enum mlx5e_priv_flag {
244 MLX5E_PFLAG_RX_CQE_BASED_MODER,
245 MLX5E_PFLAG_TX_CQE_BASED_MODER,
246 MLX5E_PFLAG_RX_CQE_COMPRESS,
247 MLX5E_PFLAG_RX_STRIDING_RQ,
248 MLX5E_PFLAG_RX_NO_CSUM_COMPLETE,
249 MLX5E_PFLAG_XDP_TX_MPWQE,
250 MLX5E_PFLAG_SKB_TX_MPWQE,
251 MLX5E_PFLAG_TX_PORT_TS,
252 MLX5E_NUM_PFLAGS, /* Keep last */
253 };
254
255 #define MLX5E_SET_PFLAG(params, pflag, enable) \
256 do { \
257 if (enable) \
258 (params)->pflags |= BIT(pflag); \
259 else \
260 (params)->pflags &= ~(BIT(pflag)); \
261 } while (0)
262
263 #define MLX5E_GET_PFLAG(params, pflag) (!!((params)->pflags & (BIT(pflag))))
264
265 enum packet_merge {
266 MLX5E_PACKET_MERGE_NONE,
267 MLX5E_PACKET_MERGE_LRO,
268 MLX5E_PACKET_MERGE_SHAMPO,
269 };
270
271 struct mlx5e_packet_merge_param {
272 enum packet_merge type;
273 u32 timeout;
274 struct {
275 u8 match_criteria_type;
276 u8 alignment_granularity;
277 } shampo;
278 };
279
280 struct mlx5e_params {
281 u8 log_sq_size;
282 u8 rq_wq_type;
283 u8 log_rq_mtu_frames;
284 u16 num_channels;
285 struct {
286 u16 mode;
287 u8 num_tc;
288 struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
289 struct {
290 u64 max_rate[TC_MAX_QUEUE];
291 u32 hw_id[TC_MAX_QUEUE];
292 } channel;
293 } mqprio;
294 bool rx_cqe_compress_def;
295 struct dim_cq_moder rx_cq_moderation;
296 struct dim_cq_moder tx_cq_moderation;
297 struct mlx5e_packet_merge_param packet_merge;
298 u8 tx_min_inline_mode;
299 bool vlan_strip_disable;
300 bool scatter_fcs_en;
301 bool rx_dim_enabled;
302 bool tx_dim_enabled;
303 bool rx_moder_use_cqe_mode;
304 bool tx_moder_use_cqe_mode;
305 u32 pflags;
306 struct bpf_prog *xdp_prog;
307 struct mlx5e_xsk *xsk;
308 unsigned int sw_mtu;
309 int hard_mtu;
310 bool ptp_rx;
311 __be32 terminate_lkey_be;
312 };
313
mlx5e_get_dcb_num_tc(struct mlx5e_params * params)314 static inline u8 mlx5e_get_dcb_num_tc(struct mlx5e_params *params)
315 {
316 return params->mqprio.mode == TC_MQPRIO_MODE_DCB ?
317 params->mqprio.num_tc : 1;
318 }
319
320 /* Keep this enum consistent with the corresponding strings array
321 * declared in en/reporter_rx.c
322 */
323 enum {
324 MLX5E_RQ_STATE_ENABLED = 0,
325 MLX5E_RQ_STATE_RECOVERING,
326 MLX5E_RQ_STATE_DIM,
327 MLX5E_RQ_STATE_NO_CSUM_COMPLETE,
328 MLX5E_RQ_STATE_CSUM_FULL, /* cqe_csum_full hw bit is set */
329 MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, /* set when mini_cqe_resp_stride_index cap is used */
330 MLX5E_RQ_STATE_SHAMPO, /* set when SHAMPO cap is used */
331 MLX5E_RQ_STATE_MINI_CQE_ENHANCED, /* set when enhanced mini_cqe_cap is used */
332 MLX5E_RQ_STATE_XSK, /* set to indicate an xsk rq */
333 MLX5E_NUM_RQ_STATES, /* Must be kept last */
334 };
335
336 struct mlx5e_cq {
337 /* data path - accessed per cqe */
338 struct mlx5_cqwq wq;
339
340 /* data path - accessed per napi poll */
341 u16 event_ctr;
342 struct napi_struct *napi;
343 struct mlx5_core_cq mcq;
344 struct mlx5e_ch_stats *ch_stats;
345
346 /* control */
347 struct net_device *netdev;
348 struct mlx5_core_dev *mdev;
349 struct workqueue_struct *workqueue;
350 struct mlx5_wq_ctrl wq_ctrl;
351 } ____cacheline_aligned_in_smp;
352
353 struct mlx5e_cq_decomp {
354 /* cqe decompression */
355 struct mlx5_cqe64 title;
356 struct mlx5_mini_cqe8 mini_arr[MLX5_MINI_CQE_ARRAY_SIZE];
357 u8 mini_arr_idx;
358 u16 left;
359 u16 wqe_counter;
360 bool last_cqe_title;
361 } ____cacheline_aligned_in_smp;
362
363 enum mlx5e_dma_map_type {
364 MLX5E_DMA_MAP_SINGLE,
365 MLX5E_DMA_MAP_PAGE
366 };
367
368 struct mlx5e_sq_dma {
369 dma_addr_t addr;
370 u32 size;
371 enum mlx5e_dma_map_type type;
372 };
373
374 /* Keep this enum consistent with with the corresponding strings array
375 * declared in en/reporter_tx.c
376 */
377 enum {
378 MLX5E_SQ_STATE_ENABLED = 0,
379 MLX5E_SQ_STATE_MPWQE,
380 MLX5E_SQ_STATE_RECOVERING,
381 MLX5E_SQ_STATE_IPSEC,
382 MLX5E_SQ_STATE_DIM,
383 MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE,
384 MLX5E_SQ_STATE_PENDING_XSK_TX,
385 MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC,
386 MLX5E_NUM_SQ_STATES, /* Must be kept last */
387 };
388
389 struct mlx5e_tx_mpwqe {
390 /* Current MPWQE session */
391 struct mlx5e_tx_wqe *wqe;
392 u32 bytes_count;
393 u8 ds_count;
394 u8 pkt_count;
395 u8 inline_on;
396 };
397
398 struct mlx5e_skb_fifo {
399 struct sk_buff **fifo;
400 u16 *pc;
401 u16 *cc;
402 u16 mask;
403 };
404
405 struct mlx5e_ptpsq;
406
407 struct mlx5e_txqsq {
408 /* data path */
409
410 /* dirtied @completion */
411 u16 cc;
412 u16 skb_fifo_cc;
413 u32 dma_fifo_cc;
414 struct dim *dim; /* Adaptive Moderation */
415
416 /* dirtied @xmit */
417 u16 pc ____cacheline_aligned_in_smp;
418 u16 skb_fifo_pc;
419 u32 dma_fifo_pc;
420 struct mlx5e_tx_mpwqe mpwqe;
421
422 struct mlx5e_cq cq;
423
424 /* read only */
425 struct mlx5_wq_cyc wq;
426 u32 dma_fifo_mask;
427 struct mlx5e_sq_stats *stats;
428 struct {
429 struct mlx5e_sq_dma *dma_fifo;
430 struct mlx5e_skb_fifo skb_fifo;
431 struct mlx5e_tx_wqe_info *wqe_info;
432 } db;
433 void __iomem *uar_map;
434 struct netdev_queue *txq;
435 u32 sqn;
436 u16 stop_room;
437 u8 max_sq_mpw_wqebbs;
438 u8 min_inline_mode;
439 struct device *pdev;
440 __be32 mkey_be;
441 unsigned long state;
442 unsigned int hw_mtu;
443 struct mlx5_clock *clock;
444 struct net_device *netdev;
445 struct mlx5_core_dev *mdev;
446 struct mlx5e_channel *channel;
447 struct mlx5e_priv *priv;
448
449 /* control path */
450 struct mlx5_wq_ctrl wq_ctrl;
451 int ch_ix;
452 int txq_ix;
453 u32 rate_limit;
454 struct work_struct recover_work;
455 struct mlx5e_ptpsq *ptpsq;
456 cqe_ts_to_ns ptp_cyc2time;
457 } ____cacheline_aligned_in_smp;
458
459 struct mlx5e_xdp_info_fifo {
460 union mlx5e_xdp_info *xi;
461 u32 *cc;
462 u32 *pc;
463 u32 mask;
464 };
465
466 struct mlx5e_xdpsq;
467 struct mlx5e_xmit_data;
468 struct xsk_tx_metadata;
469 typedef int (*mlx5e_fp_xmit_xdp_frame_check)(struct mlx5e_xdpsq *);
470 typedef bool (*mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq *,
471 struct mlx5e_xmit_data *,
472 int,
473 struct xsk_tx_metadata *);
474
475 struct mlx5e_xdpsq {
476 /* data path */
477
478 /* dirtied @completion */
479 u32 xdpi_fifo_cc;
480 u16 cc;
481
482 /* dirtied @xmit */
483 u32 xdpi_fifo_pc ____cacheline_aligned_in_smp;
484 u16 pc;
485 struct mlx5_wqe_ctrl_seg *doorbell_cseg;
486 struct mlx5e_tx_mpwqe mpwqe;
487
488 struct mlx5e_cq cq;
489
490 /* read only */
491 struct xsk_buff_pool *xsk_pool;
492 struct mlx5_wq_cyc wq;
493 struct mlx5e_xdpsq_stats *stats;
494 mlx5e_fp_xmit_xdp_frame_check xmit_xdp_frame_check;
495 mlx5e_fp_xmit_xdp_frame xmit_xdp_frame;
496 struct {
497 struct mlx5e_xdp_wqe_info *wqe_info;
498 struct mlx5e_xdp_info_fifo xdpi_fifo;
499 } db;
500 void __iomem *uar_map;
501 u32 sqn;
502 struct device *pdev;
503 __be32 mkey_be;
504 u16 stop_room;
505 u8 max_sq_mpw_wqebbs;
506 u8 min_inline_mode;
507 unsigned long state;
508 unsigned int hw_mtu;
509
510 /* control path */
511 struct mlx5_wq_ctrl wq_ctrl;
512 struct mlx5e_channel *channel;
513 } ____cacheline_aligned_in_smp;
514
515 struct mlx5e_ktls_resync_resp;
516
517 struct mlx5e_icosq {
518 /* data path */
519 u16 cc;
520 u16 pc;
521
522 struct mlx5_wqe_ctrl_seg *doorbell_cseg;
523 struct mlx5e_cq cq;
524
525 /* write@xmit, read@completion */
526 struct {
527 struct mlx5e_icosq_wqe_info *wqe_info;
528 } db;
529
530 /* read only */
531 struct mlx5_wq_cyc wq;
532 void __iomem *uar_map;
533 u32 sqn;
534 u16 reserved_room;
535 unsigned long state;
536 struct mlx5e_ktls_resync_resp *ktls_resync;
537
538 /* control path */
539 struct mlx5_wq_ctrl wq_ctrl;
540 struct mlx5e_channel *channel;
541
542 struct work_struct recover_work;
543 } ____cacheline_aligned_in_smp;
544
545 struct mlx5e_frag_page {
546 struct page *page;
547 u16 frags;
548 };
549
550 enum mlx5e_wqe_frag_flag {
551 MLX5E_WQE_FRAG_LAST_IN_PAGE,
552 MLX5E_WQE_FRAG_SKIP_RELEASE,
553 };
554
555 struct mlx5e_wqe_frag_info {
556 union {
557 struct mlx5e_frag_page *frag_page;
558 struct xdp_buff **xskp;
559 };
560 u32 offset;
561 u8 flags;
562 };
563
564 union mlx5e_alloc_units {
565 DECLARE_FLEX_ARRAY(struct mlx5e_frag_page, frag_pages);
566 DECLARE_FLEX_ARRAY(struct page *, pages);
567 DECLARE_FLEX_ARRAY(struct xdp_buff *, xsk_buffs);
568 };
569
570 struct mlx5e_mpw_info {
571 u16 consumed_strides;
572 DECLARE_BITMAP(skip_release_bitmap, MLX5_MPWRQ_MAX_PAGES_PER_WQE);
573 struct mlx5e_frag_page linear_page;
574 union mlx5e_alloc_units alloc_units;
575 };
576
577 #define MLX5E_MAX_RX_FRAGS 4
578
579 struct mlx5e_rq;
580 typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*);
581 typedef struct sk_buff *
582 (*mlx5e_fp_skb_from_cqe_mpwrq)(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
583 struct mlx5_cqe64 *cqe, u16 cqe_bcnt,
584 u32 head_offset, u32 page_idx);
585 typedef struct sk_buff *
586 (*mlx5e_fp_skb_from_cqe)(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
587 struct mlx5_cqe64 *cqe, u32 cqe_bcnt);
588 typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq);
589 typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);
590 typedef void (*mlx5e_fp_shampo_dealloc_hd)(struct mlx5e_rq*, u16, u16, bool);
591
592 int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk);
593 void mlx5e_rq_set_trap_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params);
594
595 enum mlx5e_rq_flag {
596 MLX5E_RQ_FLAG_XDP_XMIT,
597 MLX5E_RQ_FLAG_XDP_REDIRECT,
598 };
599
600 struct mlx5e_rq_frag_info {
601 int frag_size;
602 int frag_stride;
603 };
604
605 struct mlx5e_rq_frags_info {
606 struct mlx5e_rq_frag_info arr[MLX5E_MAX_RX_FRAGS];
607 u8 num_frags;
608 u8 log_num_frags;
609 u16 wqe_bulk;
610 u16 refill_unit;
611 u8 wqe_index_mask;
612 };
613
614 struct mlx5e_dma_info {
615 dma_addr_t addr;
616 union {
617 struct mlx5e_frag_page *frag_page;
618 struct page *page;
619 };
620 };
621
622 struct mlx5e_shampo_hd {
623 u32 mkey;
624 struct mlx5e_dma_info *info;
625 struct mlx5e_frag_page *pages;
626 u16 curr_page_index;
627 u32 hd_per_wq;
628 u16 hd_per_wqe;
629 unsigned long *bitmap;
630 u16 pi;
631 u16 ci;
632 __be32 key;
633 u64 last_addr;
634 };
635
636 struct mlx5e_hw_gro_data {
637 struct sk_buff *skb;
638 struct flow_keys fk;
639 int second_ip_id;
640 };
641
642 enum mlx5e_mpwrq_umr_mode {
643 MLX5E_MPWRQ_UMR_MODE_ALIGNED,
644 MLX5E_MPWRQ_UMR_MODE_UNALIGNED,
645 MLX5E_MPWRQ_UMR_MODE_OVERSIZED,
646 MLX5E_MPWRQ_UMR_MODE_TRIPLE,
647 };
648
649 struct mlx5e_rq {
650 /* data path */
651 union {
652 struct {
653 struct mlx5_wq_cyc wq;
654 struct mlx5e_wqe_frag_info *frags;
655 union mlx5e_alloc_units *alloc_units;
656 struct mlx5e_rq_frags_info info;
657 mlx5e_fp_skb_from_cqe skb_from_cqe;
658 } wqe;
659 struct {
660 struct mlx5_wq_ll wq;
661 struct mlx5e_umr_wqe umr_wqe;
662 struct mlx5e_mpw_info *info;
663 mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq;
664 __be32 umr_mkey_be;
665 u16 num_strides;
666 u16 actual_wq_head;
667 u8 log_stride_sz;
668 u8 umr_in_progress;
669 u8 umr_last_bulk;
670 u8 umr_completed;
671 u8 min_wqe_bulk;
672 u8 page_shift;
673 u8 pages_per_wqe;
674 u8 umr_wqebbs;
675 u8 mtts_per_wqe;
676 u8 umr_mode;
677 struct mlx5e_shampo_hd *shampo;
678 } mpwqe;
679 };
680 struct {
681 u16 headroom;
682 u32 frame0_sz;
683 u8 map_dir; /* dma map direction */
684 } buff;
685
686 struct device *pdev;
687 struct net_device *netdev;
688 struct mlx5e_rq_stats *stats;
689 struct mlx5e_cq cq;
690 struct mlx5e_cq_decomp cqd;
691 struct hwtstamp_config *tstamp;
692 struct mlx5_clock *clock;
693 struct mlx5e_icosq *icosq;
694 struct mlx5e_priv *priv;
695
696 struct mlx5e_hw_gro_data *hw_gro_data;
697
698 mlx5e_fp_handle_rx_cqe handle_rx_cqe;
699 mlx5e_fp_post_rx_wqes post_wqes;
700 mlx5e_fp_dealloc_wqe dealloc_wqe;
701
702 unsigned long state;
703 int ix;
704 unsigned int hw_mtu;
705
706 struct dim *dim; /* Dynamic Interrupt Moderation */
707
708 /* XDP */
709 struct bpf_prog __rcu *xdp_prog;
710 struct mlx5e_xdpsq *xdpsq;
711 DECLARE_BITMAP(flags, 8);
712 struct page_pool *page_pool;
713
714 /* AF_XDP zero-copy */
715 struct xsk_buff_pool *xsk_pool;
716
717 struct work_struct recover_work;
718
719 /* control */
720 struct mlx5_wq_ctrl wq_ctrl;
721 __be32 mkey_be;
722 u8 wq_type;
723 u32 rqn;
724 struct mlx5_core_dev *mdev;
725 struct mlx5e_channel *channel;
726 struct mlx5e_dma_info wqe_overflow;
727
728 /* XDP read-mostly */
729 struct xdp_rxq_info xdp_rxq;
730 cqe_ts_to_ns ptp_cyc2time;
731 } ____cacheline_aligned_in_smp;
732
733 enum mlx5e_channel_state {
734 MLX5E_CHANNEL_STATE_XSK,
735 MLX5E_CHANNEL_NUM_STATES
736 };
737
738 struct mlx5e_channel {
739 /* data path */
740 struct mlx5e_rq rq;
741 struct mlx5e_xdpsq rq_xdpsq;
742 struct mlx5e_txqsq sq[MLX5_MAX_NUM_TC];
743 struct mlx5e_icosq icosq; /* internal control operations */
744 struct mlx5e_txqsq __rcu * __rcu *qos_sqs;
745 bool xdp;
746 struct napi_struct napi;
747 struct device *pdev;
748 struct net_device *netdev;
749 __be32 mkey_be;
750 u16 qos_sqs_size;
751 u8 num_tc;
752 u8 lag_port;
753
754 /* XDP_REDIRECT */
755 struct mlx5e_xdpsq xdpsq;
756
757 /* AF_XDP zero-copy */
758 struct mlx5e_rq xskrq;
759 struct mlx5e_xdpsq xsksq;
760
761 /* Async ICOSQ */
762 struct mlx5e_icosq async_icosq;
763 /* async_icosq can be accessed from any CPU - the spinlock protects it. */
764 spinlock_t async_icosq_lock;
765
766 /* data path - accessed per napi poll */
767 const struct cpumask *aff_mask;
768 struct mlx5e_ch_stats *stats;
769
770 /* control */
771 struct mlx5e_priv *priv;
772 struct mlx5_core_dev *mdev;
773 struct hwtstamp_config *tstamp;
774 DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES);
775 int ix;
776 int vec_ix;
777 int sd_ix;
778 int cpu;
779 /* Sync between icosq recovery and XSK enable/disable. */
780 struct mutex icosq_recovery_lock;
781
782 /* coalescing configuration */
783 struct dim_cq_moder rx_cq_moder;
784 struct dim_cq_moder tx_cq_moder;
785 };
786
787 struct mlx5e_ptp;
788
789 struct mlx5e_channels {
790 struct mlx5e_channel **c;
791 struct mlx5e_ptp *ptp;
792 unsigned int num;
793 struct mlx5e_params params;
794 };
795
796 struct mlx5e_channel_stats {
797 struct mlx5e_ch_stats ch;
798 struct mlx5e_sq_stats sq[MLX5_MAX_NUM_TC];
799 struct mlx5e_rq_stats rq;
800 struct mlx5e_rq_stats xskrq;
801 struct mlx5e_xdpsq_stats rq_xdpsq;
802 struct mlx5e_xdpsq_stats xdpsq;
803 struct mlx5e_xdpsq_stats xsksq;
804 } ____cacheline_aligned_in_smp;
805
806 struct mlx5e_ptp_stats {
807 struct mlx5e_ch_stats ch;
808 struct mlx5e_sq_stats sq[MLX5_MAX_NUM_TC];
809 struct mlx5e_ptp_cq_stats cq[MLX5_MAX_NUM_TC];
810 struct mlx5e_rq_stats rq;
811 } ____cacheline_aligned_in_smp;
812
813 enum {
814 MLX5E_STATE_OPENED,
815 MLX5E_STATE_DESTROYING,
816 MLX5E_STATE_XDP_TX_ENABLED,
817 MLX5E_STATE_XDP_ACTIVE,
818 MLX5E_STATE_CHANNELS_ACTIVE,
819 };
820
821 struct mlx5e_modify_sq_param {
822 int curr_state;
823 int next_state;
824 int rl_update;
825 int rl_index;
826 bool qos_update;
827 u16 qos_queue_group_id;
828 };
829
830 #if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE)
831 struct mlx5e_hv_vhca_stats_agent {
832 struct mlx5_hv_vhca_agent *agent;
833 struct delayed_work work;
834 u16 delay;
835 void *buf;
836 };
837 #endif
838
839 struct mlx5e_xsk {
840 /* XSK buffer pools are stored separately from channels,
841 * because we don't want to lose them when channels are
842 * recreated. The kernel also stores buffer pool, but it doesn't
843 * distinguish between zero-copy and non-zero-copy UMEMs, so
844 * rely on our mechanism.
845 */
846 struct xsk_buff_pool **pools;
847 u16 refcnt;
848 bool ever_used;
849 };
850
851 /* Temporary storage for variables that are allocated when struct mlx5e_priv is
852 * initialized, and used where we can't allocate them because that functions
853 * must not fail. Use with care and make sure the same variable is not used
854 * simultaneously by multiple users.
855 */
856 struct mlx5e_scratchpad {
857 cpumask_var_t cpumask;
858 };
859
860 struct mlx5e_trap;
861 struct mlx5e_htb;
862
863 struct mlx5e_priv {
864 /* priv data path fields - start */
865 struct mlx5e_selq selq;
866 struct mlx5e_txqsq **txq2sq;
867 struct mlx5e_sq_stats **txq2sq_stats;
868
869 #ifdef CONFIG_MLX5_CORE_EN_DCB
870 struct mlx5e_dcbx_dp dcbx_dp;
871 #endif
872 /* priv data path fields - end */
873
874 unsigned long state;
875 struct mutex state_lock; /* Protects Interface state */
876 struct mlx5e_rq drop_rq;
877
878 struct mlx5e_channels channels;
879 struct mlx5e_rx_res *rx_res;
880 u32 *tx_rates;
881
882 struct mlx5e_flow_steering *fs;
883
884 struct workqueue_struct *wq;
885 struct work_struct update_carrier_work;
886 struct work_struct set_rx_mode_work;
887 struct work_struct tx_timeout_work;
888 struct work_struct update_stats_work;
889 struct work_struct monitor_counters_work;
890 struct mlx5_nb monitor_counters_nb;
891
892 struct mlx5_core_dev *mdev;
893 struct net_device *netdev;
894 struct mlx5e_trap *en_trap;
895 struct mlx5e_stats stats;
896 struct mlx5e_channel_stats **channel_stats;
897 struct mlx5e_channel_stats trap_stats;
898 struct mlx5e_ptp_stats ptp_stats;
899 struct mlx5e_sq_stats **htb_qos_sq_stats;
900 u16 htb_max_qos_sqs;
901 u16 stats_nch;
902 u16 max_nch;
903 u8 max_opened_tc;
904 bool tx_ptp_opened;
905 bool rx_ptp_opened;
906 struct hwtstamp_config tstamp;
907 u16 q_counter[MLX5_SD_MAX_GROUP_SZ];
908 u16 drop_rq_q_counter;
909 struct notifier_block events_nb;
910 struct notifier_block blocking_events_nb;
911
912 struct udp_tunnel_nic_info nic_info;
913 #ifdef CONFIG_MLX5_CORE_EN_DCB
914 struct mlx5e_dcbx dcbx;
915 #endif
916
917 const struct mlx5e_profile *profile;
918 void *ppriv;
919 #ifdef CONFIG_MLX5_MACSEC
920 struct mlx5e_macsec *macsec;
921 #endif
922 #ifdef CONFIG_MLX5_EN_IPSEC
923 struct mlx5e_ipsec *ipsec;
924 #endif
925 #ifdef CONFIG_MLX5_EN_TLS
926 struct mlx5e_tls *tls;
927 #endif
928 struct devlink_health_reporter *tx_reporter;
929 struct devlink_health_reporter *rx_reporter;
930 struct mlx5e_xsk xsk;
931 #if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE)
932 struct mlx5e_hv_vhca_stats_agent stats_agent;
933 #endif
934 struct mlx5e_scratchpad scratchpad;
935 struct mlx5e_htb *htb;
936 struct mlx5e_mqprio_rl *mqprio_rl;
937 struct dentry *dfs_root;
938 struct mlx5_devcom_comp_dev *devcom;
939 };
940
941 struct mlx5e_dev {
942 struct mlx5e_priv *priv;
943 struct devlink_port dl_port;
944 };
945
946 struct mlx5e_rx_handlers {
947 mlx5e_fp_handle_rx_cqe handle_rx_cqe;
948 mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe;
949 mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe_shampo;
950 };
951
952 extern const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic;
953
954 enum mlx5e_profile_feature {
955 MLX5E_PROFILE_FEATURE_PTP_RX,
956 MLX5E_PROFILE_FEATURE_PTP_TX,
957 MLX5E_PROFILE_FEATURE_QOS_HTB,
958 MLX5E_PROFILE_FEATURE_FS_VLAN,
959 MLX5E_PROFILE_FEATURE_FS_TC,
960 };
961
962 struct mlx5e_profile {
963 int (*init)(struct mlx5_core_dev *mdev,
964 struct net_device *netdev);
965 void (*cleanup)(struct mlx5e_priv *priv);
966 int (*init_rx)(struct mlx5e_priv *priv);
967 void (*cleanup_rx)(struct mlx5e_priv *priv);
968 int (*init_tx)(struct mlx5e_priv *priv);
969 void (*cleanup_tx)(struct mlx5e_priv *priv);
970 void (*enable)(struct mlx5e_priv *priv);
971 void (*disable)(struct mlx5e_priv *priv);
972 int (*update_rx)(struct mlx5e_priv *priv);
973 void (*update_stats)(struct mlx5e_priv *priv);
974 void (*update_carrier)(struct mlx5e_priv *priv);
975 int (*max_nch_limit)(struct mlx5_core_dev *mdev);
976 u32 (*get_tisn)(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv,
977 u8 lag_port, u8 tc);
978 unsigned int (*stats_grps_num)(struct mlx5e_priv *priv);
979 mlx5e_stats_grp_t *stats_grps;
980 const struct mlx5e_rx_handlers *rx_handlers;
981 int max_tc;
982 u32 features;
983 };
984
985 u32 mlx5e_profile_get_tisn(struct mlx5_core_dev *mdev,
986 struct mlx5e_priv *priv,
987 const struct mlx5e_profile *profile,
988 u8 lag_port, u8 tc);
989
990 #define mlx5e_profile_feature_cap(profile, feature) \
991 ((profile)->features & BIT(MLX5E_PROFILE_FEATURE_##feature))
992
993 void mlx5e_build_ptys2ethtool_map(void);
994
995 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift,
996 enum mlx5e_mpwrq_umr_mode umr_mode);
997
998 void mlx5e_shampo_fill_umr(struct mlx5e_rq *rq, int len);
999 void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq);
1000 void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats);
1001 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s);
1002
1003 int mlx5e_self_test_num(struct mlx5e_priv *priv);
1004 int mlx5e_self_test_fill_strings(struct mlx5e_priv *priv, u8 *data);
1005 void mlx5e_self_test(struct net_device *ndev, struct ethtool_test *etest,
1006 u64 *buf);
1007 void mlx5e_set_rx_mode_work(struct work_struct *work);
1008
1009 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr);
1010 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr);
1011 int mlx5e_modify_rx_cqe_compression_locked(struct mlx5e_priv *priv, bool val, bool rx_filter);
1012
1013 int mlx5e_vlan_rx_add_vid(struct net_device *dev, __always_unused __be16 proto,
1014 u16 vid);
1015 int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __always_unused __be16 proto,
1016 u16 vid);
1017 void mlx5e_timestamp_init(struct mlx5e_priv *priv);
1018
1019 struct mlx5e_xsk_param;
1020
1021 struct mlx5e_rq_param;
1022 int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
1023 struct mlx5e_xsk_param *xsk, int node, u16 q_counter,
1024 struct mlx5e_rq *rq);
1025 #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
1026 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time);
1027 void mlx5e_close_rq(struct mlx5e_rq *rq);
1028 int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter);
1029 void mlx5e_destroy_rq(struct mlx5e_rq *rq);
1030
1031 bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,
1032 bool dim_enabled);
1033 bool mlx5e_reset_rx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode,
1034 bool dim_enabled, bool keep_dim_state);
1035
1036 struct mlx5e_sq_param;
1037 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
1038 struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
1039 struct mlx5e_xdpsq *sq, bool is_redirect);
1040 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq);
1041
1042 struct mlx5e_create_cq_param {
1043 struct net_device *netdev;
1044 struct workqueue_struct *wq;
1045 struct napi_struct *napi;
1046 struct mlx5e_ch_stats *ch_stats;
1047 int node;
1048 int ix;
1049 };
1050
1051 struct mlx5e_cq_param;
1052 int mlx5e_open_cq(struct mlx5_core_dev *mdev, struct dim_cq_moder moder,
1053 struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp,
1054 struct mlx5e_cq *cq);
1055 void mlx5e_close_cq(struct mlx5e_cq *cq);
1056 int mlx5e_modify_cq_period_mode(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
1057 u8 cq_period_mode);
1058 int mlx5e_modify_cq_moderation(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
1059 u16 cq_period, u16 cq_max_count, u8 cq_period_mode);
1060
1061 int mlx5e_open_locked(struct net_device *netdev);
1062 int mlx5e_close_locked(struct net_device *netdev);
1063
1064 void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c);
1065 void mlx5e_trigger_napi_sched(struct napi_struct *napi);
1066
1067 int mlx5e_open_channels(struct mlx5e_priv *priv,
1068 struct mlx5e_channels *chs);
1069 void mlx5e_close_channels(struct mlx5e_channels *chs);
1070
1071 /* Function pointer to be used to modify HW or kernel settings while
1072 * switching channels
1073 */
1074 typedef int (*mlx5e_fp_preactivate)(struct mlx5e_priv *priv, void *context);
1075 #define MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(fn) \
1076 int fn##_ctx(struct mlx5e_priv *priv, void *context) \
1077 { \
1078 return fn(priv); \
1079 }
1080 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv);
1081 int mlx5e_safe_switch_params(struct mlx5e_priv *priv,
1082 struct mlx5e_params *new_params,
1083 mlx5e_fp_preactivate preactivate,
1084 void *context, bool reset);
1085 int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv);
1086 int mlx5e_num_channels_changed_ctx(struct mlx5e_priv *priv, void *context);
1087 int mlx5e_update_tc_and_tx_queues_ctx(struct mlx5e_priv *priv, void *context);
1088 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv);
1089 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv);
1090 int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx);
1091
1092 int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state);
1093 void mlx5e_activate_rq(struct mlx5e_rq *rq);
1094 void mlx5e_deactivate_rq(struct mlx5e_rq *rq);
1095 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq);
1096 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq);
1097
1098 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
1099 struct mlx5e_modify_sq_param *p);
1100 int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix,
1101 struct mlx5e_params *params, struct mlx5e_sq_param *param,
1102 struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id,
1103 struct mlx5e_sq_stats *sq_stats);
1104 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq);
1105 void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq);
1106 void mlx5e_free_txqsq(struct mlx5e_txqsq *sq);
1107 void mlx5e_tx_disable_queue(struct netdev_queue *txq);
1108 int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa);
1109 void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq);
1110 struct mlx5e_create_sq_param;
1111 int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
1112 struct mlx5e_sq_param *param,
1113 struct mlx5e_create_sq_param *csp,
1114 u16 qos_queue_group_id,
1115 u32 *sqn);
1116 void mlx5e_tx_err_cqe_work(struct work_struct *recover_work);
1117 void mlx5e_close_txqsq(struct mlx5e_txqsq *sq);
1118
1119 bool mlx5e_reset_tx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,
1120 bool dim_enabled);
1121 bool mlx5e_reset_tx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode,
1122 bool dim_enabled, bool keep_dim_state);
1123
mlx5_tx_swp_supported(struct mlx5_core_dev * mdev)1124 static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev)
1125 {
1126 return MLX5_CAP_ETH(mdev, swp) &&
1127 MLX5_CAP_ETH(mdev, swp_csum) && MLX5_CAP_ETH(mdev, swp_lso);
1128 }
1129
1130 extern const struct ethtool_ops mlx5e_ethtool_ops;
1131
1132 int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey);
1133 int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises);
1134 void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev);
1135 int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb,
1136 bool enable_mc_lb);
1137 void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc);
1138
1139 /* common netdev helpers */
1140 void mlx5e_create_q_counters(struct mlx5e_priv *priv);
1141 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv);
1142 int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
1143 struct mlx5e_rq *drop_rq);
1144 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq);
1145
1146 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn);
1147 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn);
1148
1149 void mlx5e_update_carrier(struct mlx5e_priv *priv);
1150 int mlx5e_close(struct net_device *netdev);
1151 int mlx5e_open(struct net_device *netdev);
1152
1153 void mlx5e_queue_update_stats(struct mlx5e_priv *priv);
1154
1155 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv);
1156 int mlx5e_set_dev_port_mtu_ctx(struct mlx5e_priv *priv, void *context);
1157 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
1158 mlx5e_fp_preactivate preactivate);
1159 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv);
1160
1161 /* ethtool helpers */
1162 void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv,
1163 struct ethtool_drvinfo *drvinfo);
1164 void mlx5e_ethtool_get_strings(struct mlx5e_priv *priv,
1165 u32 stringset, u8 *data);
1166 int mlx5e_ethtool_get_sset_count(struct mlx5e_priv *priv, int sset);
1167 void mlx5e_ethtool_get_ethtool_stats(struct mlx5e_priv *priv,
1168 struct ethtool_stats *stats, u64 *data);
1169 void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv,
1170 struct ethtool_ringparam *param,
1171 struct kernel_ethtool_ringparam *kernel_param);
1172 int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv,
1173 struct ethtool_ringparam *param,
1174 struct netlink_ext_ack *extack);
1175 void mlx5e_ethtool_get_channels(struct mlx5e_priv *priv,
1176 struct ethtool_channels *ch);
1177 int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv,
1178 struct ethtool_channels *ch);
1179 int mlx5e_ethtool_get_coalesce(struct mlx5e_priv *priv,
1180 struct ethtool_coalesce *coal,
1181 struct kernel_ethtool_coalesce *kernel_coal,
1182 struct netlink_ext_ack *extack);
1183 int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv,
1184 struct ethtool_coalesce *coal,
1185 struct kernel_ethtool_coalesce *kernel_coal,
1186 struct netlink_ext_ack *extack);
1187 int mlx5e_get_per_queue_coalesce(struct net_device *dev, u32 queue,
1188 struct ethtool_coalesce *coal);
1189 int mlx5e_set_per_queue_coalesce(struct net_device *dev, u32 queue,
1190 struct ethtool_coalesce *coal);
1191 u32 mlx5e_ethtool_get_rxfh_key_size(struct mlx5e_priv *priv);
1192 u32 mlx5e_ethtool_get_rxfh_indir_size(struct mlx5e_priv *priv);
1193 int mlx5e_ethtool_get_ts_info(struct mlx5e_priv *priv,
1194 struct kernel_ethtool_ts_info *info);
1195 int mlx5e_ethtool_flash_device(struct mlx5e_priv *priv,
1196 struct ethtool_flash *flash);
1197
1198 /* mlx5e generic netdev management API */
1199 static inline bool
mlx5e_tx_mpwqe_supported(struct mlx5_core_dev * mdev)1200 mlx5e_tx_mpwqe_supported(struct mlx5_core_dev *mdev)
1201 {
1202 return !is_kdump_kernel() &&
1203 MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe);
1204 }
1205
1206 int mlx5e_get_pf_num_tirs(struct mlx5_core_dev *mdev);
1207 int mlx5e_priv_init(struct mlx5e_priv *priv,
1208 const struct mlx5e_profile *profile,
1209 struct net_device *netdev,
1210 struct mlx5_core_dev *mdev);
1211 void mlx5e_priv_cleanup(struct mlx5e_priv *priv);
1212 struct net_device *
1213 mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile);
1214 int mlx5e_attach_netdev(struct mlx5e_priv *priv);
1215 void mlx5e_detach_netdev(struct mlx5e_priv *priv);
1216 void mlx5e_destroy_netdev(struct mlx5e_priv *priv);
1217 int mlx5e_netdev_change_profile(struct mlx5e_priv *priv,
1218 const struct mlx5e_profile *new_profile, void *new_ppriv);
1219 void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv *priv);
1220 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv);
1221 void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu);
1222
1223 void mlx5e_set_xdp_feature(struct net_device *netdev);
1224 netdev_features_t mlx5e_features_check(struct sk_buff *skb,
1225 struct net_device *netdev,
1226 netdev_features_t features);
1227 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features);
1228 #ifdef CONFIG_MLX5_ESWITCH
1229 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac);
1230 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, int max_tx_rate);
1231 int mlx5e_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi);
1232 int mlx5e_get_vf_stats(struct net_device *dev, int vf, struct ifla_vf_stats *vf_stats);
1233 #endif
1234 int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey);
1235 #endif /* __MLX5_EN_H__ */
1236