1 /*
2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34 #ifndef _MLX4_EN_H_
35 #define _MLX4_EN_H_
36
37 #include <linux/bitops.h>
38 #include <linux/compiler.h>
39 #include <linux/list.h>
40 #include <linux/mutex.h>
41 #include <linux/netdevice.h>
42 #include <linux/if_vlan.h>
43 #include <linux/net_tstamp.h>
44 #ifdef CONFIG_MLX4_EN_DCB
45 #include <linux/dcbnl.h>
46 #endif
47 #include <linux/cpu_rmap.h>
48 #include <linux/ptp_clock_kernel.h>
49
50 #include <linux/mlx4/device.h>
51 #include <linux/mlx4/qp.h>
52 #include <linux/mlx4/cq.h>
53 #include <linux/mlx4/srq.h>
54 #include <linux/mlx4/doorbell.h>
55 #include <linux/mlx4/cmd.h>
56
57 #include "en_port.h"
58
59 #define DRV_NAME "mlx4_en"
60 #define DRV_VERSION "2.2-1"
61 #define DRV_RELDATE "Feb 2014"
62
63 #define MLX4_EN_MSG_LEVEL (NETIF_MSG_LINK | NETIF_MSG_IFDOWN)
64
65 /*
66 * Device constants
67 */
68
69
70 #define MLX4_EN_PAGE_SHIFT 12
71 #define MLX4_EN_PAGE_SIZE (1 << MLX4_EN_PAGE_SHIFT)
72 #define DEF_RX_RINGS 16
73 #define MAX_RX_RINGS 128
74 #define MIN_RX_RINGS 4
75 #define TXBB_SIZE 64
76 #define HEADROOM (2048 / TXBB_SIZE + 1)
77 #define STAMP_STRIDE 64
78 #define STAMP_DWORDS (STAMP_STRIDE / 4)
79 #define STAMP_SHIFT 31
80 #define STAMP_VAL 0x7fffffff
81 #define STATS_DELAY (HZ / 4)
82 #define SERVICE_TASK_DELAY (HZ / 4)
83 #define MAX_NUM_OF_FS_RULES 256
84
85 #define MLX4_EN_FILTER_HASH_SHIFT 4
86 #define MLX4_EN_FILTER_EXPIRY_QUOTA 60
87
88 /* Typical TSO descriptor with 16 gather entries is 352 bytes... */
89 #define MAX_DESC_SIZE 512
90 #define MAX_DESC_TXBBS (MAX_DESC_SIZE / TXBB_SIZE)
91
92 /*
93 * OS related constants and tunables
94 */
95
96 #define MLX4_EN_PRIV_FLAGS_BLUEFLAME 1
97
98 #define MLX4_EN_WATCHDOG_TIMEOUT (15 * HZ)
99
100 /* Use the maximum between 16384 and a single page */
101 #define MLX4_EN_ALLOC_SIZE PAGE_ALIGN(16384)
102
103 #define MLX4_EN_ALLOC_PREFER_ORDER PAGE_ALLOC_COSTLY_ORDER
104
105 /* Receive fragment sizes; we use at most 3 fragments (for 9600 byte MTU
106 * and 4K allocations) */
107 enum {
108 FRAG_SZ0 = 1536 - NET_IP_ALIGN,
109 FRAG_SZ1 = 4096,
110 FRAG_SZ2 = 4096,
111 FRAG_SZ3 = MLX4_EN_ALLOC_SIZE
112 };
113 #define MLX4_EN_MAX_RX_FRAGS 4
114
115 /* Maximum ring sizes */
116 #define MLX4_EN_MAX_TX_SIZE 8192
117 #define MLX4_EN_MAX_RX_SIZE 8192
118
119 /* Minimum ring size for our page-allocation scheme to work */
120 #define MLX4_EN_MIN_RX_SIZE (MLX4_EN_ALLOC_SIZE / SMP_CACHE_BYTES)
121 #define MLX4_EN_MIN_TX_SIZE (4096 / TXBB_SIZE)
122
123 #define MLX4_EN_SMALL_PKT_SIZE 64
124 #define MLX4_EN_MIN_TX_RING_P_UP 1
125 #define MLX4_EN_MAX_TX_RING_P_UP 32
126 #define MLX4_EN_NUM_UP 8
127 #define MLX4_EN_DEF_TX_RING_SIZE 512
128 #define MLX4_EN_DEF_RX_RING_SIZE 1024
129 #define MAX_TX_RINGS (MLX4_EN_MAX_TX_RING_P_UP * \
130 MLX4_EN_NUM_UP)
131
132 #define MLX4_EN_DEFAULT_TX_WORK 256
133
134 /* Target number of packets to coalesce with interrupt moderation */
135 #define MLX4_EN_RX_COAL_TARGET 44
136 #define MLX4_EN_RX_COAL_TIME 0x10
137
138 #define MLX4_EN_TX_COAL_PKTS 16
139 #define MLX4_EN_TX_COAL_TIME 0x10
140
141 #define MLX4_EN_RX_RATE_LOW 400000
142 #define MLX4_EN_RX_COAL_TIME_LOW 0
143 #define MLX4_EN_RX_RATE_HIGH 450000
144 #define MLX4_EN_RX_COAL_TIME_HIGH 128
145 #define MLX4_EN_RX_SIZE_THRESH 1024
146 #define MLX4_EN_RX_RATE_THRESH (1000000 / MLX4_EN_RX_COAL_TIME_HIGH)
147 #define MLX4_EN_SAMPLE_INTERVAL 0
148 #define MLX4_EN_AVG_PKT_SMALL 256
149
150 #define MLX4_EN_AUTO_CONF 0xffff
151
152 #define MLX4_EN_DEF_RX_PAUSE 1
153 #define MLX4_EN_DEF_TX_PAUSE 1
154
155 /* Interval between successive polls in the Tx routine when polling is used
156 instead of interrupts (in per-core Tx rings) - should be power of 2 */
157 #define MLX4_EN_TX_POLL_MODER 16
158 #define MLX4_EN_TX_POLL_TIMEOUT (HZ / 4)
159
160 #define SMALL_PACKET_SIZE (256 - NET_IP_ALIGN)
161 #define HEADER_COPY_SIZE (128 - NET_IP_ALIGN)
162 #define MLX4_LOOPBACK_TEST_PAYLOAD (HEADER_COPY_SIZE - ETH_HLEN)
163
164 #define MLX4_EN_MIN_MTU 46
165 #define ETH_BCAST 0xffffffffffffULL
166
167 #define MLX4_EN_LOOPBACK_RETRIES 5
168 #define MLX4_EN_LOOPBACK_TIMEOUT 100
169
170 #ifdef MLX4_EN_PERF_STAT
171 /* Number of samples to 'average' */
172 #define AVG_SIZE 128
173 #define AVG_FACTOR 1024
174 #define NUM_PERF_STATS NUM_PERF_COUNTERS
175
176 #define INC_PERF_COUNTER(cnt) (++(cnt))
177 #define ADD_PERF_COUNTER(cnt, add) ((cnt) += (add))
178 #define AVG_PERF_COUNTER(cnt, sample) \
179 ((cnt) = ((cnt) * (AVG_SIZE - 1) + (sample) * AVG_FACTOR) / AVG_SIZE)
180 #define GET_PERF_COUNTER(cnt) (cnt)
181 #define GET_AVG_PERF_COUNTER(cnt) ((cnt) / AVG_FACTOR)
182
183 #else
184
185 #define NUM_PERF_STATS 0
186 #define INC_PERF_COUNTER(cnt) do {} while (0)
187 #define ADD_PERF_COUNTER(cnt, add) do {} while (0)
188 #define AVG_PERF_COUNTER(cnt, sample) do {} while (0)
189 #define GET_PERF_COUNTER(cnt) (0)
190 #define GET_AVG_PERF_COUNTER(cnt) (0)
191 #endif /* MLX4_EN_PERF_STAT */
192
193 /* Constants for TX flow */
194 enum {
195 MAX_INLINE = 104, /* 128 - 16 - 4 - 4 */
196 MAX_BF = 256,
197 MIN_PKT_LEN = 17,
198 };
199
200 /*
201 * Configurables
202 */
203
204 enum cq_type {
205 RX = 0,
206 TX = 1,
207 };
208
209
210 /*
211 * Useful macros
212 */
213 #define ROUNDUP_LOG2(x) ilog2(roundup_pow_of_two(x))
214 #define XNOR(x, y) (!(x) == !(y))
215
216
217 struct mlx4_en_tx_info {
218 struct sk_buff *skb;
219 dma_addr_t map0_dma;
220 u32 map0_byte_count;
221 u32 nr_txbb;
222 u32 nr_bytes;
223 u8 linear;
224 u8 data_offset;
225 u8 inl;
226 u8 ts_requested;
227 u8 nr_maps;
228 } ____cacheline_aligned_in_smp;
229
230
231 #define MLX4_EN_BIT_DESC_OWN 0x80000000
232 #define CTRL_SIZE sizeof(struct mlx4_wqe_ctrl_seg)
233 #define MLX4_EN_MEMTYPE_PAD 0x100
234 #define DS_SIZE sizeof(struct mlx4_wqe_data_seg)
235
236
237 struct mlx4_en_tx_desc {
238 struct mlx4_wqe_ctrl_seg ctrl;
239 union {
240 struct mlx4_wqe_data_seg data; /* at least one data segment */
241 struct mlx4_wqe_lso_seg lso;
242 struct mlx4_wqe_inline_seg inl;
243 };
244 };
245
246 #define MLX4_EN_USE_SRQ 0x01000000
247
248 #define MLX4_EN_CX3_LOW_ID 0x1000
249 #define MLX4_EN_CX3_HIGH_ID 0x1005
250
251 struct mlx4_en_rx_alloc {
252 struct page *page;
253 dma_addr_t dma;
254 u32 page_offset;
255 u32 page_size;
256 };
257
258 struct mlx4_en_tx_ring {
259 /* cache line used and dirtied in tx completion
260 * (mlx4_en_free_tx_buf())
261 */
262 u32 last_nr_txbb;
263 u32 cons;
264 unsigned long wake_queue;
265
266 /* cache line used and dirtied in mlx4_en_xmit() */
267 u32 prod ____cacheline_aligned_in_smp;
268 unsigned long bytes;
269 unsigned long packets;
270 unsigned long tx_csum;
271 unsigned long tso_packets;
272 unsigned long xmit_more;
273 struct mlx4_bf bf;
274 unsigned long queue_stopped;
275
276 /* Following part should be mostly read */
277 cpumask_t affinity_mask;
278 struct mlx4_qp qp;
279 struct mlx4_hwq_resources wqres;
280 u32 size; /* number of TXBBs */
281 u32 size_mask;
282 u16 stride;
283 u32 full_size;
284 u16 cqn; /* index of port CQ associated with this ring */
285 u32 buf_size;
286 __be32 doorbell_qpn;
287 __be32 mr_key;
288 void *buf;
289 struct mlx4_en_tx_info *tx_info;
290 u8 *bounce_buf;
291 struct mlx4_qp_context context;
292 int qpn;
293 enum mlx4_qp_state qp_state;
294 u8 queue_index;
295 bool bf_enabled;
296 bool bf_alloced;
297 struct netdev_queue *tx_queue;
298 int hwtstamp_tx_type;
299 } ____cacheline_aligned_in_smp;
300
301 struct mlx4_en_rx_desc {
302 /* actual number of entries depends on rx ring stride */
303 struct mlx4_wqe_data_seg data[0];
304 };
305
306 struct mlx4_en_rx_ring {
307 struct mlx4_hwq_resources wqres;
308 struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS];
309 u32 size ; /* number of Rx descs*/
310 u32 actual_size;
311 u32 size_mask;
312 u16 stride;
313 u16 log_stride;
314 u16 cqn; /* index of port CQ associated with this ring */
315 u32 prod;
316 u32 cons;
317 u32 buf_size;
318 u8 fcs_del;
319 void *buf;
320 void *rx_info;
321 unsigned long bytes;
322 unsigned long packets;
323 #ifdef CONFIG_NET_RX_BUSY_POLL
324 unsigned long yields;
325 unsigned long misses;
326 unsigned long cleaned;
327 #endif
328 unsigned long csum_ok;
329 unsigned long csum_none;
330 int hwtstamp_rx_filter;
331 cpumask_var_t affinity_mask;
332 };
333
334 struct mlx4_en_cq {
335 struct mlx4_cq mcq;
336 struct mlx4_hwq_resources wqres;
337 int ring;
338 struct net_device *dev;
339 struct napi_struct napi;
340 int size;
341 int buf_size;
342 unsigned vector;
343 enum cq_type is_tx;
344 u16 moder_time;
345 u16 moder_cnt;
346 struct mlx4_cqe *buf;
347 #define MLX4_EN_OPCODE_ERROR 0x1e
348
349 #ifdef CONFIG_NET_RX_BUSY_POLL
350 unsigned int state;
351 #define MLX4_EN_CQ_STATE_IDLE 0
352 #define MLX4_EN_CQ_STATE_NAPI 1 /* NAPI owns this CQ */
353 #define MLX4_EN_CQ_STATE_POLL 2 /* poll owns this CQ */
354 #define MLX4_CQ_LOCKED (MLX4_EN_CQ_STATE_NAPI | MLX4_EN_CQ_STATE_POLL)
355 #define MLX4_EN_CQ_STATE_NAPI_YIELD 4 /* NAPI yielded this CQ */
356 #define MLX4_EN_CQ_STATE_POLL_YIELD 8 /* poll yielded this CQ */
357 #define CQ_YIELD (MLX4_EN_CQ_STATE_NAPI_YIELD | MLX4_EN_CQ_STATE_POLL_YIELD)
358 #define CQ_USER_PEND (MLX4_EN_CQ_STATE_POLL | MLX4_EN_CQ_STATE_POLL_YIELD)
359 spinlock_t poll_lock; /* protects from LLS/napi conflicts */
360 #endif /* CONFIG_NET_RX_BUSY_POLL */
361 struct irq_desc *irq_desc;
362 };
363
364 struct mlx4_en_port_profile {
365 u32 flags;
366 u32 tx_ring_num;
367 u32 rx_ring_num;
368 u32 tx_ring_size;
369 u32 rx_ring_size;
370 u8 rx_pause;
371 u8 rx_ppp;
372 u8 tx_pause;
373 u8 tx_ppp;
374 int rss_rings;
375 int inline_thold;
376 };
377
378 struct mlx4_en_profile {
379 int rss_xor;
380 int udp_rss;
381 u8 rss_mask;
382 u32 active_ports;
383 u32 small_pkt_int;
384 u8 no_reset;
385 u8 num_tx_rings_p_up;
386 struct mlx4_en_port_profile prof[MLX4_MAX_PORTS + 1];
387 };
388
389 struct mlx4_en_dev {
390 struct mlx4_dev *dev;
391 struct pci_dev *pdev;
392 struct mutex state_lock;
393 struct net_device *pndev[MLX4_MAX_PORTS + 1];
394 u32 port_cnt;
395 bool device_up;
396 struct mlx4_en_profile profile;
397 u32 LSO_support;
398 struct workqueue_struct *workqueue;
399 struct device *dma_device;
400 void __iomem *uar_map;
401 struct mlx4_uar priv_uar;
402 struct mlx4_mr mr;
403 u32 priv_pdn;
404 spinlock_t uar_lock;
405 u8 mac_removed[MLX4_MAX_PORTS + 1];
406 rwlock_t clock_lock;
407 u32 nominal_c_mult;
408 struct cyclecounter cycles;
409 struct timecounter clock;
410 unsigned long last_overflow_check;
411 unsigned long overflow_period;
412 struct ptp_clock *ptp_clock;
413 struct ptp_clock_info ptp_clock_info;
414 };
415
416
417 struct mlx4_en_rss_map {
418 int base_qpn;
419 struct mlx4_qp qps[MAX_RX_RINGS];
420 enum mlx4_qp_state state[MAX_RX_RINGS];
421 struct mlx4_qp indir_qp;
422 enum mlx4_qp_state indir_state;
423 };
424
425 struct mlx4_en_port_state {
426 int link_state;
427 int link_speed;
428 int transciver;
429 };
430
431 struct mlx4_en_pkt_stats {
432 unsigned long broadcast;
433 unsigned long rx_prio[8];
434 unsigned long tx_prio[8];
435 #define NUM_PKT_STATS 17
436 };
437
438 struct mlx4_en_port_stats {
439 unsigned long tso_packets;
440 unsigned long xmit_more;
441 unsigned long queue_stopped;
442 unsigned long wake_queue;
443 unsigned long tx_timeout;
444 unsigned long rx_alloc_failed;
445 unsigned long rx_chksum_good;
446 unsigned long rx_chksum_none;
447 unsigned long tx_chksum_offload;
448 #define NUM_PORT_STATS 9
449 };
450
451 struct mlx4_en_perf_stats {
452 u32 tx_poll;
453 u64 tx_pktsz_avg;
454 u32 inflight_avg;
455 u16 tx_coal_avg;
456 u16 rx_coal_avg;
457 u32 napi_quota;
458 #define NUM_PERF_COUNTERS 6
459 };
460
461 enum mlx4_en_mclist_act {
462 MCLIST_NONE,
463 MCLIST_REM,
464 MCLIST_ADD,
465 };
466
467 struct mlx4_en_mc_list {
468 struct list_head list;
469 enum mlx4_en_mclist_act action;
470 u8 addr[ETH_ALEN];
471 u64 reg_id;
472 u64 tunnel_reg_id;
473 };
474
475 struct mlx4_en_frag_info {
476 u16 frag_size;
477 u16 frag_prefix_size;
478 u16 frag_stride;
479 u16 frag_align;
480 };
481
482 #ifdef CONFIG_MLX4_EN_DCB
483 /* Minimal TC BW - setting to 0 will block traffic */
484 #define MLX4_EN_BW_MIN 1
485 #define MLX4_EN_BW_MAX 100 /* Utilize 100% of the line */
486
487 #define MLX4_EN_TC_ETS 7
488
489 #endif
490
491 struct ethtool_flow_id {
492 struct list_head list;
493 struct ethtool_rx_flow_spec flow_spec;
494 u64 id;
495 };
496
497 enum {
498 MLX4_EN_FLAG_PROMISC = (1 << 0),
499 MLX4_EN_FLAG_MC_PROMISC = (1 << 1),
500 /* whether we need to enable hardware loopback by putting dmac
501 * in Tx WQE
502 */
503 MLX4_EN_FLAG_ENABLE_HW_LOOPBACK = (1 << 2),
504 /* whether we need to drop packets that hardware loopback-ed */
505 MLX4_EN_FLAG_RX_FILTER_NEEDED = (1 << 3),
506 MLX4_EN_FLAG_FORCE_PROMISC = (1 << 4)
507 };
508
509 #define MLX4_EN_MAC_HASH_SIZE (1 << BITS_PER_BYTE)
510 #define MLX4_EN_MAC_HASH_IDX 5
511
512 struct mlx4_en_priv {
513 struct mlx4_en_dev *mdev;
514 struct mlx4_en_port_profile *prof;
515 struct net_device *dev;
516 unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
517 struct net_device_stats stats;
518 struct net_device_stats ret_stats;
519 struct mlx4_en_port_state port_state;
520 spinlock_t stats_lock;
521 struct ethtool_flow_id ethtool_rules[MAX_NUM_OF_FS_RULES];
522 /* To allow rules removal while port is going down */
523 struct list_head ethtool_list;
524
525 unsigned long last_moder_packets[MAX_RX_RINGS];
526 unsigned long last_moder_tx_packets;
527 unsigned long last_moder_bytes[MAX_RX_RINGS];
528 unsigned long last_moder_jiffies;
529 int last_moder_time[MAX_RX_RINGS];
530 u16 rx_usecs;
531 u16 rx_frames;
532 u16 tx_usecs;
533 u16 tx_frames;
534 u32 pkt_rate_low;
535 u16 rx_usecs_low;
536 u32 pkt_rate_high;
537 u16 rx_usecs_high;
538 u16 sample_interval;
539 u16 adaptive_rx_coal;
540 u32 msg_enable;
541 u32 loopback_ok;
542 u32 validate_loopback;
543
544 struct mlx4_hwq_resources res;
545 int link_state;
546 int last_link_state;
547 bool port_up;
548 int port;
549 int registered;
550 int allocated;
551 int stride;
552 unsigned char current_mac[ETH_ALEN + 2];
553 int mac_index;
554 unsigned max_mtu;
555 int base_qpn;
556 int cqe_factor;
557 int cqe_size;
558
559 struct mlx4_en_rss_map rss_map;
560 __be32 ctrl_flags;
561 u32 flags;
562 u8 num_tx_rings_p_up;
563 u32 tx_work_limit;
564 u32 tx_ring_num;
565 u32 rx_ring_num;
566 u32 rx_skb_size;
567 struct mlx4_en_frag_info frag_info[MLX4_EN_MAX_RX_FRAGS];
568 u16 num_frags;
569 u16 log_rx_info;
570
571 struct mlx4_en_tx_ring **tx_ring;
572 struct mlx4_en_rx_ring *rx_ring[MAX_RX_RINGS];
573 struct mlx4_en_cq **tx_cq;
574 struct mlx4_en_cq *rx_cq[MAX_RX_RINGS];
575 struct mlx4_qp drop_qp;
576 struct work_struct rx_mode_task;
577 struct work_struct watchdog_task;
578 struct work_struct linkstate_task;
579 struct delayed_work stats_task;
580 struct delayed_work service_task;
581 #ifdef CONFIG_MLX4_EN_VXLAN
582 struct work_struct vxlan_add_task;
583 struct work_struct vxlan_del_task;
584 #endif
585 struct mlx4_en_perf_stats pstats;
586 struct mlx4_en_pkt_stats pkstats;
587 struct mlx4_en_port_stats port_stats;
588 u64 stats_bitmap;
589 struct list_head mc_list;
590 struct list_head curr_list;
591 u64 broadcast_id;
592 struct mlx4_en_stat_out_mbox hw_stats;
593 int vids[128];
594 bool wol;
595 struct device *ddev;
596 int base_tx_qpn;
597 struct hlist_head mac_hash[MLX4_EN_MAC_HASH_SIZE];
598 struct hwtstamp_config hwtstamp_config;
599
600 #ifdef CONFIG_MLX4_EN_DCB
601 struct ieee_ets ets;
602 u16 maxrate[IEEE_8021QAZ_MAX_TCS];
603 #endif
604 #ifdef CONFIG_RFS_ACCEL
605 spinlock_t filters_lock;
606 int last_filter_id;
607 struct list_head filters;
608 struct hlist_head filter_hash[1 << MLX4_EN_FILTER_HASH_SHIFT];
609 #endif
610 u64 tunnel_reg_id;
611 __be16 vxlan_port;
612
613 u32 pflags;
614 };
615
616 enum mlx4_en_wol {
617 MLX4_EN_WOL_MAGIC = (1ULL << 61),
618 MLX4_EN_WOL_ENABLED = (1ULL << 62),
619 };
620
621 struct mlx4_mac_entry {
622 struct hlist_node hlist;
623 unsigned char mac[ETH_ALEN + 2];
624 u64 reg_id;
625 struct rcu_head rcu;
626 };
627
mlx4_en_get_cqe(void * buf,int idx,int cqe_sz)628 static inline struct mlx4_cqe *mlx4_en_get_cqe(void *buf, int idx, int cqe_sz)
629 {
630 return buf + idx * cqe_sz;
631 }
632
633 #ifdef CONFIG_NET_RX_BUSY_POLL
mlx4_en_cq_init_lock(struct mlx4_en_cq * cq)634 static inline void mlx4_en_cq_init_lock(struct mlx4_en_cq *cq)
635 {
636 spin_lock_init(&cq->poll_lock);
637 cq->state = MLX4_EN_CQ_STATE_IDLE;
638 }
639
640 /* called from the device poll rutine to get ownership of a cq */
mlx4_en_cq_lock_napi(struct mlx4_en_cq * cq)641 static inline bool mlx4_en_cq_lock_napi(struct mlx4_en_cq *cq)
642 {
643 int rc = true;
644 spin_lock(&cq->poll_lock);
645 if (cq->state & MLX4_CQ_LOCKED) {
646 WARN_ON(cq->state & MLX4_EN_CQ_STATE_NAPI);
647 cq->state |= MLX4_EN_CQ_STATE_NAPI_YIELD;
648 rc = false;
649 } else
650 /* we don't care if someone yielded */
651 cq->state = MLX4_EN_CQ_STATE_NAPI;
652 spin_unlock(&cq->poll_lock);
653 return rc;
654 }
655
656 /* returns true is someone tried to get the cq while napi had it */
mlx4_en_cq_unlock_napi(struct mlx4_en_cq * cq)657 static inline bool mlx4_en_cq_unlock_napi(struct mlx4_en_cq *cq)
658 {
659 int rc = false;
660 spin_lock(&cq->poll_lock);
661 WARN_ON(cq->state & (MLX4_EN_CQ_STATE_POLL |
662 MLX4_EN_CQ_STATE_NAPI_YIELD));
663
664 if (cq->state & MLX4_EN_CQ_STATE_POLL_YIELD)
665 rc = true;
666 cq->state = MLX4_EN_CQ_STATE_IDLE;
667 spin_unlock(&cq->poll_lock);
668 return rc;
669 }
670
671 /* called from mlx4_en_low_latency_poll() */
mlx4_en_cq_lock_poll(struct mlx4_en_cq * cq)672 static inline bool mlx4_en_cq_lock_poll(struct mlx4_en_cq *cq)
673 {
674 int rc = true;
675 spin_lock_bh(&cq->poll_lock);
676 if ((cq->state & MLX4_CQ_LOCKED)) {
677 struct net_device *dev = cq->dev;
678 struct mlx4_en_priv *priv = netdev_priv(dev);
679 struct mlx4_en_rx_ring *rx_ring = priv->rx_ring[cq->ring];
680
681 cq->state |= MLX4_EN_CQ_STATE_POLL_YIELD;
682 rc = false;
683 rx_ring->yields++;
684 } else
685 /* preserve yield marks */
686 cq->state |= MLX4_EN_CQ_STATE_POLL;
687 spin_unlock_bh(&cq->poll_lock);
688 return rc;
689 }
690
691 /* returns true if someone tried to get the cq while it was locked */
mlx4_en_cq_unlock_poll(struct mlx4_en_cq * cq)692 static inline bool mlx4_en_cq_unlock_poll(struct mlx4_en_cq *cq)
693 {
694 int rc = false;
695 spin_lock_bh(&cq->poll_lock);
696 WARN_ON(cq->state & (MLX4_EN_CQ_STATE_NAPI));
697
698 if (cq->state & MLX4_EN_CQ_STATE_POLL_YIELD)
699 rc = true;
700 cq->state = MLX4_EN_CQ_STATE_IDLE;
701 spin_unlock_bh(&cq->poll_lock);
702 return rc;
703 }
704
705 /* true if a socket is polling, even if it did not get the lock */
mlx4_en_cq_busy_polling(struct mlx4_en_cq * cq)706 static inline bool mlx4_en_cq_busy_polling(struct mlx4_en_cq *cq)
707 {
708 WARN_ON(!(cq->state & MLX4_CQ_LOCKED));
709 return cq->state & CQ_USER_PEND;
710 }
711 #else
mlx4_en_cq_init_lock(struct mlx4_en_cq * cq)712 static inline void mlx4_en_cq_init_lock(struct mlx4_en_cq *cq)
713 {
714 }
715
mlx4_en_cq_lock_napi(struct mlx4_en_cq * cq)716 static inline bool mlx4_en_cq_lock_napi(struct mlx4_en_cq *cq)
717 {
718 return true;
719 }
720
mlx4_en_cq_unlock_napi(struct mlx4_en_cq * cq)721 static inline bool mlx4_en_cq_unlock_napi(struct mlx4_en_cq *cq)
722 {
723 return false;
724 }
725
mlx4_en_cq_lock_poll(struct mlx4_en_cq * cq)726 static inline bool mlx4_en_cq_lock_poll(struct mlx4_en_cq *cq)
727 {
728 return false;
729 }
730
mlx4_en_cq_unlock_poll(struct mlx4_en_cq * cq)731 static inline bool mlx4_en_cq_unlock_poll(struct mlx4_en_cq *cq)
732 {
733 return false;
734 }
735
mlx4_en_cq_busy_polling(struct mlx4_en_cq * cq)736 static inline bool mlx4_en_cq_busy_polling(struct mlx4_en_cq *cq)
737 {
738 return false;
739 }
740 #endif /* CONFIG_NET_RX_BUSY_POLL */
741
742 #define MLX4_EN_WOL_DO_MODIFY (1ULL << 63)
743
744 void mlx4_en_update_loopback_state(struct net_device *dev,
745 netdev_features_t features);
746
747 void mlx4_en_destroy_netdev(struct net_device *dev);
748 int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
749 struct mlx4_en_port_profile *prof);
750
751 int mlx4_en_start_port(struct net_device *dev);
752 void mlx4_en_stop_port(struct net_device *dev, int detach);
753
754 void mlx4_en_free_resources(struct mlx4_en_priv *priv);
755 int mlx4_en_alloc_resources(struct mlx4_en_priv *priv);
756
757 int mlx4_en_create_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq,
758 int entries, int ring, enum cq_type mode, int node);
759 void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq);
760 int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
761 int cq_idx);
762 void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
763 int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
764 int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
765
766 void mlx4_en_tx_irq(struct mlx4_cq *mcq);
767 u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb,
768 void *accel_priv, select_queue_fallback_t fallback);
769 netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev);
770
771 int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
772 struct mlx4_en_tx_ring **pring,
773 int qpn, u32 size, u16 stride,
774 int node, int queue_index);
775 void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
776 struct mlx4_en_tx_ring **pring);
777 int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
778 struct mlx4_en_tx_ring *ring,
779 int cq, int user_prio);
780 void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv,
781 struct mlx4_en_tx_ring *ring);
782 void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev);
783 void mlx4_en_recover_from_oom(struct mlx4_en_priv *priv);
784 int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
785 struct mlx4_en_rx_ring **pring,
786 u32 size, u16 stride, int node);
787 void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
788 struct mlx4_en_rx_ring **pring,
789 u32 size, u16 stride);
790 int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv);
791 void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
792 struct mlx4_en_rx_ring *ring);
793 int mlx4_en_process_rx_cq(struct net_device *dev,
794 struct mlx4_en_cq *cq,
795 int budget);
796 int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget);
797 int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget);
798 void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride,
799 int is_tx, int rss, int qpn, int cqn, int user_prio,
800 struct mlx4_qp_context *context);
801 void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event);
802 int mlx4_en_map_buffer(struct mlx4_buf *buf);
803 void mlx4_en_unmap_buffer(struct mlx4_buf *buf);
804
805 void mlx4_en_calc_rx_buf(struct net_device *dev);
806 int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv);
807 void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv);
808 int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv);
809 void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv);
810 int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring);
811 void mlx4_en_rx_irq(struct mlx4_cq *mcq);
812
813 int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode);
814 int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, struct mlx4_en_priv *priv);
815
816 int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset);
817 int mlx4_en_QUERY_PORT(struct mlx4_en_dev *mdev, u8 port);
818
819 #ifdef CONFIG_MLX4_EN_DCB
820 extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_ops;
821 extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_pfc_ops;
822 #endif
823
824 int mlx4_en_setup_tc(struct net_device *dev, u8 up);
825
826 #ifdef CONFIG_RFS_ACCEL
827 void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv);
828 #endif
829
830 #define MLX4_EN_NUM_SELF_TEST 5
831 void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf);
832 void mlx4_en_ptp_overflow_check(struct mlx4_en_dev *mdev);
833
834 /*
835 * Functions for time stamping
836 */
837 u64 mlx4_en_get_cqe_ts(struct mlx4_cqe *cqe);
838 void mlx4_en_fill_hwtstamps(struct mlx4_en_dev *mdev,
839 struct skb_shared_hwtstamps *hwts,
840 u64 timestamp);
841 void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev);
842 void mlx4_en_remove_timestamp(struct mlx4_en_dev *mdev);
843 int mlx4_en_timestamp_config(struct net_device *dev,
844 int tx_type,
845 int rx_filter);
846
847 /* Globals
848 */
849 extern const struct ethtool_ops mlx4_en_ethtool_ops;
850
851
852
853 /*
854 * printk / logging functions
855 */
856
857 __printf(3, 4)
858 void en_print(const char *level, const struct mlx4_en_priv *priv,
859 const char *format, ...);
860
861 #define en_dbg(mlevel, priv, format, ...) \
862 do { \
863 if (NETIF_MSG_##mlevel & (priv)->msg_enable) \
864 en_print(KERN_DEBUG, priv, format, ##__VA_ARGS__); \
865 } while (0)
866 #define en_warn(priv, format, ...) \
867 en_print(KERN_WARNING, priv, format, ##__VA_ARGS__)
868 #define en_err(priv, format, ...) \
869 en_print(KERN_ERR, priv, format, ##__VA_ARGS__)
870 #define en_info(priv, format, ...) \
871 en_print(KERN_INFO, priv, format, ##__VA_ARGS__)
872
873 #define mlx4_err(mdev, format, ...) \
874 pr_err(DRV_NAME " %s: " format, \
875 dev_name(&(mdev)->pdev->dev), ##__VA_ARGS__)
876 #define mlx4_info(mdev, format, ...) \
877 pr_info(DRV_NAME " %s: " format, \
878 dev_name(&(mdev)->pdev->dev), ##__VA_ARGS__)
879 #define mlx4_warn(mdev, format, ...) \
880 pr_warn(DRV_NAME " %s: " format, \
881 dev_name(&(mdev)->pdev->dev), ##__VA_ARGS__)
882
883 #endif
884