1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Marvell RVU Ethernet driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #ifndef OTX2_COMMON_H
9 #define OTX2_COMMON_H
10
11 #include <linux/ethtool.h>
12 #include <linux/pci.h>
13 #include <linux/iommu.h>
14 #include <linux/net_tstamp.h>
15 #include <linux/ptp_clock_kernel.h>
16 #include <linux/timecounter.h>
17 #include <linux/soc/marvell/octeontx2/asm.h>
18 #include <net/pkt_cls.h>
19 #include <net/devlink.h>
20 #include <linux/time64.h>
21 #include <linux/dim.h>
22 #include <uapi/linux/if_macsec.h>
23
24 #include <mbox.h>
25 #include <npc.h>
26 #include "otx2_reg.h"
27 #include "otx2_txrx.h"
28 #include "otx2_devlink.h"
29 #include <rvu_trace.h>
30 #include "qos.h"
31
32 /* PCI device IDs */
33 #define PCI_DEVID_OCTEONTX2_RVU_PF 0xA063
34 #define PCI_DEVID_OCTEONTX2_RVU_VF 0xA064
35 #define PCI_DEVID_OCTEONTX2_RVU_AFVF 0xA0F8
36
37 #define PCI_SUBSYS_DEVID_96XX_RVU_PFVF 0xB200
38 #define PCI_SUBSYS_DEVID_CN10K_B_RVU_PFVF 0xBD00
39
40 /* PCI BAR nos */
41 #define PCI_CFG_REG_BAR_NUM 2
42 #define PCI_MBOX_BAR_NUM 4
43
44 #define NAME_SIZE 32
45
46 #ifdef CONFIG_DCB
47 /* Max priority supported for PFC */
48 #define NIX_PF_PFC_PRIO_MAX 8
49 #endif
50
51 enum arua_mapped_qtypes {
52 AURA_NIX_RQ,
53 AURA_NIX_SQ,
54 };
55
56 /* NIX LF interrupts range*/
57 #define NIX_LF_QINT_VEC_START 0x00
58 #define NIX_LF_CINT_VEC_START 0x40
59 #define NIX_LF_GINT_VEC 0x80
60 #define NIX_LF_ERR_VEC 0x81
61 #define NIX_LF_POISON_VEC 0x82
62
63 /* Send skid of 2000 packets required for CQ size of 4K CQEs. */
64 #define SEND_CQ_SKID 2000
65
66 #define OTX2_GET_RX_STATS(reg) \
67 otx2_read64(pfvf, NIX_LF_RX_STATX(reg))
68 #define OTX2_GET_TX_STATS(reg) \
69 otx2_read64(pfvf, NIX_LF_TX_STATX(reg))
70
71 struct otx2_lmt_info {
72 u64 lmt_addr;
73 u16 lmt_id;
74 };
75 /* RSS configuration */
76 struct otx2_rss_ctx {
77 u8 ind_tbl[MAX_RSS_INDIR_TBL_SIZE];
78 };
79
80 struct otx2_rss_info {
81 u8 enable;
82 u32 flowkey_cfg;
83 u16 rss_size;
84 #define RSS_HASH_KEY_SIZE 44 /* 352 bit key */
85 u8 key[RSS_HASH_KEY_SIZE];
86 struct otx2_rss_ctx *rss_ctx[MAX_RSS_GROUPS];
87 };
88
89 /* NIX (or NPC) RX errors */
90 enum otx2_errlvl {
91 NPC_ERRLVL_RE,
92 NPC_ERRLVL_LID_LA,
93 NPC_ERRLVL_LID_LB,
94 NPC_ERRLVL_LID_LC,
95 NPC_ERRLVL_LID_LD,
96 NPC_ERRLVL_LID_LE,
97 NPC_ERRLVL_LID_LF,
98 NPC_ERRLVL_LID_LG,
99 NPC_ERRLVL_LID_LH,
100 NPC_ERRLVL_NIX = 0x0F,
101 };
102
103 enum otx2_errcodes_re {
104 /* NPC_ERRLVL_RE errcodes */
105 ERRCODE_FCS = 0x7,
106 ERRCODE_FCS_RCV = 0x8,
107 ERRCODE_UNDERSIZE = 0x10,
108 ERRCODE_OVERSIZE = 0x11,
109 ERRCODE_OL2_LEN_MISMATCH = 0x12,
110 /* NPC_ERRLVL_NIX errcodes */
111 ERRCODE_OL3_LEN = 0x10,
112 ERRCODE_OL4_LEN = 0x11,
113 ERRCODE_OL4_CSUM = 0x12,
114 ERRCODE_IL3_LEN = 0x20,
115 ERRCODE_IL4_LEN = 0x21,
116 ERRCODE_IL4_CSUM = 0x22,
117 };
118
119 /* NIX TX stats */
120 enum nix_stat_lf_tx {
121 TX_UCAST = 0x0,
122 TX_BCAST = 0x1,
123 TX_MCAST = 0x2,
124 TX_DROP = 0x3,
125 TX_OCTS = 0x4,
126 TX_STATS_ENUM_LAST,
127 };
128
129 /* NIX RX stats */
130 enum nix_stat_lf_rx {
131 RX_OCTS = 0x0,
132 RX_UCAST = 0x1,
133 RX_BCAST = 0x2,
134 RX_MCAST = 0x3,
135 RX_DROP = 0x4,
136 RX_DROP_OCTS = 0x5,
137 RX_FCS = 0x6,
138 RX_ERR = 0x7,
139 RX_DRP_BCAST = 0x8,
140 RX_DRP_MCAST = 0x9,
141 RX_DRP_L3BCAST = 0xa,
142 RX_DRP_L3MCAST = 0xb,
143 RX_STATS_ENUM_LAST,
144 };
145
146 struct otx2_dev_stats {
147 u64 rx_bytes;
148 u64 rx_frames;
149 u64 rx_ucast_frames;
150 u64 rx_bcast_frames;
151 u64 rx_mcast_frames;
152 u64 rx_drops;
153
154 u64 tx_bytes;
155 u64 tx_frames;
156 u64 tx_ucast_frames;
157 u64 tx_bcast_frames;
158 u64 tx_mcast_frames;
159 u64 tx_drops;
160 };
161
162 /* Driver counted stats */
163 struct otx2_drv_stats {
164 atomic_t rx_fcs_errs;
165 atomic_t rx_oversize_errs;
166 atomic_t rx_undersize_errs;
167 atomic_t rx_csum_errs;
168 atomic_t rx_len_errs;
169 atomic_t rx_other_errs;
170 };
171
172 struct mbox {
173 struct otx2_mbox mbox;
174 struct work_struct mbox_wrk;
175 struct otx2_mbox mbox_up;
176 struct work_struct mbox_up_wrk;
177 struct otx2_nic *pfvf;
178 void *bbuf_base; /* Bounce buffer for mbox memory */
179 struct mutex lock; /* serialize mailbox access */
180 int num_msgs; /* mbox number of messages */
181 int up_num_msgs; /* mbox_up number of messages */
182 };
183
184 struct otx2_hw {
185 struct pci_dev *pdev;
186 struct otx2_rss_info rss_info;
187 u16 rx_queues;
188 u16 tx_queues;
189 u16 xdp_queues;
190 u16 tc_tx_queues;
191 u16 non_qos_queues; /* tx queues plus xdp queues */
192 u16 max_queues;
193 u16 pool_cnt;
194 u16 rqpool_cnt;
195 u16 sqpool_cnt;
196
197 #define OTX2_DEFAULT_RBUF_LEN 2048
198 u16 rbuf_len;
199 u32 xqe_size;
200
201 /* NPA */
202 u32 stack_pg_ptrs; /* No of ptrs per stack page */
203 u32 stack_pg_bytes; /* Size of stack page */
204 u16 sqb_size;
205
206 /* NIX */
207 u8 txschq_link_cfg_lvl;
208 u16 txschq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC];
209 u16 matchall_ipolicer;
210 u32 dwrr_mtu;
211
212 /* HW settings, coalescing etc */
213 u16 rx_chan_base;
214 u16 tx_chan_base;
215 u16 cq_qcount_wait;
216 u16 cq_ecount_wait;
217 u16 rq_skid;
218 u8 cq_time_wait;
219
220 /* Segmentation */
221 u8 lso_tsov4_idx;
222 u8 lso_tsov6_idx;
223 u8 lso_udpv4_idx;
224 u8 lso_udpv6_idx;
225
226 /* RSS */
227 u8 flowkey_alg_idx;
228
229 /* MSI-X */
230 u8 cint_cnt; /* CQ interrupt count */
231 u16 npa_msixoff; /* Offset of NPA vectors */
232 u16 nix_msixoff; /* Offset of NIX vectors */
233 char *irq_name;
234 cpumask_var_t *affinity_mask;
235
236 /* Stats */
237 struct otx2_dev_stats dev_stats;
238 struct otx2_drv_stats drv_stats;
239 u64 cgx_rx_stats[CGX_RX_STATS_COUNT];
240 u64 cgx_tx_stats[CGX_TX_STATS_COUNT];
241 u64 cgx_fec_corr_blks;
242 u64 cgx_fec_uncorr_blks;
243 u8 cgx_links; /* No. of CGX links present in HW */
244 u8 lbk_links; /* No. of LBK links present in HW */
245 u8 tx_link; /* Transmit channel link number */
246 #define HW_TSO 0
247 #define CN10K_MBOX 1
248 #define CN10K_LMTST 2
249 #define CN10K_RPM 3
250 #define CN10K_PTP_ONESTEP 4
251 #define CN10K_HW_MACSEC 5
252 unsigned long cap_flag;
253
254 #define LMT_LINE_SIZE 128
255 #define LMT_BURST_SIZE 32 /* 32 LMTST lines for burst SQE flush */
256 u64 *lmt_base;
257 struct otx2_lmt_info __percpu *lmt_info;
258 };
259
260 enum vfperm {
261 OTX2_RESET_VF_PERM,
262 OTX2_TRUSTED_VF,
263 };
264
265 struct otx2_vf_config {
266 struct otx2_nic *pf;
267 struct delayed_work link_event_work;
268 bool intf_down; /* interface was either configured or not */
269 u8 mac[ETH_ALEN];
270 u16 vlan;
271 int tx_vtag_idx;
272 bool trusted;
273 };
274
275 struct flr_work {
276 struct work_struct work;
277 struct otx2_nic *pf;
278 };
279
280 struct refill_work {
281 struct delayed_work pool_refill_work;
282 struct otx2_nic *pf;
283 };
284
285 /* PTPv2 originTimestamp structure */
286 struct ptpv2_tstamp {
287 __be16 seconds_msb; /* 16 bits + */
288 __be32 seconds_lsb; /* 32 bits = 48 bits*/
289 __be32 nanoseconds;
290 } __packed;
291
292 struct otx2_ptp {
293 struct ptp_clock_info ptp_info;
294 struct ptp_clock *ptp_clock;
295 struct otx2_nic *nic;
296
297 struct cyclecounter cycle_counter;
298 struct timecounter time_counter;
299
300 struct delayed_work extts_work;
301 u64 last_extts;
302 u64 thresh;
303
304 struct ptp_pin_desc extts_config;
305 u64 (*convert_rx_ptp_tstmp)(u64 timestamp);
306 u64 (*convert_tx_ptp_tstmp)(u64 timestamp);
307 struct delayed_work synctstamp_work;
308 u64 tstamp;
309 u32 base_ns;
310 };
311
312 #define OTX2_HW_TIMESTAMP_LEN 8
313
314 struct otx2_mac_table {
315 u8 addr[ETH_ALEN];
316 u16 mcam_entry;
317 bool inuse;
318 };
319
320 struct otx2_flow_config {
321 u16 *flow_ent;
322 u16 *def_ent;
323 u16 nr_flows;
324 #define OTX2_DEFAULT_FLOWCOUNT 16
325 #define OTX2_MAX_UNICAST_FLOWS 8
326 #define OTX2_MAX_VLAN_FLOWS 1
327 #define OTX2_MAX_TC_FLOWS OTX2_DEFAULT_FLOWCOUNT
328 #define OTX2_MCAM_COUNT (OTX2_DEFAULT_FLOWCOUNT + \
329 OTX2_MAX_UNICAST_FLOWS + \
330 OTX2_MAX_VLAN_FLOWS)
331 u16 unicast_offset;
332 u16 rx_vlan_offset;
333 u16 vf_vlan_offset;
334 #define OTX2_PER_VF_VLAN_FLOWS 2 /* Rx + Tx per VF */
335 #define OTX2_VF_VLAN_RX_INDEX 0
336 #define OTX2_VF_VLAN_TX_INDEX 1
337 u32 *bmap_to_dmacindex;
338 unsigned long *dmacflt_bmap;
339 struct list_head flow_list;
340 u32 dmacflt_max_flows;
341 u16 max_flows;
342 struct list_head flow_list_tc;
343 bool ntuple;
344 };
345
346 struct dev_hw_ops {
347 int (*sq_aq_init)(void *dev, u16 qidx, u16 sqb_aura);
348 void (*sqe_flush)(void *dev, struct otx2_snd_queue *sq,
349 int size, int qidx);
350 void (*refill_pool_ptrs)(void *dev, struct otx2_cq_queue *cq);
351 void (*aura_freeptr)(void *dev, int aura, u64 buf);
352 };
353
354 #define CN10K_MCS_SA_PER_SC 4
355
356 /* Stats which need to be accumulated in software because
357 * of shared counters in hardware.
358 */
359 struct cn10k_txsc_stats {
360 u64 InPktsUntagged;
361 u64 InPktsNoTag;
362 u64 InPktsBadTag;
363 u64 InPktsUnknownSCI;
364 u64 InPktsNoSCI;
365 u64 InPktsOverrun;
366 };
367
368 struct cn10k_rxsc_stats {
369 u64 InOctetsValidated;
370 u64 InOctetsDecrypted;
371 u64 InPktsUnchecked;
372 u64 InPktsDelayed;
373 u64 InPktsOK;
374 u64 InPktsInvalid;
375 u64 InPktsLate;
376 u64 InPktsNotValid;
377 u64 InPktsNotUsingSA;
378 u64 InPktsUnusedSA;
379 };
380
381 struct cn10k_mcs_txsc {
382 struct macsec_secy *sw_secy;
383 struct cn10k_txsc_stats stats;
384 struct list_head entry;
385 enum macsec_validation_type last_validate_frames;
386 bool last_replay_protect;
387 u16 hw_secy_id_tx;
388 u16 hw_secy_id_rx;
389 u16 hw_flow_id;
390 u16 hw_sc_id;
391 u16 hw_sa_id[CN10K_MCS_SA_PER_SC];
392 u8 sa_bmap;
393 u8 sa_key[CN10K_MCS_SA_PER_SC][MACSEC_MAX_KEY_LEN];
394 u8 encoding_sa;
395 };
396
397 struct cn10k_mcs_rxsc {
398 struct macsec_secy *sw_secy;
399 struct macsec_rx_sc *sw_rxsc;
400 struct cn10k_rxsc_stats stats;
401 struct list_head entry;
402 u16 hw_flow_id;
403 u16 hw_sc_id;
404 u16 hw_sa_id[CN10K_MCS_SA_PER_SC];
405 u8 sa_bmap;
406 u8 sa_key[CN10K_MCS_SA_PER_SC][MACSEC_MAX_KEY_LEN];
407 };
408
409 struct cn10k_mcs_cfg {
410 struct list_head txsc_list;
411 struct list_head rxsc_list;
412 };
413
414 struct otx2_nic {
415 void __iomem *reg_base;
416 struct net_device *netdev;
417 struct dev_hw_ops *hw_ops;
418 void *iommu_domain;
419 u16 tx_max_pktlen;
420 u16 rbsize; /* Receive buffer size */
421
422 #define OTX2_FLAG_RX_TSTAMP_ENABLED BIT_ULL(0)
423 #define OTX2_FLAG_TX_TSTAMP_ENABLED BIT_ULL(1)
424 #define OTX2_FLAG_INTF_DOWN BIT_ULL(2)
425 #define OTX2_FLAG_MCAM_ENTRIES_ALLOC BIT_ULL(3)
426 #define OTX2_FLAG_NTUPLE_SUPPORT BIT_ULL(4)
427 #define OTX2_FLAG_UCAST_FLTR_SUPPORT BIT_ULL(5)
428 #define OTX2_FLAG_RX_VLAN_SUPPORT BIT_ULL(6)
429 #define OTX2_FLAG_VF_VLAN_SUPPORT BIT_ULL(7)
430 #define OTX2_FLAG_PF_SHUTDOWN BIT_ULL(8)
431 #define OTX2_FLAG_RX_PAUSE_ENABLED BIT_ULL(9)
432 #define OTX2_FLAG_TX_PAUSE_ENABLED BIT_ULL(10)
433 #define OTX2_FLAG_TC_FLOWER_SUPPORT BIT_ULL(11)
434 #define OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED BIT_ULL(12)
435 #define OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED BIT_ULL(13)
436 #define OTX2_FLAG_DMACFLTR_SUPPORT BIT_ULL(14)
437 #define OTX2_FLAG_PTP_ONESTEP_SYNC BIT_ULL(15)
438 #define OTX2_FLAG_ADPTV_INT_COAL_ENABLED BIT_ULL(16)
439 u64 flags;
440 u64 *cq_op_addr;
441
442 struct bpf_prog *xdp_prog;
443 struct otx2_qset qset;
444 struct otx2_hw hw;
445 struct pci_dev *pdev;
446 struct device *dev;
447
448 /* Mbox */
449 struct mbox mbox;
450 struct mbox *mbox_pfvf;
451 struct workqueue_struct *mbox_wq;
452 struct workqueue_struct *mbox_pfvf_wq;
453
454 u8 total_vfs;
455 u16 pcifunc; /* RVU PF_FUNC */
456 u16 bpid[NIX_MAX_BPID_CHAN];
457 struct otx2_vf_config *vf_configs;
458 struct cgx_link_user_info linfo;
459
460 /* NPC MCAM */
461 struct otx2_flow_config *flow_cfg;
462 struct otx2_mac_table *mac_table;
463
464 u64 reset_count;
465 struct work_struct reset_task;
466 struct workqueue_struct *flr_wq;
467 struct flr_work *flr_wrk;
468 struct refill_work *refill_wrk;
469 struct workqueue_struct *otx2_wq;
470 struct work_struct rx_mode_work;
471
472 /* Ethtool stuff */
473 u32 msg_enable;
474
475 /* Block address of NIX either BLKADDR_NIX0 or BLKADDR_NIX1 */
476 int nix_blkaddr;
477 /* LMTST Lines info */
478 struct qmem *dync_lmt;
479 u16 tot_lmt_lines;
480 u16 npa_lmt_lines;
481 u32 nix_lmt_size;
482
483 struct otx2_ptp *ptp;
484 struct hwtstamp_config tstamp;
485
486 unsigned long rq_bmap;
487
488 /* Devlink */
489 struct otx2_devlink *dl;
490 #ifdef CONFIG_DCB
491 /* PFC */
492 u8 pfc_en;
493 u8 *queue_to_pfc_map;
494 u16 pfc_schq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC];
495 bool pfc_alloc_status[NIX_PF_PFC_PRIO_MAX];
496 #endif
497 /* qos */
498 struct otx2_qos qos;
499
500 /* napi event count. It is needed for adaptive irq coalescing. */
501 u32 napi_events;
502
503 #if IS_ENABLED(CONFIG_MACSEC)
504 struct cn10k_mcs_cfg *macsec_cfg;
505 #endif
506 };
507
is_otx2_lbkvf(struct pci_dev * pdev)508 static inline bool is_otx2_lbkvf(struct pci_dev *pdev)
509 {
510 return pdev->device == PCI_DEVID_OCTEONTX2_RVU_AFVF;
511 }
512
is_96xx_A0(struct pci_dev * pdev)513 static inline bool is_96xx_A0(struct pci_dev *pdev)
514 {
515 return (pdev->revision == 0x00) &&
516 (pdev->subsystem_device == PCI_SUBSYS_DEVID_96XX_RVU_PFVF);
517 }
518
is_96xx_B0(struct pci_dev * pdev)519 static inline bool is_96xx_B0(struct pci_dev *pdev)
520 {
521 return (pdev->revision == 0x01) &&
522 (pdev->subsystem_device == PCI_SUBSYS_DEVID_96XX_RVU_PFVF);
523 }
524
525 /* REVID for PCIe devices.
526 * Bits 0..1: minor pass, bit 3..2: major pass
527 * bits 7..4: midr id
528 */
529 #define PCI_REVISION_ID_96XX 0x00
530 #define PCI_REVISION_ID_95XX 0x10
531 #define PCI_REVISION_ID_95XXN 0x20
532 #define PCI_REVISION_ID_98XX 0x30
533 #define PCI_REVISION_ID_95XXMM 0x40
534 #define PCI_REVISION_ID_95XXO 0xE0
535
is_dev_otx2(struct pci_dev * pdev)536 static inline bool is_dev_otx2(struct pci_dev *pdev)
537 {
538 u8 midr = pdev->revision & 0xF0;
539
540 return (midr == PCI_REVISION_ID_96XX || midr == PCI_REVISION_ID_95XX ||
541 midr == PCI_REVISION_ID_95XXN || midr == PCI_REVISION_ID_98XX ||
542 midr == PCI_REVISION_ID_95XXMM || midr == PCI_REVISION_ID_95XXO);
543 }
544
is_dev_cn10kb(struct pci_dev * pdev)545 static inline bool is_dev_cn10kb(struct pci_dev *pdev)
546 {
547 return pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_B_RVU_PFVF;
548 }
549
otx2_setup_dev_hw_settings(struct otx2_nic * pfvf)550 static inline void otx2_setup_dev_hw_settings(struct otx2_nic *pfvf)
551 {
552 struct otx2_hw *hw = &pfvf->hw;
553
554 pfvf->hw.cq_time_wait = CQ_TIMER_THRESH_DEFAULT;
555 pfvf->hw.cq_ecount_wait = CQ_CQE_THRESH_DEFAULT;
556 pfvf->hw.cq_qcount_wait = CQ_QCOUNT_DEFAULT;
557
558 __set_bit(HW_TSO, &hw->cap_flag);
559
560 if (is_96xx_A0(pfvf->pdev)) {
561 __clear_bit(HW_TSO, &hw->cap_flag);
562
563 /* Time based irq coalescing is not supported */
564 pfvf->hw.cq_qcount_wait = 0x0;
565
566 /* Due to HW issue previous silicons required minimum
567 * 600 unused CQE to avoid CQ overflow.
568 */
569 pfvf->hw.rq_skid = 600;
570 pfvf->qset.rqe_cnt = Q_COUNT(Q_SIZE_1K);
571 }
572 if (is_96xx_B0(pfvf->pdev))
573 __clear_bit(HW_TSO, &hw->cap_flag);
574
575 if (!is_dev_otx2(pfvf->pdev)) {
576 __set_bit(CN10K_MBOX, &hw->cap_flag);
577 __set_bit(CN10K_LMTST, &hw->cap_flag);
578 __set_bit(CN10K_RPM, &hw->cap_flag);
579 __set_bit(CN10K_PTP_ONESTEP, &hw->cap_flag);
580 }
581
582 if (is_dev_cn10kb(pfvf->pdev))
583 __set_bit(CN10K_HW_MACSEC, &hw->cap_flag);
584 }
585
586 /* Register read/write APIs */
otx2_get_regaddr(struct otx2_nic * nic,u64 offset)587 static inline void __iomem *otx2_get_regaddr(struct otx2_nic *nic, u64 offset)
588 {
589 u64 blkaddr;
590
591 switch ((offset >> RVU_FUNC_BLKADDR_SHIFT) & RVU_FUNC_BLKADDR_MASK) {
592 case BLKTYPE_NIX:
593 blkaddr = nic->nix_blkaddr;
594 break;
595 case BLKTYPE_NPA:
596 blkaddr = BLKADDR_NPA;
597 break;
598 default:
599 blkaddr = BLKADDR_RVUM;
600 break;
601 }
602
603 offset &= ~(RVU_FUNC_BLKADDR_MASK << RVU_FUNC_BLKADDR_SHIFT);
604 offset |= (blkaddr << RVU_FUNC_BLKADDR_SHIFT);
605
606 return nic->reg_base + offset;
607 }
608
otx2_write64(struct otx2_nic * nic,u64 offset,u64 val)609 static inline void otx2_write64(struct otx2_nic *nic, u64 offset, u64 val)
610 {
611 void __iomem *addr = otx2_get_regaddr(nic, offset);
612
613 writeq(val, addr);
614 }
615
otx2_read64(struct otx2_nic * nic,u64 offset)616 static inline u64 otx2_read64(struct otx2_nic *nic, u64 offset)
617 {
618 void __iomem *addr = otx2_get_regaddr(nic, offset);
619
620 return readq(addr);
621 }
622
623 /* Mbox bounce buffer APIs */
otx2_mbox_bbuf_init(struct mbox * mbox,struct pci_dev * pdev)624 static inline int otx2_mbox_bbuf_init(struct mbox *mbox, struct pci_dev *pdev)
625 {
626 struct otx2_mbox *otx2_mbox;
627 struct otx2_mbox_dev *mdev;
628
629 mbox->bbuf_base = devm_kmalloc(&pdev->dev, MBOX_SIZE, GFP_KERNEL);
630 if (!mbox->bbuf_base)
631 return -ENOMEM;
632
633 /* Overwrite mbox mbase to point to bounce buffer, so that PF/VF
634 * prepare all mbox messages in bounce buffer instead of directly
635 * in hw mbox memory.
636 */
637 otx2_mbox = &mbox->mbox;
638 mdev = &otx2_mbox->dev[0];
639 mdev->mbase = mbox->bbuf_base;
640
641 otx2_mbox = &mbox->mbox_up;
642 mdev = &otx2_mbox->dev[0];
643 mdev->mbase = mbox->bbuf_base;
644 return 0;
645 }
646
otx2_sync_mbox_bbuf(struct otx2_mbox * mbox,int devid)647 static inline void otx2_sync_mbox_bbuf(struct otx2_mbox *mbox, int devid)
648 {
649 u16 msgs_offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
650 void *hw_mbase = mbox->hwbase + (devid * MBOX_SIZE);
651 struct otx2_mbox_dev *mdev = &mbox->dev[devid];
652 struct mbox_hdr *hdr;
653 u64 msg_size;
654
655 if (mdev->mbase == hw_mbase)
656 return;
657
658 hdr = hw_mbase + mbox->rx_start;
659 msg_size = hdr->msg_size;
660
661 if (msg_size > mbox->rx_size - msgs_offset)
662 msg_size = mbox->rx_size - msgs_offset;
663
664 /* Copy mbox messages from mbox memory to bounce buffer */
665 memcpy(mdev->mbase + mbox->rx_start,
666 hw_mbase + mbox->rx_start, msg_size + msgs_offset);
667 }
668
669 /* With the absence of API for 128-bit IO memory access for arm64,
670 * implement required operations at place.
671 */
672 #if defined(CONFIG_ARM64)
otx2_write128(u64 lo,u64 hi,void __iomem * addr)673 static inline void otx2_write128(u64 lo, u64 hi, void __iomem *addr)
674 {
675 __asm__ volatile("stp %x[x0], %x[x1], [%x[p1],#0]!"
676 ::[x0]"r"(lo), [x1]"r"(hi), [p1]"r"(addr));
677 }
678
otx2_atomic64_add(u64 incr,u64 * ptr)679 static inline u64 otx2_atomic64_add(u64 incr, u64 *ptr)
680 {
681 u64 result;
682
683 __asm__ volatile(".cpu generic+lse\n"
684 "ldadd %x[i], %x[r], [%[b]]"
685 : [r]"=r"(result), "+m"(*ptr)
686 : [i]"r"(incr), [b]"r"(ptr)
687 : "memory");
688 return result;
689 }
690
691 #else
692 #define otx2_write128(lo, hi, addr) writeq((hi) | (lo), addr)
693 #define otx2_atomic64_add(incr, ptr) ({ *ptr += incr; })
694 #endif
695
__cn10k_aura_freeptr(struct otx2_nic * pfvf,u64 aura,u64 * ptrs,u64 num_ptrs)696 static inline void __cn10k_aura_freeptr(struct otx2_nic *pfvf, u64 aura,
697 u64 *ptrs, u64 num_ptrs)
698 {
699 struct otx2_lmt_info *lmt_info;
700 u64 size = 0, count_eot = 0;
701 u64 tar_addr, val = 0;
702
703 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id());
704 tar_addr = (__force u64)otx2_get_regaddr(pfvf, NPA_LF_AURA_BATCH_FREE0);
705 /* LMTID is same as AURA Id */
706 val = (lmt_info->lmt_id & 0x7FF) | BIT_ULL(63);
707 /* Set if [127:64] of last 128bit word has a valid pointer */
708 count_eot = (num_ptrs % 2) ? 0ULL : 1ULL;
709 /* Set AURA ID to free pointer */
710 ptrs[0] = (count_eot << 32) | (aura & 0xFFFFF);
711 /* Target address for LMTST flush tells HW how many 128bit
712 * words are valid from NPA_LF_AURA_BATCH_FREE0.
713 *
714 * tar_addr[6:4] is LMTST size-1 in units of 128b.
715 */
716 if (num_ptrs > 2) {
717 size = (sizeof(u64) * num_ptrs) / 16;
718 if (!count_eot)
719 size++;
720 tar_addr |= ((size - 1) & 0x7) << 4;
721 }
722 dma_wmb();
723 memcpy((u64 *)lmt_info->lmt_addr, ptrs, sizeof(u64) * num_ptrs);
724 /* Perform LMTST flush */
725 cn10k_lmt_flush(val, tar_addr);
726 }
727
cn10k_aura_freeptr(void * dev,int aura,u64 buf)728 static inline void cn10k_aura_freeptr(void *dev, int aura, u64 buf)
729 {
730 struct otx2_nic *pfvf = dev;
731 u64 ptrs[2];
732
733 ptrs[1] = buf;
734 get_cpu();
735 /* Free only one buffer at time during init and teardown */
736 __cn10k_aura_freeptr(pfvf, aura, ptrs, 2);
737 put_cpu();
738 }
739
740 /* Alloc pointer from pool/aura */
otx2_aura_allocptr(struct otx2_nic * pfvf,int aura)741 static inline u64 otx2_aura_allocptr(struct otx2_nic *pfvf, int aura)
742 {
743 u64 *ptr = (__force u64 *)otx2_get_regaddr(pfvf, NPA_LF_AURA_OP_ALLOCX(0));
744 u64 incr = (u64)aura | BIT_ULL(63);
745
746 return otx2_atomic64_add(incr, ptr);
747 }
748
749 /* Free pointer to a pool/aura */
otx2_aura_freeptr(void * dev,int aura,u64 buf)750 static inline void otx2_aura_freeptr(void *dev, int aura, u64 buf)
751 {
752 struct otx2_nic *pfvf = dev;
753 void __iomem *addr = otx2_get_regaddr(pfvf, NPA_LF_AURA_OP_FREE0);
754
755 otx2_write128(buf, (u64)aura | BIT_ULL(63), addr);
756 }
757
otx2_get_pool_idx(struct otx2_nic * pfvf,int type,int idx)758 static inline int otx2_get_pool_idx(struct otx2_nic *pfvf, int type, int idx)
759 {
760 if (type == AURA_NIX_SQ)
761 return pfvf->hw.rqpool_cnt + idx;
762
763 /* AURA_NIX_RQ */
764 return idx;
765 }
766
767 /* Mbox APIs */
otx2_sync_mbox_msg(struct mbox * mbox)768 static inline int otx2_sync_mbox_msg(struct mbox *mbox)
769 {
770 int err;
771
772 if (!otx2_mbox_nonempty(&mbox->mbox, 0))
773 return 0;
774 otx2_mbox_msg_send(&mbox->mbox, 0);
775 err = otx2_mbox_wait_for_rsp(&mbox->mbox, 0);
776 if (err)
777 return err;
778
779 return otx2_mbox_check_rsp_msgs(&mbox->mbox, 0);
780 }
781
otx2_sync_mbox_up_msg(struct mbox * mbox,int devid)782 static inline int otx2_sync_mbox_up_msg(struct mbox *mbox, int devid)
783 {
784 int err;
785
786 if (!otx2_mbox_nonempty(&mbox->mbox_up, devid))
787 return 0;
788 otx2_mbox_msg_send(&mbox->mbox_up, devid);
789 err = otx2_mbox_wait_for_rsp(&mbox->mbox_up, devid);
790 if (err)
791 return err;
792
793 return otx2_mbox_check_rsp_msgs(&mbox->mbox_up, devid);
794 }
795
796 /* Use this API to send mbox msgs in atomic context
797 * where sleeping is not allowed
798 */
otx2_sync_mbox_msg_busy_poll(struct mbox * mbox)799 static inline int otx2_sync_mbox_msg_busy_poll(struct mbox *mbox)
800 {
801 int err;
802
803 if (!otx2_mbox_nonempty(&mbox->mbox, 0))
804 return 0;
805 otx2_mbox_msg_send(&mbox->mbox, 0);
806 err = otx2_mbox_busy_poll_for_rsp(&mbox->mbox, 0);
807 if (err)
808 return err;
809
810 return otx2_mbox_check_rsp_msgs(&mbox->mbox, 0);
811 }
812
813 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \
814 static struct _req_type __maybe_unused \
815 *otx2_mbox_alloc_msg_ ## _fn_name(struct mbox *mbox) \
816 { \
817 struct _req_type *req; \
818 \
819 req = (struct _req_type *)otx2_mbox_alloc_msg_rsp( \
820 &mbox->mbox, 0, sizeof(struct _req_type), \
821 sizeof(struct _rsp_type)); \
822 if (!req) \
823 return NULL; \
824 req->hdr.sig = OTX2_MBOX_REQ_SIG; \
825 req->hdr.id = _id; \
826 trace_otx2_msg_alloc(mbox->mbox.pdev, _id, sizeof(*req)); \
827 return req; \
828 }
829
830 MBOX_MESSAGES
831 #undef M
832
833 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \
834 int \
835 otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf, \
836 struct _req_type *req, \
837 struct _rsp_type *rsp); \
838
839 MBOX_UP_CGX_MESSAGES
840 MBOX_UP_MCS_MESSAGES
841 #undef M
842
843 /* Time to wait before watchdog kicks off */
844 #define OTX2_TX_TIMEOUT (100 * HZ)
845
846 #define RVU_PFVF_PF_SHIFT 10
847 #define RVU_PFVF_PF_MASK 0x3F
848 #define RVU_PFVF_FUNC_SHIFT 0
849 #define RVU_PFVF_FUNC_MASK 0x3FF
850
is_otx2_vf(u16 pcifunc)851 static inline bool is_otx2_vf(u16 pcifunc)
852 {
853 return !!(pcifunc & RVU_PFVF_FUNC_MASK);
854 }
855
rvu_get_pf(u16 pcifunc)856 static inline int rvu_get_pf(u16 pcifunc)
857 {
858 return (pcifunc >> RVU_PFVF_PF_SHIFT) & RVU_PFVF_PF_MASK;
859 }
860
otx2_dma_map_page(struct otx2_nic * pfvf,struct page * page,size_t offset,size_t size,enum dma_data_direction dir)861 static inline dma_addr_t otx2_dma_map_page(struct otx2_nic *pfvf,
862 struct page *page,
863 size_t offset, size_t size,
864 enum dma_data_direction dir)
865 {
866 dma_addr_t iova;
867
868 iova = dma_map_page_attrs(pfvf->dev, page,
869 offset, size, dir, DMA_ATTR_SKIP_CPU_SYNC);
870 if (unlikely(dma_mapping_error(pfvf->dev, iova)))
871 return (dma_addr_t)NULL;
872 return iova;
873 }
874
otx2_dma_unmap_page(struct otx2_nic * pfvf,dma_addr_t addr,size_t size,enum dma_data_direction dir)875 static inline void otx2_dma_unmap_page(struct otx2_nic *pfvf,
876 dma_addr_t addr, size_t size,
877 enum dma_data_direction dir)
878 {
879 dma_unmap_page_attrs(pfvf->dev, addr, size,
880 dir, DMA_ATTR_SKIP_CPU_SYNC);
881 }
882
otx2_get_smq_idx(struct otx2_nic * pfvf,u16 qidx)883 static inline u16 otx2_get_smq_idx(struct otx2_nic *pfvf, u16 qidx)
884 {
885 u16 smq;
886 #ifdef CONFIG_DCB
887 if (qidx < NIX_PF_PFC_PRIO_MAX && pfvf->pfc_alloc_status[qidx])
888 return pfvf->pfc_schq_list[NIX_TXSCH_LVL_SMQ][qidx];
889 #endif
890 /* check if qidx falls under QOS queues */
891 if (qidx >= pfvf->hw.non_qos_queues)
892 smq = pfvf->qos.qid_to_sqmap[qidx - pfvf->hw.non_qos_queues];
893 else
894 smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0];
895
896 return smq;
897 }
898
otx2_get_total_tx_queues(struct otx2_nic * pfvf)899 static inline u16 otx2_get_total_tx_queues(struct otx2_nic *pfvf)
900 {
901 return pfvf->hw.non_qos_queues + pfvf->hw.tc_tx_queues;
902 }
903
904 /* MSI-X APIs */
905 void otx2_free_cints(struct otx2_nic *pfvf, int n);
906 void otx2_set_cints_affinity(struct otx2_nic *pfvf);
907 int otx2_set_mac_address(struct net_device *netdev, void *p);
908 int otx2_hw_set_mtu(struct otx2_nic *pfvf, int mtu);
909 void otx2_tx_timeout(struct net_device *netdev, unsigned int txq);
910 void otx2_get_mac_from_af(struct net_device *netdev);
911 void otx2_config_irq_coalescing(struct otx2_nic *pfvf, int qidx);
912 int otx2_config_pause_frm(struct otx2_nic *pfvf);
913 void otx2_setup_segmentation(struct otx2_nic *pfvf);
914
915 /* RVU block related APIs */
916 int otx2_attach_npa_nix(struct otx2_nic *pfvf);
917 int otx2_detach_resources(struct mbox *mbox);
918 int otx2_config_npa(struct otx2_nic *pfvf);
919 int otx2_sq_aura_pool_init(struct otx2_nic *pfvf);
920 int otx2_rq_aura_pool_init(struct otx2_nic *pfvf);
921 void otx2_aura_pool_free(struct otx2_nic *pfvf);
922 void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type);
923 void otx2_sq_free_sqbs(struct otx2_nic *pfvf);
924 int otx2_config_nix(struct otx2_nic *pfvf);
925 int otx2_config_nix_queues(struct otx2_nic *pfvf);
926 int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool pfc_en);
927 int otx2_txsch_alloc(struct otx2_nic *pfvf);
928 void otx2_txschq_stop(struct otx2_nic *pfvf);
929 void otx2_txschq_free_one(struct otx2_nic *pfvf, u16 lvl, u16 schq);
930 void otx2_free_pending_sqe(struct otx2_nic *pfvf);
931 void otx2_sqb_flush(struct otx2_nic *pfvf);
932 int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
933 dma_addr_t *dma);
934 int otx2_rxtx_enable(struct otx2_nic *pfvf, bool enable);
935 void otx2_ctx_disable(struct mbox *mbox, int type, bool npa);
936 int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable);
937 void otx2_cleanup_rx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq);
938 void otx2_cleanup_tx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq);
939 int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura);
940 int otx2_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura);
941 int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura);
942 int otx2_alloc_buffer(struct otx2_nic *pfvf, struct otx2_cq_queue *cq,
943 dma_addr_t *dma);
944 int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id,
945 int stack_pages, int numptrs, int buf_size);
946 int otx2_aura_init(struct otx2_nic *pfvf, int aura_id,
947 int pool_id, int numptrs);
948
949 /* RSS configuration APIs*/
950 int otx2_rss_init(struct otx2_nic *pfvf);
951 int otx2_set_flowkey_cfg(struct otx2_nic *pfvf);
952 void otx2_set_rss_key(struct otx2_nic *pfvf);
953 int otx2_set_rss_table(struct otx2_nic *pfvf, int ctx_id);
954
955 /* Mbox handlers */
956 void mbox_handler_msix_offset(struct otx2_nic *pfvf,
957 struct msix_offset_rsp *rsp);
958 void mbox_handler_npa_lf_alloc(struct otx2_nic *pfvf,
959 struct npa_lf_alloc_rsp *rsp);
960 void mbox_handler_nix_lf_alloc(struct otx2_nic *pfvf,
961 struct nix_lf_alloc_rsp *rsp);
962 void mbox_handler_nix_txsch_alloc(struct otx2_nic *pf,
963 struct nix_txsch_alloc_rsp *rsp);
964 void mbox_handler_cgx_stats(struct otx2_nic *pfvf,
965 struct cgx_stats_rsp *rsp);
966 void mbox_handler_cgx_fec_stats(struct otx2_nic *pfvf,
967 struct cgx_fec_stats_rsp *rsp);
968 void otx2_set_fec_stats_count(struct otx2_nic *pfvf);
969 void mbox_handler_nix_bp_enable(struct otx2_nic *pfvf,
970 struct nix_bp_cfg_rsp *rsp);
971
972 /* Device stats APIs */
973 void otx2_get_dev_stats(struct otx2_nic *pfvf);
974 void otx2_get_stats64(struct net_device *netdev,
975 struct rtnl_link_stats64 *stats);
976 void otx2_update_lmac_stats(struct otx2_nic *pfvf);
977 void otx2_update_lmac_fec_stats(struct otx2_nic *pfvf);
978 int otx2_update_rq_stats(struct otx2_nic *pfvf, int qidx);
979 int otx2_update_sq_stats(struct otx2_nic *pfvf, int qidx);
980 void otx2_set_ethtool_ops(struct net_device *netdev);
981 void otx2vf_set_ethtool_ops(struct net_device *netdev);
982
983 int otx2_open(struct net_device *netdev);
984 int otx2_stop(struct net_device *netdev);
985 int otx2_set_real_num_queues(struct net_device *netdev,
986 int tx_queues, int rx_queues);
987 int otx2_ioctl(struct net_device *netdev, struct ifreq *req, int cmd);
988 int otx2_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr);
989
990 /* MCAM filter related APIs */
991 int otx2_mcam_flow_init(struct otx2_nic *pf);
992 int otx2vf_mcam_flow_init(struct otx2_nic *pfvf);
993 int otx2_alloc_mcam_entries(struct otx2_nic *pfvf, u16 count);
994 void otx2_mcam_flow_del(struct otx2_nic *pf);
995 int otx2_destroy_ntuple_flows(struct otx2_nic *pf);
996 int otx2_destroy_mcam_flows(struct otx2_nic *pfvf);
997 int otx2_get_flow(struct otx2_nic *pfvf,
998 struct ethtool_rxnfc *nfc, u32 location);
999 int otx2_get_all_flows(struct otx2_nic *pfvf,
1000 struct ethtool_rxnfc *nfc, u32 *rule_locs);
1001 int otx2_add_flow(struct otx2_nic *pfvf,
1002 struct ethtool_rxnfc *nfc);
1003 int otx2_remove_flow(struct otx2_nic *pfvf, u32 location);
1004 int otx2_get_maxflows(struct otx2_flow_config *flow_cfg);
1005 void otx2_rss_ctx_flow_del(struct otx2_nic *pfvf, int ctx_id);
1006 int otx2_del_macfilter(struct net_device *netdev, const u8 *mac);
1007 int otx2_add_macfilter(struct net_device *netdev, const u8 *mac);
1008 int otx2_enable_rxvlan(struct otx2_nic *pf, bool enable);
1009 int otx2_install_rxvlan_offload_flow(struct otx2_nic *pfvf);
1010 bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx);
1011 u16 otx2_get_max_mtu(struct otx2_nic *pfvf);
1012 int otx2_handle_ntuple_tc_features(struct net_device *netdev,
1013 netdev_features_t features);
1014 int otx2_smq_flush(struct otx2_nic *pfvf, int smq);
1015
1016 /* tc support */
1017 int otx2_init_tc(struct otx2_nic *nic);
1018 void otx2_shutdown_tc(struct otx2_nic *nic);
1019 int otx2_setup_tc(struct net_device *netdev, enum tc_setup_type type,
1020 void *type_data);
1021 void otx2_tc_apply_ingress_police_rules(struct otx2_nic *nic);
1022
1023 /* CGX/RPM DMAC filters support */
1024 int otx2_dmacflt_get_max_cnt(struct otx2_nic *pf);
1025 int otx2_dmacflt_add(struct otx2_nic *pf, const u8 *mac, u32 bit_pos);
1026 int otx2_dmacflt_remove(struct otx2_nic *pf, const u8 *mac, u32 bit_pos);
1027 int otx2_dmacflt_update(struct otx2_nic *pf, u8 *mac, u32 bit_pos);
1028 void otx2_dmacflt_reinstall_flows(struct otx2_nic *pf);
1029 void otx2_dmacflt_update_pfmac_flow(struct otx2_nic *pfvf);
1030
1031 #ifdef CONFIG_DCB
1032 /* DCB support*/
1033 void otx2_update_bpid_in_rqctx(struct otx2_nic *pfvf, int vlan_prio, int qidx, bool pfc_enable);
1034 int otx2_config_priority_flow_ctrl(struct otx2_nic *pfvf);
1035 int otx2_dcbnl_set_ops(struct net_device *dev);
1036 /* PFC support */
1037 int otx2_pfc_txschq_config(struct otx2_nic *pfvf);
1038 int otx2_pfc_txschq_alloc(struct otx2_nic *pfvf);
1039 int otx2_pfc_txschq_update(struct otx2_nic *pfvf);
1040 int otx2_pfc_txschq_stop(struct otx2_nic *pfvf);
1041 #endif
1042
1043 #if IS_ENABLED(CONFIG_MACSEC)
1044 /* MACSEC offload support */
1045 int cn10k_mcs_init(struct otx2_nic *pfvf);
1046 void cn10k_mcs_free(struct otx2_nic *pfvf);
1047 void cn10k_handle_mcs_event(struct otx2_nic *pfvf, struct mcs_intr_info *event);
1048 #else
cn10k_mcs_init(struct otx2_nic * pfvf)1049 static inline int cn10k_mcs_init(struct otx2_nic *pfvf) { return 0; }
cn10k_mcs_free(struct otx2_nic * pfvf)1050 static inline void cn10k_mcs_free(struct otx2_nic *pfvf) {}
cn10k_handle_mcs_event(struct otx2_nic * pfvf,struct mcs_intr_info * event)1051 static inline void cn10k_handle_mcs_event(struct otx2_nic *pfvf,
1052 struct mcs_intr_info *event)
1053 {}
1054 #endif /* CONFIG_MACSEC */
1055
1056 /* qos support */
otx2_qos_init(struct otx2_nic * pfvf,int qos_txqs)1057 static inline void otx2_qos_init(struct otx2_nic *pfvf, int qos_txqs)
1058 {
1059 struct otx2_hw *hw = &pfvf->hw;
1060
1061 hw->tc_tx_queues = qos_txqs;
1062 }
1063
1064 u16 otx2_select_queue(struct net_device *netdev, struct sk_buff *skb,
1065 struct net_device *sb_dev);
1066 #endif /* OTX2_COMMON_H */
1067