1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2 /*
3 * Copyright(c) 2015 - 2020 Intel Corporation.
4 */
5
6 /*
7 * This file contains all of the code that is specific to the HFI chip
8 */
9
10 #include <linux/pci.h>
11 #include <linux/delay.h>
12 #include <linux/interrupt.h>
13 #include <linux/module.h>
14
15 #include "hfi.h"
16 #include "trace.h"
17 #include "mad.h"
18 #include "pio.h"
19 #include "sdma.h"
20 #include "eprom.h"
21 #include "efivar.h"
22 #include "platform.h"
23 #include "aspm.h"
24 #include "affinity.h"
25 #include "debugfs.h"
26 #include "fault.h"
27 #include "netdev.h"
28
29 uint num_vls = HFI1_MAX_VLS_SUPPORTED;
30 module_param(num_vls, uint, S_IRUGO);
31 MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)");
32
33 /*
34 * Default time to aggregate two 10K packets from the idle state
35 * (timer not running). The timer starts at the end of the first packet,
36 * so only the time for one 10K packet and header plus a bit extra is needed.
37 * 10 * 1024 + 64 header byte = 10304 byte
38 * 10304 byte / 12.5 GB/s = 824.32ns
39 */
40 uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */
41 module_param(rcv_intr_timeout, uint, S_IRUGO);
42 MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns");
43
44 uint rcv_intr_count = 16; /* same as qib */
45 module_param(rcv_intr_count, uint, S_IRUGO);
46 MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count");
47
48 ushort link_crc_mask = SUPPORTED_CRCS;
49 module_param(link_crc_mask, ushort, S_IRUGO);
50 MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link");
51
52 uint loopback;
53 module_param_named(loopback, loopback, uint, S_IRUGO);
54 MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable");
55
56 /* Other driver tunables */
57 uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/
58 static ushort crc_14b_sideband = 1;
59 static uint use_flr = 1;
60 uint quick_linkup; /* skip LNI */
61
62 struct flag_table {
63 u64 flag; /* the flag */
64 char *str; /* description string */
65 u16 extra; /* extra information */
66 u16 unused0;
67 u32 unused1;
68 };
69
70 /* str must be a string constant */
71 #define FLAG_ENTRY(str, extra, flag) {flag, str, extra}
72 #define FLAG_ENTRY0(str, flag) {flag, str, 0}
73
74 /* Send Error Consequences */
75 #define SEC_WRITE_DROPPED 0x1
76 #define SEC_PACKET_DROPPED 0x2
77 #define SEC_SC_HALTED 0x4 /* per-context only */
78 #define SEC_SPC_FREEZE 0x8 /* per-HFI only */
79
80 #define DEFAULT_KRCVQS 2
81 #define MIN_KERNEL_KCTXTS 2
82 #define FIRST_KERNEL_KCTXT 1
83
84 /*
85 * RSM instance allocation
86 * 0 - User Fecn Handling
87 * 1 - Vnic
88 * 2 - AIP
89 * 3 - Verbs
90 */
91 #define RSM_INS_FECN 0
92 #define RSM_INS_VNIC 1
93 #define RSM_INS_AIP 2
94 #define RSM_INS_VERBS 3
95
96 /* Bit offset into the GUID which carries HFI id information */
97 #define GUID_HFI_INDEX_SHIFT 39
98
99 /* extract the emulation revision */
100 #define emulator_rev(dd) ((dd)->irev >> 8)
101 /* parallel and serial emulation versions are 3 and 4 respectively */
102 #define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3)
103 #define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4)
104
105 /* RSM fields for Verbs */
106 /* packet type */
107 #define IB_PACKET_TYPE 2ull
108 #define QW_SHIFT 6ull
109 /* QPN[7..1] */
110 #define QPN_WIDTH 7ull
111
112 /* LRH.BTH: QW 0, OFFSET 48 - for match */
113 #define LRH_BTH_QW 0ull
114 #define LRH_BTH_BIT_OFFSET 48ull
115 #define LRH_BTH_OFFSET(off) ((LRH_BTH_QW << QW_SHIFT) | (off))
116 #define LRH_BTH_MATCH_OFFSET LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET)
117 #define LRH_BTH_SELECT
118 #define LRH_BTH_MASK 3ull
119 #define LRH_BTH_VALUE 2ull
120
121 /* LRH.SC[3..0] QW 0, OFFSET 56 - for match */
122 #define LRH_SC_QW 0ull
123 #define LRH_SC_BIT_OFFSET 56ull
124 #define LRH_SC_OFFSET(off) ((LRH_SC_QW << QW_SHIFT) | (off))
125 #define LRH_SC_MATCH_OFFSET LRH_SC_OFFSET(LRH_SC_BIT_OFFSET)
126 #define LRH_SC_MASK 128ull
127 #define LRH_SC_VALUE 0ull
128
129 /* SC[n..0] QW 0, OFFSET 60 - for select */
130 #define LRH_SC_SELECT_OFFSET ((LRH_SC_QW << QW_SHIFT) | (60ull))
131
132 /* QPN[m+n:1] QW 1, OFFSET 1 */
133 #define QPN_SELECT_OFFSET ((1ull << QW_SHIFT) | (1ull))
134
135 /* RSM fields for AIP */
136 /* LRH.BTH above is reused for this rule */
137
138 /* BTH.DESTQP: QW 1, OFFSET 16 for match */
139 #define BTH_DESTQP_QW 1ull
140 #define BTH_DESTQP_BIT_OFFSET 16ull
141 #define BTH_DESTQP_OFFSET(off) ((BTH_DESTQP_QW << QW_SHIFT) | (off))
142 #define BTH_DESTQP_MATCH_OFFSET BTH_DESTQP_OFFSET(BTH_DESTQP_BIT_OFFSET)
143 #define BTH_DESTQP_MASK 0xFFull
144 #define BTH_DESTQP_VALUE 0x81ull
145
146 /* DETH.SQPN: QW 1 Offset 56 for select */
147 /* We use 8 most significant Soure QPN bits as entropy fpr AIP */
148 #define DETH_AIP_SQPN_QW 3ull
149 #define DETH_AIP_SQPN_BIT_OFFSET 56ull
150 #define DETH_AIP_SQPN_OFFSET(off) ((DETH_AIP_SQPN_QW << QW_SHIFT) | (off))
151 #define DETH_AIP_SQPN_SELECT_OFFSET \
152 DETH_AIP_SQPN_OFFSET(DETH_AIP_SQPN_BIT_OFFSET)
153
154 /* RSM fields for Vnic */
155 /* L2_TYPE: QW 0, OFFSET 61 - for match */
156 #define L2_TYPE_QW 0ull
157 #define L2_TYPE_BIT_OFFSET 61ull
158 #define L2_TYPE_OFFSET(off) ((L2_TYPE_QW << QW_SHIFT) | (off))
159 #define L2_TYPE_MATCH_OFFSET L2_TYPE_OFFSET(L2_TYPE_BIT_OFFSET)
160 #define L2_TYPE_MASK 3ull
161 #define L2_16B_VALUE 2ull
162
163 /* L4_TYPE QW 1, OFFSET 0 - for match */
164 #define L4_TYPE_QW 1ull
165 #define L4_TYPE_BIT_OFFSET 0ull
166 #define L4_TYPE_OFFSET(off) ((L4_TYPE_QW << QW_SHIFT) | (off))
167 #define L4_TYPE_MATCH_OFFSET L4_TYPE_OFFSET(L4_TYPE_BIT_OFFSET)
168 #define L4_16B_TYPE_MASK 0xFFull
169 #define L4_16B_ETH_VALUE 0x78ull
170
171 /* 16B VESWID - for select */
172 #define L4_16B_HDR_VESWID_OFFSET ((2 << QW_SHIFT) | (16ull))
173 /* 16B ENTROPY - for select */
174 #define L2_16B_ENTROPY_OFFSET ((1 << QW_SHIFT) | (32ull))
175
176 /* defines to build power on SC2VL table */
177 #define SC2VL_VAL( \
178 num, \
179 sc0, sc0val, \
180 sc1, sc1val, \
181 sc2, sc2val, \
182 sc3, sc3val, \
183 sc4, sc4val, \
184 sc5, sc5val, \
185 sc6, sc6val, \
186 sc7, sc7val) \
187 ( \
188 ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \
189 ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \
190 ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \
191 ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \
192 ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \
193 ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \
194 ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \
195 ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT) \
196 )
197
198 #define DC_SC_VL_VAL( \
199 range, \
200 e0, e0val, \
201 e1, e1val, \
202 e2, e2val, \
203 e3, e3val, \
204 e4, e4val, \
205 e5, e5val, \
206 e6, e6val, \
207 e7, e7val, \
208 e8, e8val, \
209 e9, e9val, \
210 e10, e10val, \
211 e11, e11val, \
212 e12, e12val, \
213 e13, e13val, \
214 e14, e14val, \
215 e15, e15val) \
216 ( \
217 ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \
218 ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \
219 ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \
220 ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \
221 ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \
222 ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \
223 ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \
224 ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \
225 ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \
226 ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \
227 ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \
228 ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \
229 ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \
230 ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \
231 ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \
232 ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \
233 )
234
235 /* all CceStatus sub-block freeze bits */
236 #define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \
237 | CCE_STATUS_RXE_FROZE_SMASK \
238 | CCE_STATUS_TXE_FROZE_SMASK \
239 | CCE_STATUS_TXE_PIO_FROZE_SMASK)
240 /* all CceStatus sub-block TXE pause bits */
241 #define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \
242 | CCE_STATUS_TXE_PAUSED_SMASK \
243 | CCE_STATUS_SDMA_PAUSED_SMASK)
244 /* all CceStatus sub-block RXE pause bits */
245 #define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK
246
247 #define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
248 #define CNTR_32BIT_MAX 0x00000000FFFFFFFF
249
250 /*
251 * CCE Error flags.
252 */
253 static struct flag_table cce_err_status_flags[] = {
254 /* 0*/ FLAG_ENTRY0("CceCsrParityErr",
255 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK),
256 /* 1*/ FLAG_ENTRY0("CceCsrReadBadAddrErr",
257 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK),
258 /* 2*/ FLAG_ENTRY0("CceCsrWriteBadAddrErr",
259 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK),
260 /* 3*/ FLAG_ENTRY0("CceTrgtAsyncFifoParityErr",
261 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK),
262 /* 4*/ FLAG_ENTRY0("CceTrgtAccessErr",
263 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK),
264 /* 5*/ FLAG_ENTRY0("CceRspdDataParityErr",
265 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK),
266 /* 6*/ FLAG_ENTRY0("CceCli0AsyncFifoParityErr",
267 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK),
268 /* 7*/ FLAG_ENTRY0("CceCsrCfgBusParityErr",
269 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK),
270 /* 8*/ FLAG_ENTRY0("CceCli2AsyncFifoParityErr",
271 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK),
272 /* 9*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
273 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK),
274 /*10*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
275 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK),
276 /*11*/ FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError",
277 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK),
278 /*12*/ FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError",
279 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK),
280 /*13*/ FLAG_ENTRY0("PcicRetryMemCorErr",
281 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK),
282 /*14*/ FLAG_ENTRY0("PcicRetryMemCorErr",
283 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK),
284 /*15*/ FLAG_ENTRY0("PcicPostHdQCorErr",
285 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK),
286 /*16*/ FLAG_ENTRY0("PcicPostHdQCorErr",
287 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK),
288 /*17*/ FLAG_ENTRY0("PcicPostHdQCorErr",
289 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK),
290 /*18*/ FLAG_ENTRY0("PcicCplDatQCorErr",
291 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK),
292 /*19*/ FLAG_ENTRY0("PcicNPostHQParityErr",
293 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK),
294 /*20*/ FLAG_ENTRY0("PcicNPostDatQParityErr",
295 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK),
296 /*21*/ FLAG_ENTRY0("PcicRetryMemUncErr",
297 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK),
298 /*22*/ FLAG_ENTRY0("PcicRetrySotMemUncErr",
299 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK),
300 /*23*/ FLAG_ENTRY0("PcicPostHdQUncErr",
301 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK),
302 /*24*/ FLAG_ENTRY0("PcicPostDatQUncErr",
303 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK),
304 /*25*/ FLAG_ENTRY0("PcicCplHdQUncErr",
305 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK),
306 /*26*/ FLAG_ENTRY0("PcicCplDatQUncErr",
307 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK),
308 /*27*/ FLAG_ENTRY0("PcicTransmitFrontParityErr",
309 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK),
310 /*28*/ FLAG_ENTRY0("PcicTransmitBackParityErr",
311 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK),
312 /*29*/ FLAG_ENTRY0("PcicReceiveParityErr",
313 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK),
314 /*30*/ FLAG_ENTRY0("CceTrgtCplTimeoutErr",
315 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK),
316 /*31*/ FLAG_ENTRY0("LATriggered",
317 CCE_ERR_STATUS_LA_TRIGGERED_SMASK),
318 /*32*/ FLAG_ENTRY0("CceSegReadBadAddrErr",
319 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK),
320 /*33*/ FLAG_ENTRY0("CceSegWriteBadAddrErr",
321 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK),
322 /*34*/ FLAG_ENTRY0("CceRcplAsyncFifoParityErr",
323 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK),
324 /*35*/ FLAG_ENTRY0("CceRxdmaConvFifoParityErr",
325 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK),
326 /*36*/ FLAG_ENTRY0("CceMsixTableCorErr",
327 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK),
328 /*37*/ FLAG_ENTRY0("CceMsixTableUncErr",
329 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK),
330 /*38*/ FLAG_ENTRY0("CceIntMapCorErr",
331 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK),
332 /*39*/ FLAG_ENTRY0("CceIntMapUncErr",
333 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK),
334 /*40*/ FLAG_ENTRY0("CceMsixCsrParityErr",
335 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK),
336 /*41-63 reserved*/
337 };
338
339 /*
340 * Misc Error flags
341 */
342 #define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK
343 static struct flag_table misc_err_status_flags[] = {
344 /* 0*/ FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)),
345 /* 1*/ FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)),
346 /* 2*/ FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)),
347 /* 3*/ FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)),
348 /* 4*/ FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)),
349 /* 5*/ FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)),
350 /* 6*/ FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)),
351 /* 7*/ FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)),
352 /* 8*/ FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)),
353 /* 9*/ FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)),
354 /*10*/ FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)),
355 /*11*/ FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)),
356 /*12*/ FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL))
357 };
358
359 /*
360 * TXE PIO Error flags and consequences
361 */
362 static struct flag_table pio_err_status_flags[] = {
363 /* 0*/ FLAG_ENTRY("PioWriteBadCtxt",
364 SEC_WRITE_DROPPED,
365 SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK),
366 /* 1*/ FLAG_ENTRY("PioWriteAddrParity",
367 SEC_SPC_FREEZE,
368 SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK),
369 /* 2*/ FLAG_ENTRY("PioCsrParity",
370 SEC_SPC_FREEZE,
371 SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK),
372 /* 3*/ FLAG_ENTRY("PioSbMemFifo0",
373 SEC_SPC_FREEZE,
374 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK),
375 /* 4*/ FLAG_ENTRY("PioSbMemFifo1",
376 SEC_SPC_FREEZE,
377 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK),
378 /* 5*/ FLAG_ENTRY("PioPccFifoParity",
379 SEC_SPC_FREEZE,
380 SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK),
381 /* 6*/ FLAG_ENTRY("PioPecFifoParity",
382 SEC_SPC_FREEZE,
383 SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK),
384 /* 7*/ FLAG_ENTRY("PioSbrdctlCrrelParity",
385 SEC_SPC_FREEZE,
386 SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK),
387 /* 8*/ FLAG_ENTRY("PioSbrdctrlCrrelFifoParity",
388 SEC_SPC_FREEZE,
389 SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK),
390 /* 9*/ FLAG_ENTRY("PioPktEvictFifoParityErr",
391 SEC_SPC_FREEZE,
392 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK),
393 /*10*/ FLAG_ENTRY("PioSmPktResetParity",
394 SEC_SPC_FREEZE,
395 SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK),
396 /*11*/ FLAG_ENTRY("PioVlLenMemBank0Unc",
397 SEC_SPC_FREEZE,
398 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK),
399 /*12*/ FLAG_ENTRY("PioVlLenMemBank1Unc",
400 SEC_SPC_FREEZE,
401 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK),
402 /*13*/ FLAG_ENTRY("PioVlLenMemBank0Cor",
403 0,
404 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK),
405 /*14*/ FLAG_ENTRY("PioVlLenMemBank1Cor",
406 0,
407 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK),
408 /*15*/ FLAG_ENTRY("PioCreditRetFifoParity",
409 SEC_SPC_FREEZE,
410 SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK),
411 /*16*/ FLAG_ENTRY("PioPpmcPblFifo",
412 SEC_SPC_FREEZE,
413 SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK),
414 /*17*/ FLAG_ENTRY("PioInitSmIn",
415 0,
416 SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK),
417 /*18*/ FLAG_ENTRY("PioPktEvictSmOrArbSm",
418 SEC_SPC_FREEZE,
419 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK),
420 /*19*/ FLAG_ENTRY("PioHostAddrMemUnc",
421 SEC_SPC_FREEZE,
422 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK),
423 /*20*/ FLAG_ENTRY("PioHostAddrMemCor",
424 0,
425 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK),
426 /*21*/ FLAG_ENTRY("PioWriteDataParity",
427 SEC_SPC_FREEZE,
428 SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK),
429 /*22*/ FLAG_ENTRY("PioStateMachine",
430 SEC_SPC_FREEZE,
431 SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK),
432 /*23*/ FLAG_ENTRY("PioWriteQwValidParity",
433 SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
434 SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK),
435 /*24*/ FLAG_ENTRY("PioBlockQwCountParity",
436 SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
437 SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK),
438 /*25*/ FLAG_ENTRY("PioVlfVlLenParity",
439 SEC_SPC_FREEZE,
440 SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK),
441 /*26*/ FLAG_ENTRY("PioVlfSopParity",
442 SEC_SPC_FREEZE,
443 SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK),
444 /*27*/ FLAG_ENTRY("PioVlFifoParity",
445 SEC_SPC_FREEZE,
446 SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK),
447 /*28*/ FLAG_ENTRY("PioPpmcBqcMemParity",
448 SEC_SPC_FREEZE,
449 SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK),
450 /*29*/ FLAG_ENTRY("PioPpmcSopLen",
451 SEC_SPC_FREEZE,
452 SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK),
453 /*30-31 reserved*/
454 /*32*/ FLAG_ENTRY("PioCurrentFreeCntParity",
455 SEC_SPC_FREEZE,
456 SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK),
457 /*33*/ FLAG_ENTRY("PioLastReturnedCntParity",
458 SEC_SPC_FREEZE,
459 SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK),
460 /*34*/ FLAG_ENTRY("PioPccSopHeadParity",
461 SEC_SPC_FREEZE,
462 SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK),
463 /*35*/ FLAG_ENTRY("PioPecSopHeadParityErr",
464 SEC_SPC_FREEZE,
465 SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK),
466 /*36-63 reserved*/
467 };
468
469 /* TXE PIO errors that cause an SPC freeze */
470 #define ALL_PIO_FREEZE_ERR \
471 (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \
472 | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \
473 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \
474 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \
475 | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \
476 | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \
477 | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \
478 | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \
479 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \
480 | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \
481 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \
482 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \
483 | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \
484 | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \
485 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \
486 | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \
487 | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \
488 | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \
489 | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \
490 | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \
491 | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \
492 | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \
493 | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \
494 | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \
495 | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \
496 | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \
497 | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \
498 | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \
499 | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK)
500
501 /*
502 * TXE SDMA Error flags
503 */
504 static struct flag_table sdma_err_status_flags[] = {
505 /* 0*/ FLAG_ENTRY0("SDmaRpyTagErr",
506 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK),
507 /* 1*/ FLAG_ENTRY0("SDmaCsrParityErr",
508 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK),
509 /* 2*/ FLAG_ENTRY0("SDmaPcieReqTrackingUncErr",
510 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK),
511 /* 3*/ FLAG_ENTRY0("SDmaPcieReqTrackingCorErr",
512 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK),
513 /*04-63 reserved*/
514 };
515
516 /* TXE SDMA errors that cause an SPC freeze */
517 #define ALL_SDMA_FREEZE_ERR \
518 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \
519 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \
520 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK)
521
522 /* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */
523 #define PORT_DISCARD_EGRESS_ERRS \
524 (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \
525 | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \
526 | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK)
527
528 /*
529 * TXE Egress Error flags
530 */
531 #define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK
532 static struct flag_table egress_err_status_flags[] = {
533 /* 0*/ FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)),
534 /* 1*/ FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)),
535 /* 2 reserved */
536 /* 3*/ FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr",
537 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)),
538 /* 4*/ FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)),
539 /* 5*/ FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)),
540 /* 6 reserved */
541 /* 7*/ FLAG_ENTRY0("TxPioLaunchIntfParityErr",
542 SEES(TX_PIO_LAUNCH_INTF_PARITY)),
543 /* 8*/ FLAG_ENTRY0("TxSdmaLaunchIntfParityErr",
544 SEES(TX_SDMA_LAUNCH_INTF_PARITY)),
545 /* 9-10 reserved */
546 /*11*/ FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr",
547 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)),
548 /*12*/ FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)),
549 /*13*/ FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)),
550 /*14*/ FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)),
551 /*15*/ FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)),
552 /*16*/ FLAG_ENTRY0("TxSdma0DisallowedPacketErr",
553 SEES(TX_SDMA0_DISALLOWED_PACKET)),
554 /*17*/ FLAG_ENTRY0("TxSdma1DisallowedPacketErr",
555 SEES(TX_SDMA1_DISALLOWED_PACKET)),
556 /*18*/ FLAG_ENTRY0("TxSdma2DisallowedPacketErr",
557 SEES(TX_SDMA2_DISALLOWED_PACKET)),
558 /*19*/ FLAG_ENTRY0("TxSdma3DisallowedPacketErr",
559 SEES(TX_SDMA3_DISALLOWED_PACKET)),
560 /*20*/ FLAG_ENTRY0("TxSdma4DisallowedPacketErr",
561 SEES(TX_SDMA4_DISALLOWED_PACKET)),
562 /*21*/ FLAG_ENTRY0("TxSdma5DisallowedPacketErr",
563 SEES(TX_SDMA5_DISALLOWED_PACKET)),
564 /*22*/ FLAG_ENTRY0("TxSdma6DisallowedPacketErr",
565 SEES(TX_SDMA6_DISALLOWED_PACKET)),
566 /*23*/ FLAG_ENTRY0("TxSdma7DisallowedPacketErr",
567 SEES(TX_SDMA7_DISALLOWED_PACKET)),
568 /*24*/ FLAG_ENTRY0("TxSdma8DisallowedPacketErr",
569 SEES(TX_SDMA8_DISALLOWED_PACKET)),
570 /*25*/ FLAG_ENTRY0("TxSdma9DisallowedPacketErr",
571 SEES(TX_SDMA9_DISALLOWED_PACKET)),
572 /*26*/ FLAG_ENTRY0("TxSdma10DisallowedPacketErr",
573 SEES(TX_SDMA10_DISALLOWED_PACKET)),
574 /*27*/ FLAG_ENTRY0("TxSdma11DisallowedPacketErr",
575 SEES(TX_SDMA11_DISALLOWED_PACKET)),
576 /*28*/ FLAG_ENTRY0("TxSdma12DisallowedPacketErr",
577 SEES(TX_SDMA12_DISALLOWED_PACKET)),
578 /*29*/ FLAG_ENTRY0("TxSdma13DisallowedPacketErr",
579 SEES(TX_SDMA13_DISALLOWED_PACKET)),
580 /*30*/ FLAG_ENTRY0("TxSdma14DisallowedPacketErr",
581 SEES(TX_SDMA14_DISALLOWED_PACKET)),
582 /*31*/ FLAG_ENTRY0("TxSdma15DisallowedPacketErr",
583 SEES(TX_SDMA15_DISALLOWED_PACKET)),
584 /*32*/ FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr",
585 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)),
586 /*33*/ FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr",
587 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)),
588 /*34*/ FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr",
589 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)),
590 /*35*/ FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr",
591 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)),
592 /*36*/ FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr",
593 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)),
594 /*37*/ FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr",
595 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)),
596 /*38*/ FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr",
597 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)),
598 /*39*/ FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr",
599 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)),
600 /*40*/ FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr",
601 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)),
602 /*41*/ FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)),
603 /*42*/ FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)),
604 /*43*/ FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)),
605 /*44*/ FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)),
606 /*45*/ FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)),
607 /*46*/ FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)),
608 /*47*/ FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)),
609 /*48*/ FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)),
610 /*49*/ FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)),
611 /*50*/ FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)),
612 /*51*/ FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)),
613 /*52*/ FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)),
614 /*53*/ FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)),
615 /*54*/ FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)),
616 /*55*/ FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)),
617 /*56*/ FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)),
618 /*57*/ FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)),
619 /*58*/ FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)),
620 /*59*/ FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)),
621 /*60*/ FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)),
622 /*61*/ FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)),
623 /*62*/ FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr",
624 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)),
625 /*63*/ FLAG_ENTRY0("TxReadPioMemoryCsrUncErr",
626 SEES(TX_READ_PIO_MEMORY_CSR_UNC)),
627 };
628
629 /*
630 * TXE Egress Error Info flags
631 */
632 #define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK
633 static struct flag_table egress_err_info_flags[] = {
634 /* 0*/ FLAG_ENTRY0("Reserved", 0ull),
635 /* 1*/ FLAG_ENTRY0("VLErr", SEEI(VL)),
636 /* 2*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
637 /* 3*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
638 /* 4*/ FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)),
639 /* 5*/ FLAG_ENTRY0("SLIDErr", SEEI(SLID)),
640 /* 6*/ FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)),
641 /* 7*/ FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)),
642 /* 8*/ FLAG_ENTRY0("RawErr", SEEI(RAW)),
643 /* 9*/ FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)),
644 /*10*/ FLAG_ENTRY0("GRHErr", SEEI(GRH)),
645 /*11*/ FLAG_ENTRY0("BypassErr", SEEI(BYPASS)),
646 /*12*/ FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)),
647 /*13*/ FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)),
648 /*14*/ FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)),
649 /*15*/ FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)),
650 /*16*/ FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)),
651 /*17*/ FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)),
652 /*18*/ FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)),
653 /*19*/ FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)),
654 /*20*/ FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)),
655 /*21*/ FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)),
656 };
657
658 /* TXE Egress errors that cause an SPC freeze */
659 #define ALL_TXE_EGRESS_FREEZE_ERR \
660 (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \
661 | SEES(TX_PIO_LAUNCH_INTF_PARITY) \
662 | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \
663 | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \
664 | SEES(TX_LAUNCH_CSR_PARITY) \
665 | SEES(TX_SBRD_CTL_CSR_PARITY) \
666 | SEES(TX_CONFIG_PARITY) \
667 | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \
668 | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \
669 | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \
670 | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \
671 | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \
672 | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \
673 | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \
674 | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \
675 | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \
676 | SEES(TX_CREDIT_RETURN_PARITY))
677
678 /*
679 * TXE Send error flags
680 */
681 #define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK
682 static struct flag_table send_err_status_flags[] = {
683 /* 0*/ FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)),
684 /* 1*/ FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)),
685 /* 2*/ FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR))
686 };
687
688 /*
689 * TXE Send Context Error flags and consequences
690 */
691 static struct flag_table sc_err_status_flags[] = {
692 /* 0*/ FLAG_ENTRY("InconsistentSop",
693 SEC_PACKET_DROPPED | SEC_SC_HALTED,
694 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK),
695 /* 1*/ FLAG_ENTRY("DisallowedPacket",
696 SEC_PACKET_DROPPED | SEC_SC_HALTED,
697 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK),
698 /* 2*/ FLAG_ENTRY("WriteCrossesBoundary",
699 SEC_WRITE_DROPPED | SEC_SC_HALTED,
700 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK),
701 /* 3*/ FLAG_ENTRY("WriteOverflow",
702 SEC_WRITE_DROPPED | SEC_SC_HALTED,
703 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK),
704 /* 4*/ FLAG_ENTRY("WriteOutOfBounds",
705 SEC_WRITE_DROPPED | SEC_SC_HALTED,
706 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK),
707 /* 5-63 reserved*/
708 };
709
710 /*
711 * RXE Receive Error flags
712 */
713 #define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK
714 static struct flag_table rxe_err_status_flags[] = {
715 /* 0*/ FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)),
716 /* 1*/ FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)),
717 /* 2*/ FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)),
718 /* 3*/ FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)),
719 /* 4*/ FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)),
720 /* 5*/ FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)),
721 /* 6*/ FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)),
722 /* 7*/ FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)),
723 /* 8*/ FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)),
724 /* 9*/ FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)),
725 /*10*/ FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)),
726 /*11*/ FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)),
727 /*12*/ FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)),
728 /*13*/ FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)),
729 /*14*/ FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)),
730 /*15*/ FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)),
731 /*16*/ FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr",
732 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)),
733 /*17*/ FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)),
734 /*18*/ FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)),
735 /*19*/ FLAG_ENTRY0("RxRbufBlockListReadUncErr",
736 RXES(RBUF_BLOCK_LIST_READ_UNC)),
737 /*20*/ FLAG_ENTRY0("RxRbufBlockListReadCorErr",
738 RXES(RBUF_BLOCK_LIST_READ_COR)),
739 /*21*/ FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr",
740 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)),
741 /*22*/ FLAG_ENTRY0("RxRbufCsrQEntCntParityErr",
742 RXES(RBUF_CSR_QENT_CNT_PARITY)),
743 /*23*/ FLAG_ENTRY0("RxRbufCsrQNextBufParityErr",
744 RXES(RBUF_CSR_QNEXT_BUF_PARITY)),
745 /*24*/ FLAG_ENTRY0("RxRbufCsrQVldBitParityErr",
746 RXES(RBUF_CSR_QVLD_BIT_PARITY)),
747 /*25*/ FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)),
748 /*26*/ FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)),
749 /*27*/ FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr",
750 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)),
751 /*28*/ FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)),
752 /*29*/ FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)),
753 /*30*/ FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)),
754 /*31*/ FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)),
755 /*32*/ FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)),
756 /*33*/ FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)),
757 /*34*/ FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)),
758 /*35*/ FLAG_ENTRY0("RxRbufFlInitdoneParityErr",
759 RXES(RBUF_FL_INITDONE_PARITY)),
760 /*36*/ FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr",
761 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)),
762 /*37*/ FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)),
763 /*38*/ FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)),
764 /*39*/ FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)),
765 /*40*/ FLAG_ENTRY0("RxLookupDesPart1UncCorErr",
766 RXES(LOOKUP_DES_PART1_UNC_COR)),
767 /*41*/ FLAG_ENTRY0("RxLookupDesPart2ParityErr",
768 RXES(LOOKUP_DES_PART2_PARITY)),
769 /*42*/ FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)),
770 /*43*/ FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)),
771 /*44*/ FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)),
772 /*45*/ FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)),
773 /*46*/ FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)),
774 /*47*/ FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)),
775 /*48*/ FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)),
776 /*49*/ FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)),
777 /*50*/ FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)),
778 /*51*/ FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)),
779 /*52*/ FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)),
780 /*53*/ FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)),
781 /*54*/ FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)),
782 /*55*/ FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)),
783 /*56*/ FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)),
784 /*57*/ FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)),
785 /*58*/ FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)),
786 /*59*/ FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)),
787 /*60*/ FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)),
788 /*61*/ FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)),
789 /*62*/ FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)),
790 /*63*/ FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY))
791 };
792
793 /* RXE errors that will trigger an SPC freeze */
794 #define ALL_RXE_FREEZE_ERR \
795 (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \
796 | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \
797 | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \
798 | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \
799 | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \
800 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \
801 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \
802 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \
803 | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \
804 | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \
805 | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \
806 | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \
807 | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \
808 | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \
809 | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \
810 | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \
811 | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \
812 | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \
813 | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \
814 | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \
815 | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \
816 | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \
817 | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \
818 | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \
819 | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \
820 | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \
821 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \
822 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \
823 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \
824 | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \
825 | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \
826 | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \
827 | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \
828 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \
829 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \
830 | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \
831 | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \
832 | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \
833 | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \
834 | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \
835 | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \
836 | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \
837 | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \
838 | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK)
839
840 #define RXE_FREEZE_ABORT_MASK \
841 (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \
842 RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \
843 RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK)
844
845 /*
846 * DCC Error Flags
847 */
848 #define DCCE(name) DCC_ERR_FLG_##name##_SMASK
849 static struct flag_table dcc_err_flags[] = {
850 FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)),
851 FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)),
852 FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)),
853 FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)),
854 FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)),
855 FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)),
856 FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)),
857 FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)),
858 FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)),
859 FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)),
860 FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)),
861 FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)),
862 FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)),
863 FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)),
864 FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)),
865 FLAG_ENTRY0("link_err", DCCE(LINK_ERR)),
866 FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)),
867 FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)),
868 FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)),
869 FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)),
870 FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)),
871 FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)),
872 FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)),
873 FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)),
874 FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)),
875 FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)),
876 FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)),
877 FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)),
878 FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)),
879 FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)),
880 FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)),
881 FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)),
882 FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)),
883 FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)),
884 FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)),
885 FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)),
886 FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)),
887 FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)),
888 FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)),
889 FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)),
890 FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)),
891 FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)),
892 FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)),
893 FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)),
894 FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)),
895 FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)),
896 };
897
898 /*
899 * LCB error flags
900 */
901 #define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK
902 static struct flag_table lcb_err_flags[] = {
903 /* 0*/ FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)),
904 /* 1*/ FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)),
905 /* 2*/ FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)),
906 /* 3*/ FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST",
907 LCBE(ALL_LNS_FAILED_REINIT_TEST)),
908 /* 4*/ FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)),
909 /* 5*/ FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)),
910 /* 6*/ FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)),
911 /* 7*/ FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)),
912 /* 8*/ FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)),
913 /* 9*/ FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)),
914 /*10*/ FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)),
915 /*11*/ FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)),
916 /*12*/ FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)),
917 /*13*/ FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER",
918 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)),
919 /*14*/ FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)),
920 /*15*/ FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)),
921 /*16*/ FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)),
922 /*17*/ FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)),
923 /*18*/ FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)),
924 /*19*/ FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE",
925 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)),
926 /*20*/ FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)),
927 /*21*/ FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)),
928 /*22*/ FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)),
929 /*23*/ FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)),
930 /*24*/ FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)),
931 /*25*/ FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)),
932 /*26*/ FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP",
933 LCBE(RST_FOR_INCOMPLT_RND_TRIP)),
934 /*27*/ FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)),
935 /*28*/ FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE",
936 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)),
937 /*29*/ FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR",
938 LCBE(REDUNDANT_FLIT_PARITY_ERR))
939 };
940
941 /*
942 * DC8051 Error Flags
943 */
944 #define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK
945 static struct flag_table dc8051_err_flags[] = {
946 FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)),
947 FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)),
948 FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)),
949 FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)),
950 FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)),
951 FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)),
952 FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)),
953 FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)),
954 FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES",
955 D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)),
956 FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)),
957 };
958
959 /*
960 * DC8051 Information Error flags
961 *
962 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field.
963 */
964 static struct flag_table dc8051_info_err_flags[] = {
965 FLAG_ENTRY0("Spico ROM check failed", SPICO_ROM_FAILED),
966 FLAG_ENTRY0("Unknown frame received", UNKNOWN_FRAME),
967 FLAG_ENTRY0("Target BER not met", TARGET_BER_NOT_MET),
968 FLAG_ENTRY0("Serdes internal loopback failure",
969 FAILED_SERDES_INTERNAL_LOOPBACK),
970 FLAG_ENTRY0("Failed SerDes init", FAILED_SERDES_INIT),
971 FLAG_ENTRY0("Failed LNI(Polling)", FAILED_LNI_POLLING),
972 FLAG_ENTRY0("Failed LNI(Debounce)", FAILED_LNI_DEBOUNCE),
973 FLAG_ENTRY0("Failed LNI(EstbComm)", FAILED_LNI_ESTBCOMM),
974 FLAG_ENTRY0("Failed LNI(OptEq)", FAILED_LNI_OPTEQ),
975 FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1),
976 FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2),
977 FLAG_ENTRY0("Failed LNI(ConfigLT)", FAILED_LNI_CONFIGLT),
978 FLAG_ENTRY0("Host Handshake Timeout", HOST_HANDSHAKE_TIMEOUT),
979 FLAG_ENTRY0("External Device Request Timeout",
980 EXTERNAL_DEVICE_REQ_TIMEOUT),
981 };
982
983 /*
984 * DC8051 Information Host Information flags
985 *
986 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field.
987 */
988 static struct flag_table dc8051_info_host_msg_flags[] = {
989 FLAG_ENTRY0("Host request done", 0x0001),
990 FLAG_ENTRY0("BC PWR_MGM message", 0x0002),
991 FLAG_ENTRY0("BC SMA message", 0x0004),
992 FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008),
993 FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010),
994 FLAG_ENTRY0("External device config request", 0x0020),
995 FLAG_ENTRY0("VerifyCap all frames received", 0x0040),
996 FLAG_ENTRY0("LinkUp achieved", 0x0080),
997 FLAG_ENTRY0("Link going down", 0x0100),
998 FLAG_ENTRY0("Link width downgraded", 0x0200),
999 };
1000
1001 static u32 encoded_size(u32 size);
1002 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate);
1003 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state);
1004 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
1005 u8 *continuous);
1006 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
1007 u8 *vcu, u16 *vl15buf, u8 *crc_sizes);
1008 static void read_vc_remote_link_width(struct hfi1_devdata *dd,
1009 u8 *remote_tx_rate, u16 *link_widths);
1010 static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits,
1011 u8 *flag_bits, u16 *link_widths);
1012 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
1013 u8 *device_rev);
1014 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx);
1015 static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx,
1016 u8 *tx_polarity_inversion,
1017 u8 *rx_polarity_inversion, u8 *max_rate);
1018 static void handle_sdma_eng_err(struct hfi1_devdata *dd,
1019 unsigned int context, u64 err_status);
1020 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg);
1021 static void handle_dcc_err(struct hfi1_devdata *dd,
1022 unsigned int context, u64 err_status);
1023 static void handle_lcb_err(struct hfi1_devdata *dd,
1024 unsigned int context, u64 err_status);
1025 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg);
1026 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1027 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1028 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1029 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1030 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1031 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1032 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1033 static void set_partition_keys(struct hfi1_pportdata *ppd);
1034 static const char *link_state_name(u32 state);
1035 static const char *link_state_reason_name(struct hfi1_pportdata *ppd,
1036 u32 state);
1037 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
1038 u64 *out_data);
1039 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data);
1040 static int thermal_init(struct hfi1_devdata *dd);
1041
1042 static void update_statusp(struct hfi1_pportdata *ppd, u32 state);
1043 static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd,
1044 int msecs);
1045 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1046 int msecs);
1047 static void log_state_transition(struct hfi1_pportdata *ppd, u32 state);
1048 static void log_physical_state(struct hfi1_pportdata *ppd, u32 state);
1049 static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1050 int msecs);
1051 static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd,
1052 int msecs);
1053 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc);
1054 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr);
1055 static void handle_temp_err(struct hfi1_devdata *dd);
1056 static void dc_shutdown(struct hfi1_devdata *dd);
1057 static void dc_start(struct hfi1_devdata *dd);
1058 static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp,
1059 unsigned int *np);
1060 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd);
1061 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms);
1062 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index);
1063 static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width);
1064
1065 /*
1066 * Error interrupt table entry. This is used as input to the interrupt
1067 * "clear down" routine used for all second tier error interrupt register.
1068 * Second tier interrupt registers have a single bit representing them
1069 * in the top-level CceIntStatus.
1070 */
1071 struct err_reg_info {
1072 u32 status; /* status CSR offset */
1073 u32 clear; /* clear CSR offset */
1074 u32 mask; /* mask CSR offset */
1075 void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg);
1076 const char *desc;
1077 };
1078
1079 #define NUM_MISC_ERRS (IS_GENERAL_ERR_END + 1 - IS_GENERAL_ERR_START)
1080 #define NUM_DC_ERRS (IS_DC_END + 1 - IS_DC_START)
1081 #define NUM_VARIOUS (IS_VARIOUS_END + 1 - IS_VARIOUS_START)
1082
1083 /*
1084 * Helpers for building HFI and DC error interrupt table entries. Different
1085 * helpers are needed because of inconsistent register names.
1086 */
1087 #define EE(reg, handler, desc) \
1088 { reg##_STATUS, reg##_CLEAR, reg##_MASK, \
1089 handler, desc }
1090 #define DC_EE1(reg, handler, desc) \
1091 { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc }
1092 #define DC_EE2(reg, handler, desc) \
1093 { reg##_FLG, reg##_CLR, reg##_EN, handler, desc }
1094
1095 /*
1096 * Table of the "misc" grouping of error interrupts. Each entry refers to
1097 * another register containing more information.
1098 */
1099 static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = {
1100 /* 0*/ EE(CCE_ERR, handle_cce_err, "CceErr"),
1101 /* 1*/ EE(RCV_ERR, handle_rxe_err, "RxeErr"),
1102 /* 2*/ EE(MISC_ERR, handle_misc_err, "MiscErr"),
1103 /* 3*/ { 0, 0, 0, NULL }, /* reserved */
1104 /* 4*/ EE(SEND_PIO_ERR, handle_pio_err, "PioErr"),
1105 /* 5*/ EE(SEND_DMA_ERR, handle_sdma_err, "SDmaErr"),
1106 /* 6*/ EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"),
1107 /* 7*/ EE(SEND_ERR, handle_txe_err, "TxeErr")
1108 /* the rest are reserved */
1109 };
1110
1111 /*
1112 * Index into the Various section of the interrupt sources
1113 * corresponding to the Critical Temperature interrupt.
1114 */
1115 #define TCRIT_INT_SOURCE 4
1116
1117 /*
1118 * SDMA error interrupt entry - refers to another register containing more
1119 * information.
1120 */
1121 static const struct err_reg_info sdma_eng_err =
1122 EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr");
1123
1124 static const struct err_reg_info various_err[NUM_VARIOUS] = {
1125 /* 0*/ { 0, 0, 0, NULL }, /* PbcInt */
1126 /* 1*/ { 0, 0, 0, NULL }, /* GpioAssertInt */
1127 /* 2*/ EE(ASIC_QSFP1, handle_qsfp_int, "QSFP1"),
1128 /* 3*/ EE(ASIC_QSFP2, handle_qsfp_int, "QSFP2"),
1129 /* 4*/ { 0, 0, 0, NULL }, /* TCritInt */
1130 /* rest are reserved */
1131 };
1132
1133 /*
1134 * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG
1135 * register can not be derived from the MTU value because 10K is not
1136 * a power of 2. Therefore, we need a constant. Everything else can
1137 * be calculated.
1138 */
1139 #define DCC_CFG_PORT_MTU_CAP_10240 7
1140
1141 /*
1142 * Table of the DC grouping of error interrupts. Each entry refers to
1143 * another register containing more information.
1144 */
1145 static const struct err_reg_info dc_errs[NUM_DC_ERRS] = {
1146 /* 0*/ DC_EE1(DCC_ERR, handle_dcc_err, "DCC Err"),
1147 /* 1*/ DC_EE2(DC_LCB_ERR, handle_lcb_err, "LCB Err"),
1148 /* 2*/ DC_EE2(DC_DC8051_ERR, handle_8051_interrupt, "DC8051 Interrupt"),
1149 /* 3*/ /* dc_lbm_int - special, see is_dc_int() */
1150 /* the rest are reserved */
1151 };
1152
1153 struct cntr_entry {
1154 /*
1155 * counter name
1156 */
1157 char *name;
1158
1159 /*
1160 * csr to read for name (if applicable)
1161 */
1162 u64 csr;
1163
1164 /*
1165 * offset into dd or ppd to store the counter's value
1166 */
1167 int offset;
1168
1169 /*
1170 * flags
1171 */
1172 u8 flags;
1173
1174 /*
1175 * accessor for stat element, context either dd or ppd
1176 */
1177 u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl,
1178 int mode, u64 data);
1179 };
1180
1181 #define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0
1182 #define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159
1183
1184 #define CNTR_ELEM(name, csr, offset, flags, accessor) \
1185 { \
1186 name, \
1187 csr, \
1188 offset, \
1189 flags, \
1190 accessor \
1191 }
1192
1193 /* 32bit RXE */
1194 #define RXE32_PORT_CNTR_ELEM(name, counter, flags) \
1195 CNTR_ELEM(#name, \
1196 (counter * 8 + RCV_COUNTER_ARRAY32), \
1197 0, flags | CNTR_32BIT, \
1198 port_access_u32_csr)
1199
1200 #define RXE32_DEV_CNTR_ELEM(name, counter, flags) \
1201 CNTR_ELEM(#name, \
1202 (counter * 8 + RCV_COUNTER_ARRAY32), \
1203 0, flags | CNTR_32BIT, \
1204 dev_access_u32_csr)
1205
1206 /* 64bit RXE */
1207 #define RXE64_PORT_CNTR_ELEM(name, counter, flags) \
1208 CNTR_ELEM(#name, \
1209 (counter * 8 + RCV_COUNTER_ARRAY64), \
1210 0, flags, \
1211 port_access_u64_csr)
1212
1213 #define RXE64_DEV_CNTR_ELEM(name, counter, flags) \
1214 CNTR_ELEM(#name, \
1215 (counter * 8 + RCV_COUNTER_ARRAY64), \
1216 0, flags, \
1217 dev_access_u64_csr)
1218
1219 #define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx
1220 #define OVR_ELM(ctx) \
1221 CNTR_ELEM("RcvHdrOvr" #ctx, \
1222 (RCV_HDR_OVFL_CNT + ctx * 0x100), \
1223 0, CNTR_NORMAL, port_access_u64_csr)
1224
1225 /* 32bit TXE */
1226 #define TXE32_PORT_CNTR_ELEM(name, counter, flags) \
1227 CNTR_ELEM(#name, \
1228 (counter * 8 + SEND_COUNTER_ARRAY32), \
1229 0, flags | CNTR_32BIT, \
1230 port_access_u32_csr)
1231
1232 /* 64bit TXE */
1233 #define TXE64_PORT_CNTR_ELEM(name, counter, flags) \
1234 CNTR_ELEM(#name, \
1235 (counter * 8 + SEND_COUNTER_ARRAY64), \
1236 0, flags, \
1237 port_access_u64_csr)
1238
1239 # define TX64_DEV_CNTR_ELEM(name, counter, flags) \
1240 CNTR_ELEM(#name,\
1241 counter * 8 + SEND_COUNTER_ARRAY64, \
1242 0, \
1243 flags, \
1244 dev_access_u64_csr)
1245
1246 /* CCE */
1247 #define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \
1248 CNTR_ELEM(#name, \
1249 (counter * 8 + CCE_COUNTER_ARRAY32), \
1250 0, flags | CNTR_32BIT, \
1251 dev_access_u32_csr)
1252
1253 #define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \
1254 CNTR_ELEM(#name, \
1255 (counter * 8 + CCE_INT_COUNTER_ARRAY32), \
1256 0, flags | CNTR_32BIT, \
1257 dev_access_u32_csr)
1258
1259 /* DC */
1260 #define DC_PERF_CNTR(name, counter, flags) \
1261 CNTR_ELEM(#name, \
1262 counter, \
1263 0, \
1264 flags, \
1265 dev_access_u64_csr)
1266
1267 #define DC_PERF_CNTR_LCB(name, counter, flags) \
1268 CNTR_ELEM(#name, \
1269 counter, \
1270 0, \
1271 flags, \
1272 dc_access_lcb_cntr)
1273
1274 /* ibp counters */
1275 #define SW_IBP_CNTR(name, cntr) \
1276 CNTR_ELEM(#name, \
1277 0, \
1278 0, \
1279 CNTR_SYNTH, \
1280 access_ibp_##cntr)
1281
1282 /**
1283 * hfi1_addr_from_offset - return addr for readq/writeq
1284 * @dd: the dd device
1285 * @offset: the offset of the CSR within bar0
1286 *
1287 * This routine selects the appropriate base address
1288 * based on the indicated offset.
1289 */
hfi1_addr_from_offset(const struct hfi1_devdata * dd,u32 offset)1290 static inline void __iomem *hfi1_addr_from_offset(
1291 const struct hfi1_devdata *dd,
1292 u32 offset)
1293 {
1294 if (offset >= dd->base2_start)
1295 return dd->kregbase2 + (offset - dd->base2_start);
1296 return dd->kregbase1 + offset;
1297 }
1298
1299 /**
1300 * read_csr - read CSR at the indicated offset
1301 * @dd: the dd device
1302 * @offset: the offset of the CSR within bar0
1303 *
1304 * Return: the value read or all FF's if there
1305 * is no mapping
1306 */
read_csr(const struct hfi1_devdata * dd,u32 offset)1307 u64 read_csr(const struct hfi1_devdata *dd, u32 offset)
1308 {
1309 if (dd->flags & HFI1_PRESENT)
1310 return readq(hfi1_addr_from_offset(dd, offset));
1311 return -1;
1312 }
1313
1314 /**
1315 * write_csr - write CSR at the indicated offset
1316 * @dd: the dd device
1317 * @offset: the offset of the CSR within bar0
1318 * @value: value to write
1319 */
write_csr(const struct hfi1_devdata * dd,u32 offset,u64 value)1320 void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)
1321 {
1322 if (dd->flags & HFI1_PRESENT) {
1323 void __iomem *base = hfi1_addr_from_offset(dd, offset);
1324
1325 /* avoid write to RcvArray */
1326 if (WARN_ON(offset >= RCV_ARRAY && offset < dd->base2_start))
1327 return;
1328 writeq(value, base);
1329 }
1330 }
1331
1332 /**
1333 * get_csr_addr - return te iomem address for offset
1334 * @dd: the dd device
1335 * @offset: the offset of the CSR within bar0
1336 *
1337 * Return: The iomem address to use in subsequent
1338 * writeq/readq operations.
1339 */
get_csr_addr(const struct hfi1_devdata * dd,u32 offset)1340 void __iomem *get_csr_addr(
1341 const struct hfi1_devdata *dd,
1342 u32 offset)
1343 {
1344 if (dd->flags & HFI1_PRESENT)
1345 return hfi1_addr_from_offset(dd, offset);
1346 return NULL;
1347 }
1348
read_write_csr(const struct hfi1_devdata * dd,u32 csr,int mode,u64 value)1349 static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr,
1350 int mode, u64 value)
1351 {
1352 u64 ret;
1353
1354 if (mode == CNTR_MODE_R) {
1355 ret = read_csr(dd, csr);
1356 } else if (mode == CNTR_MODE_W) {
1357 write_csr(dd, csr, value);
1358 ret = value;
1359 } else {
1360 dd_dev_err(dd, "Invalid cntr register access mode");
1361 return 0;
1362 }
1363
1364 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode);
1365 return ret;
1366 }
1367
1368 /* Dev Access */
dev_access_u32_csr(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1369 static u64 dev_access_u32_csr(const struct cntr_entry *entry,
1370 void *context, int vl, int mode, u64 data)
1371 {
1372 struct hfi1_devdata *dd = context;
1373 u64 csr = entry->csr;
1374
1375 if (entry->flags & CNTR_SDMA) {
1376 if (vl == CNTR_INVALID_VL)
1377 return 0;
1378 csr += 0x100 * vl;
1379 } else {
1380 if (vl != CNTR_INVALID_VL)
1381 return 0;
1382 }
1383 return read_write_csr(dd, csr, mode, data);
1384 }
1385
access_sde_err_cnt(const struct cntr_entry * entry,void * context,int idx,int mode,u64 data)1386 static u64 access_sde_err_cnt(const struct cntr_entry *entry,
1387 void *context, int idx, int mode, u64 data)
1388 {
1389 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1390
1391 if (dd->per_sdma && idx < dd->num_sdma)
1392 return dd->per_sdma[idx].err_cnt;
1393 return 0;
1394 }
1395
access_sde_int_cnt(const struct cntr_entry * entry,void * context,int idx,int mode,u64 data)1396 static u64 access_sde_int_cnt(const struct cntr_entry *entry,
1397 void *context, int idx, int mode, u64 data)
1398 {
1399 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1400
1401 if (dd->per_sdma && idx < dd->num_sdma)
1402 return dd->per_sdma[idx].sdma_int_cnt;
1403 return 0;
1404 }
1405
access_sde_idle_int_cnt(const struct cntr_entry * entry,void * context,int idx,int mode,u64 data)1406 static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry,
1407 void *context, int idx, int mode, u64 data)
1408 {
1409 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1410
1411 if (dd->per_sdma && idx < dd->num_sdma)
1412 return dd->per_sdma[idx].idle_int_cnt;
1413 return 0;
1414 }
1415
access_sde_progress_int_cnt(const struct cntr_entry * entry,void * context,int idx,int mode,u64 data)1416 static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry,
1417 void *context, int idx, int mode,
1418 u64 data)
1419 {
1420 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1421
1422 if (dd->per_sdma && idx < dd->num_sdma)
1423 return dd->per_sdma[idx].progress_int_cnt;
1424 return 0;
1425 }
1426
dev_access_u64_csr(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1427 static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context,
1428 int vl, int mode, u64 data)
1429 {
1430 struct hfi1_devdata *dd = context;
1431
1432 u64 val = 0;
1433 u64 csr = entry->csr;
1434
1435 if (entry->flags & CNTR_VL) {
1436 if (vl == CNTR_INVALID_VL)
1437 return 0;
1438 csr += 8 * vl;
1439 } else {
1440 if (vl != CNTR_INVALID_VL)
1441 return 0;
1442 }
1443
1444 val = read_write_csr(dd, csr, mode, data);
1445 return val;
1446 }
1447
dc_access_lcb_cntr(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1448 static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context,
1449 int vl, int mode, u64 data)
1450 {
1451 struct hfi1_devdata *dd = context;
1452 u32 csr = entry->csr;
1453 int ret = 0;
1454
1455 if (vl != CNTR_INVALID_VL)
1456 return 0;
1457 if (mode == CNTR_MODE_R)
1458 ret = read_lcb_csr(dd, csr, &data);
1459 else if (mode == CNTR_MODE_W)
1460 ret = write_lcb_csr(dd, csr, data);
1461
1462 if (ret) {
1463 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);
1464 return 0;
1465 }
1466
1467 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode);
1468 return data;
1469 }
1470
1471 /* Port Access */
port_access_u32_csr(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1472 static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context,
1473 int vl, int mode, u64 data)
1474 {
1475 struct hfi1_pportdata *ppd = context;
1476
1477 if (vl != CNTR_INVALID_VL)
1478 return 0;
1479 return read_write_csr(ppd->dd, entry->csr, mode, data);
1480 }
1481
port_access_u64_csr(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1482 static u64 port_access_u64_csr(const struct cntr_entry *entry,
1483 void *context, int vl, int mode, u64 data)
1484 {
1485 struct hfi1_pportdata *ppd = context;
1486 u64 val;
1487 u64 csr = entry->csr;
1488
1489 if (entry->flags & CNTR_VL) {
1490 if (vl == CNTR_INVALID_VL)
1491 return 0;
1492 csr += 8 * vl;
1493 } else {
1494 if (vl != CNTR_INVALID_VL)
1495 return 0;
1496 }
1497 val = read_write_csr(ppd->dd, csr, mode, data);
1498 return val;
1499 }
1500
1501 /* Software defined */
read_write_sw(struct hfi1_devdata * dd,u64 * cntr,int mode,u64 data)1502 static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode,
1503 u64 data)
1504 {
1505 u64 ret;
1506
1507 if (mode == CNTR_MODE_R) {
1508 ret = *cntr;
1509 } else if (mode == CNTR_MODE_W) {
1510 *cntr = data;
1511 ret = data;
1512 } else {
1513 dd_dev_err(dd, "Invalid cntr sw access mode");
1514 return 0;
1515 }
1516
1517 hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode);
1518
1519 return ret;
1520 }
1521
access_sw_link_dn_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1522 static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context,
1523 int vl, int mode, u64 data)
1524 {
1525 struct hfi1_pportdata *ppd = context;
1526
1527 if (vl != CNTR_INVALID_VL)
1528 return 0;
1529 return read_write_sw(ppd->dd, &ppd->link_downed, mode, data);
1530 }
1531
access_sw_link_up_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1532 static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context,
1533 int vl, int mode, u64 data)
1534 {
1535 struct hfi1_pportdata *ppd = context;
1536
1537 if (vl != CNTR_INVALID_VL)
1538 return 0;
1539 return read_write_sw(ppd->dd, &ppd->link_up, mode, data);
1540 }
1541
access_sw_unknown_frame_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1542 static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry,
1543 void *context, int vl, int mode,
1544 u64 data)
1545 {
1546 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1547
1548 if (vl != CNTR_INVALID_VL)
1549 return 0;
1550 return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data);
1551 }
1552
access_sw_xmit_discards(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1553 static u64 access_sw_xmit_discards(const struct cntr_entry *entry,
1554 void *context, int vl, int mode, u64 data)
1555 {
1556 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1557 u64 zero = 0;
1558 u64 *counter;
1559
1560 if (vl == CNTR_INVALID_VL)
1561 counter = &ppd->port_xmit_discards;
1562 else if (vl >= 0 && vl < C_VL_COUNT)
1563 counter = &ppd->port_xmit_discards_vl[vl];
1564 else
1565 counter = &zero;
1566
1567 return read_write_sw(ppd->dd, counter, mode, data);
1568 }
1569
access_xmit_constraint_errs(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1570 static u64 access_xmit_constraint_errs(const struct cntr_entry *entry,
1571 void *context, int vl, int mode,
1572 u64 data)
1573 {
1574 struct hfi1_pportdata *ppd = context;
1575
1576 if (vl != CNTR_INVALID_VL)
1577 return 0;
1578
1579 return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors,
1580 mode, data);
1581 }
1582
access_rcv_constraint_errs(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1583 static u64 access_rcv_constraint_errs(const struct cntr_entry *entry,
1584 void *context, int vl, int mode, u64 data)
1585 {
1586 struct hfi1_pportdata *ppd = context;
1587
1588 if (vl != CNTR_INVALID_VL)
1589 return 0;
1590
1591 return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors,
1592 mode, data);
1593 }
1594
get_all_cpu_total(u64 __percpu * cntr)1595 u64 get_all_cpu_total(u64 __percpu *cntr)
1596 {
1597 int cpu;
1598 u64 counter = 0;
1599
1600 for_each_possible_cpu(cpu)
1601 counter += *per_cpu_ptr(cntr, cpu);
1602 return counter;
1603 }
1604
read_write_cpu(struct hfi1_devdata * dd,u64 * z_val,u64 __percpu * cntr,int vl,int mode,u64 data)1605 static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val,
1606 u64 __percpu *cntr,
1607 int vl, int mode, u64 data)
1608 {
1609 u64 ret = 0;
1610
1611 if (vl != CNTR_INVALID_VL)
1612 return 0;
1613
1614 if (mode == CNTR_MODE_R) {
1615 ret = get_all_cpu_total(cntr) - *z_val;
1616 } else if (mode == CNTR_MODE_W) {
1617 /* A write can only zero the counter */
1618 if (data == 0)
1619 *z_val = get_all_cpu_total(cntr);
1620 else
1621 dd_dev_err(dd, "Per CPU cntrs can only be zeroed");
1622 } else {
1623 dd_dev_err(dd, "Invalid cntr sw cpu access mode");
1624 return 0;
1625 }
1626
1627 return ret;
1628 }
1629
access_sw_cpu_intr(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1630 static u64 access_sw_cpu_intr(const struct cntr_entry *entry,
1631 void *context, int vl, int mode, u64 data)
1632 {
1633 struct hfi1_devdata *dd = context;
1634
1635 return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl,
1636 mode, data);
1637 }
1638
access_sw_cpu_rcv_limit(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1639 static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry,
1640 void *context, int vl, int mode, u64 data)
1641 {
1642 struct hfi1_devdata *dd = context;
1643
1644 return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl,
1645 mode, data);
1646 }
1647
access_sw_pio_wait(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1648 static u64 access_sw_pio_wait(const struct cntr_entry *entry,
1649 void *context, int vl, int mode, u64 data)
1650 {
1651 struct hfi1_devdata *dd = context;
1652
1653 return dd->verbs_dev.n_piowait;
1654 }
1655
access_sw_pio_drain(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1656 static u64 access_sw_pio_drain(const struct cntr_entry *entry,
1657 void *context, int vl, int mode, u64 data)
1658 {
1659 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1660
1661 return dd->verbs_dev.n_piodrain;
1662 }
1663
access_sw_ctx0_seq_drop(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1664 static u64 access_sw_ctx0_seq_drop(const struct cntr_entry *entry,
1665 void *context, int vl, int mode, u64 data)
1666 {
1667 struct hfi1_devdata *dd = context;
1668
1669 return dd->ctx0_seq_drop;
1670 }
1671
access_sw_vtx_wait(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1672 static u64 access_sw_vtx_wait(const struct cntr_entry *entry,
1673 void *context, int vl, int mode, u64 data)
1674 {
1675 struct hfi1_devdata *dd = context;
1676
1677 return dd->verbs_dev.n_txwait;
1678 }
1679
access_sw_kmem_wait(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1680 static u64 access_sw_kmem_wait(const struct cntr_entry *entry,
1681 void *context, int vl, int mode, u64 data)
1682 {
1683 struct hfi1_devdata *dd = context;
1684
1685 return dd->verbs_dev.n_kmem_wait;
1686 }
1687
access_sw_send_schedule(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1688 static u64 access_sw_send_schedule(const struct cntr_entry *entry,
1689 void *context, int vl, int mode, u64 data)
1690 {
1691 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1692
1693 return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl,
1694 mode, data);
1695 }
1696
1697 /* Software counters for the error status bits within MISC_ERR_STATUS */
access_misc_pll_lock_fail_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1698 static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry,
1699 void *context, int vl, int mode,
1700 u64 data)
1701 {
1702 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1703
1704 return dd->misc_err_status_cnt[12];
1705 }
1706
access_misc_mbist_fail_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1707 static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry,
1708 void *context, int vl, int mode,
1709 u64 data)
1710 {
1711 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1712
1713 return dd->misc_err_status_cnt[11];
1714 }
1715
access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1716 static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry,
1717 void *context, int vl, int mode,
1718 u64 data)
1719 {
1720 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1721
1722 return dd->misc_err_status_cnt[10];
1723 }
1724
access_misc_efuse_done_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1725 static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry,
1726 void *context, int vl,
1727 int mode, u64 data)
1728 {
1729 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1730
1731 return dd->misc_err_status_cnt[9];
1732 }
1733
access_misc_efuse_write_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1734 static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry,
1735 void *context, int vl, int mode,
1736 u64 data)
1737 {
1738 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1739
1740 return dd->misc_err_status_cnt[8];
1741 }
1742
access_misc_efuse_read_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1743 static u64 access_misc_efuse_read_bad_addr_err_cnt(
1744 const struct cntr_entry *entry,
1745 void *context, int vl, int mode, u64 data)
1746 {
1747 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1748
1749 return dd->misc_err_status_cnt[7];
1750 }
1751
access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1752 static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry,
1753 void *context, int vl,
1754 int mode, u64 data)
1755 {
1756 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1757
1758 return dd->misc_err_status_cnt[6];
1759 }
1760
access_misc_fw_auth_failed_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1761 static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry,
1762 void *context, int vl, int mode,
1763 u64 data)
1764 {
1765 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1766
1767 return dd->misc_err_status_cnt[5];
1768 }
1769
access_misc_key_mismatch_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1770 static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry,
1771 void *context, int vl, int mode,
1772 u64 data)
1773 {
1774 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1775
1776 return dd->misc_err_status_cnt[4];
1777 }
1778
access_misc_sbus_write_failed_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1779 static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry,
1780 void *context, int vl,
1781 int mode, u64 data)
1782 {
1783 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1784
1785 return dd->misc_err_status_cnt[3];
1786 }
1787
access_misc_csr_write_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1788 static u64 access_misc_csr_write_bad_addr_err_cnt(
1789 const struct cntr_entry *entry,
1790 void *context, int vl, int mode, u64 data)
1791 {
1792 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1793
1794 return dd->misc_err_status_cnt[2];
1795 }
1796
access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1797 static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1798 void *context, int vl,
1799 int mode, u64 data)
1800 {
1801 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1802
1803 return dd->misc_err_status_cnt[1];
1804 }
1805
access_misc_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1806 static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry,
1807 void *context, int vl, int mode,
1808 u64 data)
1809 {
1810 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1811
1812 return dd->misc_err_status_cnt[0];
1813 }
1814
1815 /*
1816 * Software counter for the aggregate of
1817 * individual CceErrStatus counters
1818 */
access_sw_cce_err_status_aggregated_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1819 static u64 access_sw_cce_err_status_aggregated_cnt(
1820 const struct cntr_entry *entry,
1821 void *context, int vl, int mode, u64 data)
1822 {
1823 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1824
1825 return dd->sw_cce_err_status_aggregate;
1826 }
1827
1828 /*
1829 * Software counters corresponding to each of the
1830 * error status bits within CceErrStatus
1831 */
access_cce_msix_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1832 static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry,
1833 void *context, int vl, int mode,
1834 u64 data)
1835 {
1836 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1837
1838 return dd->cce_err_status_cnt[40];
1839 }
1840
access_cce_int_map_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1841 static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry,
1842 void *context, int vl, int mode,
1843 u64 data)
1844 {
1845 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1846
1847 return dd->cce_err_status_cnt[39];
1848 }
1849
access_cce_int_map_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1850 static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry,
1851 void *context, int vl, int mode,
1852 u64 data)
1853 {
1854 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1855
1856 return dd->cce_err_status_cnt[38];
1857 }
1858
access_cce_msix_table_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1859 static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry,
1860 void *context, int vl, int mode,
1861 u64 data)
1862 {
1863 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1864
1865 return dd->cce_err_status_cnt[37];
1866 }
1867
access_cce_msix_table_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1868 static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry,
1869 void *context, int vl, int mode,
1870 u64 data)
1871 {
1872 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1873
1874 return dd->cce_err_status_cnt[36];
1875 }
1876
access_cce_rxdma_conv_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1877 static u64 access_cce_rxdma_conv_fifo_parity_err_cnt(
1878 const struct cntr_entry *entry,
1879 void *context, int vl, int mode, u64 data)
1880 {
1881 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1882
1883 return dd->cce_err_status_cnt[35];
1884 }
1885
access_cce_rcpl_async_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1886 static u64 access_cce_rcpl_async_fifo_parity_err_cnt(
1887 const struct cntr_entry *entry,
1888 void *context, int vl, int mode, u64 data)
1889 {
1890 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1891
1892 return dd->cce_err_status_cnt[34];
1893 }
1894
access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1895 static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry,
1896 void *context, int vl,
1897 int mode, u64 data)
1898 {
1899 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1900
1901 return dd->cce_err_status_cnt[33];
1902 }
1903
access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1904 static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1905 void *context, int vl, int mode,
1906 u64 data)
1907 {
1908 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1909
1910 return dd->cce_err_status_cnt[32];
1911 }
1912
access_la_triggered_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1913 static u64 access_la_triggered_cnt(const struct cntr_entry *entry,
1914 void *context, int vl, int mode, u64 data)
1915 {
1916 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1917
1918 return dd->cce_err_status_cnt[31];
1919 }
1920
access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1921 static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry,
1922 void *context, int vl, int mode,
1923 u64 data)
1924 {
1925 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1926
1927 return dd->cce_err_status_cnt[30];
1928 }
1929
access_pcic_receive_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1930 static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry,
1931 void *context, int vl, int mode,
1932 u64 data)
1933 {
1934 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1935
1936 return dd->cce_err_status_cnt[29];
1937 }
1938
access_pcic_transmit_back_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1939 static u64 access_pcic_transmit_back_parity_err_cnt(
1940 const struct cntr_entry *entry,
1941 void *context, int vl, int mode, u64 data)
1942 {
1943 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1944
1945 return dd->cce_err_status_cnt[28];
1946 }
1947
access_pcic_transmit_front_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1948 static u64 access_pcic_transmit_front_parity_err_cnt(
1949 const struct cntr_entry *entry,
1950 void *context, int vl, int mode, u64 data)
1951 {
1952 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1953
1954 return dd->cce_err_status_cnt[27];
1955 }
1956
access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1957 static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1958 void *context, int vl, int mode,
1959 u64 data)
1960 {
1961 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1962
1963 return dd->cce_err_status_cnt[26];
1964 }
1965
access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1966 static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1967 void *context, int vl, int mode,
1968 u64 data)
1969 {
1970 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1971
1972 return dd->cce_err_status_cnt[25];
1973 }
1974
access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1975 static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1976 void *context, int vl, int mode,
1977 u64 data)
1978 {
1979 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1980
1981 return dd->cce_err_status_cnt[24];
1982 }
1983
access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1984 static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1985 void *context, int vl, int mode,
1986 u64 data)
1987 {
1988 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1989
1990 return dd->cce_err_status_cnt[23];
1991 }
1992
access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1993 static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry,
1994 void *context, int vl,
1995 int mode, u64 data)
1996 {
1997 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1998
1999 return dd->cce_err_status_cnt[22];
2000 }
2001
access_pcic_retry_mem_unc_err(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2002 static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry,
2003 void *context, int vl, int mode,
2004 u64 data)
2005 {
2006 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2007
2008 return dd->cce_err_status_cnt[21];
2009 }
2010
access_pcic_n_post_dat_q_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2011 static u64 access_pcic_n_post_dat_q_parity_err_cnt(
2012 const struct cntr_entry *entry,
2013 void *context, int vl, int mode, u64 data)
2014 {
2015 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2016
2017 return dd->cce_err_status_cnt[20];
2018 }
2019
access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2020 static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry,
2021 void *context, int vl,
2022 int mode, u64 data)
2023 {
2024 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2025
2026 return dd->cce_err_status_cnt[19];
2027 }
2028
access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2029 static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry,
2030 void *context, int vl, int mode,
2031 u64 data)
2032 {
2033 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2034
2035 return dd->cce_err_status_cnt[18];
2036 }
2037
access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2038 static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry,
2039 void *context, int vl, int mode,
2040 u64 data)
2041 {
2042 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2043
2044 return dd->cce_err_status_cnt[17];
2045 }
2046
access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2047 static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry,
2048 void *context, int vl, int mode,
2049 u64 data)
2050 {
2051 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2052
2053 return dd->cce_err_status_cnt[16];
2054 }
2055
access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2056 static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry,
2057 void *context, int vl, int mode,
2058 u64 data)
2059 {
2060 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2061
2062 return dd->cce_err_status_cnt[15];
2063 }
2064
access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2065 static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry,
2066 void *context, int vl,
2067 int mode, u64 data)
2068 {
2069 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2070
2071 return dd->cce_err_status_cnt[14];
2072 }
2073
access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2074 static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry,
2075 void *context, int vl, int mode,
2076 u64 data)
2077 {
2078 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2079
2080 return dd->cce_err_status_cnt[13];
2081 }
2082
access_cce_cli1_async_fifo_dbg_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2083 static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt(
2084 const struct cntr_entry *entry,
2085 void *context, int vl, int mode, u64 data)
2086 {
2087 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2088
2089 return dd->cce_err_status_cnt[12];
2090 }
2091
access_cce_cli1_async_fifo_rxdma_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2092 static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt(
2093 const struct cntr_entry *entry,
2094 void *context, int vl, int mode, u64 data)
2095 {
2096 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2097
2098 return dd->cce_err_status_cnt[11];
2099 }
2100
access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2101 static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt(
2102 const struct cntr_entry *entry,
2103 void *context, int vl, int mode, u64 data)
2104 {
2105 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2106
2107 return dd->cce_err_status_cnt[10];
2108 }
2109
access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2110 static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt(
2111 const struct cntr_entry *entry,
2112 void *context, int vl, int mode, u64 data)
2113 {
2114 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2115
2116 return dd->cce_err_status_cnt[9];
2117 }
2118
access_cce_cli2_async_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2119 static u64 access_cce_cli2_async_fifo_parity_err_cnt(
2120 const struct cntr_entry *entry,
2121 void *context, int vl, int mode, u64 data)
2122 {
2123 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2124
2125 return dd->cce_err_status_cnt[8];
2126 }
2127
access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2128 static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry,
2129 void *context, int vl,
2130 int mode, u64 data)
2131 {
2132 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2133
2134 return dd->cce_err_status_cnt[7];
2135 }
2136
access_cce_cli0_async_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2137 static u64 access_cce_cli0_async_fifo_parity_err_cnt(
2138 const struct cntr_entry *entry,
2139 void *context, int vl, int mode, u64 data)
2140 {
2141 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2142
2143 return dd->cce_err_status_cnt[6];
2144 }
2145
access_cce_rspd_data_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2146 static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry,
2147 void *context, int vl, int mode,
2148 u64 data)
2149 {
2150 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2151
2152 return dd->cce_err_status_cnt[5];
2153 }
2154
access_cce_trgt_access_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2155 static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry,
2156 void *context, int vl, int mode,
2157 u64 data)
2158 {
2159 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2160
2161 return dd->cce_err_status_cnt[4];
2162 }
2163
access_cce_trgt_async_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2164 static u64 access_cce_trgt_async_fifo_parity_err_cnt(
2165 const struct cntr_entry *entry,
2166 void *context, int vl, int mode, u64 data)
2167 {
2168 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2169
2170 return dd->cce_err_status_cnt[3];
2171 }
2172
access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2173 static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2174 void *context, int vl,
2175 int mode, u64 data)
2176 {
2177 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2178
2179 return dd->cce_err_status_cnt[2];
2180 }
2181
access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2182 static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2183 void *context, int vl,
2184 int mode, u64 data)
2185 {
2186 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2187
2188 return dd->cce_err_status_cnt[1];
2189 }
2190
access_ccs_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2191 static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry,
2192 void *context, int vl, int mode,
2193 u64 data)
2194 {
2195 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2196
2197 return dd->cce_err_status_cnt[0];
2198 }
2199
2200 /*
2201 * Software counters corresponding to each of the
2202 * error status bits within RcvErrStatus
2203 */
access_rx_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2204 static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry,
2205 void *context, int vl, int mode,
2206 u64 data)
2207 {
2208 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2209
2210 return dd->rcv_err_status_cnt[63];
2211 }
2212
access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2213 static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2214 void *context, int vl,
2215 int mode, u64 data)
2216 {
2217 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2218
2219 return dd->rcv_err_status_cnt[62];
2220 }
2221
access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2222 static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2223 void *context, int vl, int mode,
2224 u64 data)
2225 {
2226 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2227
2228 return dd->rcv_err_status_cnt[61];
2229 }
2230
access_rx_dma_csr_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2231 static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry,
2232 void *context, int vl, int mode,
2233 u64 data)
2234 {
2235 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2236
2237 return dd->rcv_err_status_cnt[60];
2238 }
2239
access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2240 static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2241 void *context, int vl,
2242 int mode, u64 data)
2243 {
2244 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2245
2246 return dd->rcv_err_status_cnt[59];
2247 }
2248
access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2249 static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2250 void *context, int vl,
2251 int mode, u64 data)
2252 {
2253 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2254
2255 return dd->rcv_err_status_cnt[58];
2256 }
2257
access_rx_dma_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2258 static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry,
2259 void *context, int vl, int mode,
2260 u64 data)
2261 {
2262 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2263
2264 return dd->rcv_err_status_cnt[57];
2265 }
2266
access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2267 static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry,
2268 void *context, int vl, int mode,
2269 u64 data)
2270 {
2271 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2272
2273 return dd->rcv_err_status_cnt[56];
2274 }
2275
access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2276 static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry,
2277 void *context, int vl, int mode,
2278 u64 data)
2279 {
2280 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2281
2282 return dd->rcv_err_status_cnt[55];
2283 }
2284
access_rx_dma_data_fifo_rd_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2285 static u64 access_rx_dma_data_fifo_rd_cor_err_cnt(
2286 const struct cntr_entry *entry,
2287 void *context, int vl, int mode, u64 data)
2288 {
2289 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2290
2291 return dd->rcv_err_status_cnt[54];
2292 }
2293
access_rx_dma_data_fifo_rd_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2294 static u64 access_rx_dma_data_fifo_rd_unc_err_cnt(
2295 const struct cntr_entry *entry,
2296 void *context, int vl, int mode, u64 data)
2297 {
2298 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2299
2300 return dd->rcv_err_status_cnt[53];
2301 }
2302
access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2303 static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry,
2304 void *context, int vl,
2305 int mode, u64 data)
2306 {
2307 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2308
2309 return dd->rcv_err_status_cnt[52];
2310 }
2311
access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2312 static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry,
2313 void *context, int vl,
2314 int mode, u64 data)
2315 {
2316 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2317
2318 return dd->rcv_err_status_cnt[51];
2319 }
2320
access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2321 static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry,
2322 void *context, int vl,
2323 int mode, u64 data)
2324 {
2325 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2326
2327 return dd->rcv_err_status_cnt[50];
2328 }
2329
access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2330 static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry,
2331 void *context, int vl,
2332 int mode, u64 data)
2333 {
2334 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2335
2336 return dd->rcv_err_status_cnt[49];
2337 }
2338
access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2339 static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry,
2340 void *context, int vl,
2341 int mode, u64 data)
2342 {
2343 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2344
2345 return dd->rcv_err_status_cnt[48];
2346 }
2347
access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2348 static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry,
2349 void *context, int vl,
2350 int mode, u64 data)
2351 {
2352 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2353
2354 return dd->rcv_err_status_cnt[47];
2355 }
2356
access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2357 static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry,
2358 void *context, int vl, int mode,
2359 u64 data)
2360 {
2361 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2362
2363 return dd->rcv_err_status_cnt[46];
2364 }
2365
access_rx_hq_intr_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2366 static u64 access_rx_hq_intr_csr_parity_err_cnt(
2367 const struct cntr_entry *entry,
2368 void *context, int vl, int mode, u64 data)
2369 {
2370 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2371
2372 return dd->rcv_err_status_cnt[45];
2373 }
2374
access_rx_lookup_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2375 static u64 access_rx_lookup_csr_parity_err_cnt(
2376 const struct cntr_entry *entry,
2377 void *context, int vl, int mode, u64 data)
2378 {
2379 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2380
2381 return dd->rcv_err_status_cnt[44];
2382 }
2383
access_rx_lookup_rcv_array_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2384 static u64 access_rx_lookup_rcv_array_cor_err_cnt(
2385 const struct cntr_entry *entry,
2386 void *context, int vl, int mode, u64 data)
2387 {
2388 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2389
2390 return dd->rcv_err_status_cnt[43];
2391 }
2392
access_rx_lookup_rcv_array_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2393 static u64 access_rx_lookup_rcv_array_unc_err_cnt(
2394 const struct cntr_entry *entry,
2395 void *context, int vl, int mode, u64 data)
2396 {
2397 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2398
2399 return dd->rcv_err_status_cnt[42];
2400 }
2401
access_rx_lookup_des_part2_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2402 static u64 access_rx_lookup_des_part2_parity_err_cnt(
2403 const struct cntr_entry *entry,
2404 void *context, int vl, int mode, u64 data)
2405 {
2406 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2407
2408 return dd->rcv_err_status_cnt[41];
2409 }
2410
access_rx_lookup_des_part1_unc_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2411 static u64 access_rx_lookup_des_part1_unc_cor_err_cnt(
2412 const struct cntr_entry *entry,
2413 void *context, int vl, int mode, u64 data)
2414 {
2415 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2416
2417 return dd->rcv_err_status_cnt[40];
2418 }
2419
access_rx_lookup_des_part1_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2420 static u64 access_rx_lookup_des_part1_unc_err_cnt(
2421 const struct cntr_entry *entry,
2422 void *context, int vl, int mode, u64 data)
2423 {
2424 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2425
2426 return dd->rcv_err_status_cnt[39];
2427 }
2428
access_rx_rbuf_next_free_buf_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2429 static u64 access_rx_rbuf_next_free_buf_cor_err_cnt(
2430 const struct cntr_entry *entry,
2431 void *context, int vl, int mode, u64 data)
2432 {
2433 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2434
2435 return dd->rcv_err_status_cnt[38];
2436 }
2437
access_rx_rbuf_next_free_buf_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2438 static u64 access_rx_rbuf_next_free_buf_unc_err_cnt(
2439 const struct cntr_entry *entry,
2440 void *context, int vl, int mode, u64 data)
2441 {
2442 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2443
2444 return dd->rcv_err_status_cnt[37];
2445 }
2446
access_rbuf_fl_init_wr_addr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2447 static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt(
2448 const struct cntr_entry *entry,
2449 void *context, int vl, int mode, u64 data)
2450 {
2451 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2452
2453 return dd->rcv_err_status_cnt[36];
2454 }
2455
access_rx_rbuf_fl_initdone_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2456 static u64 access_rx_rbuf_fl_initdone_parity_err_cnt(
2457 const struct cntr_entry *entry,
2458 void *context, int vl, int mode, u64 data)
2459 {
2460 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2461
2462 return dd->rcv_err_status_cnt[35];
2463 }
2464
access_rx_rbuf_fl_write_addr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2465 static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt(
2466 const struct cntr_entry *entry,
2467 void *context, int vl, int mode, u64 data)
2468 {
2469 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2470
2471 return dd->rcv_err_status_cnt[34];
2472 }
2473
access_rx_rbuf_fl_rd_addr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2474 static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt(
2475 const struct cntr_entry *entry,
2476 void *context, int vl, int mode, u64 data)
2477 {
2478 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2479
2480 return dd->rcv_err_status_cnt[33];
2481 }
2482
access_rx_rbuf_empty_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2483 static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry,
2484 void *context, int vl, int mode,
2485 u64 data)
2486 {
2487 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2488
2489 return dd->rcv_err_status_cnt[32];
2490 }
2491
access_rx_rbuf_full_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2492 static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry,
2493 void *context, int vl, int mode,
2494 u64 data)
2495 {
2496 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2497
2498 return dd->rcv_err_status_cnt[31];
2499 }
2500
access_rbuf_bad_lookup_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2501 static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry,
2502 void *context, int vl, int mode,
2503 u64 data)
2504 {
2505 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2506
2507 return dd->rcv_err_status_cnt[30];
2508 }
2509
access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2510 static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry,
2511 void *context, int vl, int mode,
2512 u64 data)
2513 {
2514 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2515
2516 return dd->rcv_err_status_cnt[29];
2517 }
2518
access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2519 static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry,
2520 void *context, int vl,
2521 int mode, u64 data)
2522 {
2523 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2524
2525 return dd->rcv_err_status_cnt[28];
2526 }
2527
access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2528 static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt(
2529 const struct cntr_entry *entry,
2530 void *context, int vl, int mode, u64 data)
2531 {
2532 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2533
2534 return dd->rcv_err_status_cnt[27];
2535 }
2536
access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2537 static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt(
2538 const struct cntr_entry *entry,
2539 void *context, int vl, int mode, u64 data)
2540 {
2541 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2542
2543 return dd->rcv_err_status_cnt[26];
2544 }
2545
access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2546 static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt(
2547 const struct cntr_entry *entry,
2548 void *context, int vl, int mode, u64 data)
2549 {
2550 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2551
2552 return dd->rcv_err_status_cnt[25];
2553 }
2554
access_rx_rbuf_csr_q_vld_bit_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2555 static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt(
2556 const struct cntr_entry *entry,
2557 void *context, int vl, int mode, u64 data)
2558 {
2559 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2560
2561 return dd->rcv_err_status_cnt[24];
2562 }
2563
access_rx_rbuf_csr_q_next_buf_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2564 static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt(
2565 const struct cntr_entry *entry,
2566 void *context, int vl, int mode, u64 data)
2567 {
2568 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2569
2570 return dd->rcv_err_status_cnt[23];
2571 }
2572
access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2573 static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt(
2574 const struct cntr_entry *entry,
2575 void *context, int vl, int mode, u64 data)
2576 {
2577 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2578
2579 return dd->rcv_err_status_cnt[22];
2580 }
2581
access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2582 static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt(
2583 const struct cntr_entry *entry,
2584 void *context, int vl, int mode, u64 data)
2585 {
2586 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2587
2588 return dd->rcv_err_status_cnt[21];
2589 }
2590
access_rx_rbuf_block_list_read_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2591 static u64 access_rx_rbuf_block_list_read_cor_err_cnt(
2592 const struct cntr_entry *entry,
2593 void *context, int vl, int mode, u64 data)
2594 {
2595 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2596
2597 return dd->rcv_err_status_cnt[20];
2598 }
2599
access_rx_rbuf_block_list_read_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2600 static u64 access_rx_rbuf_block_list_read_unc_err_cnt(
2601 const struct cntr_entry *entry,
2602 void *context, int vl, int mode, u64 data)
2603 {
2604 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2605
2606 return dd->rcv_err_status_cnt[19];
2607 }
2608
access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2609 static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry,
2610 void *context, int vl,
2611 int mode, u64 data)
2612 {
2613 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2614
2615 return dd->rcv_err_status_cnt[18];
2616 }
2617
access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2618 static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry,
2619 void *context, int vl,
2620 int mode, u64 data)
2621 {
2622 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2623
2624 return dd->rcv_err_status_cnt[17];
2625 }
2626
access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2627 static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt(
2628 const struct cntr_entry *entry,
2629 void *context, int vl, int mode, u64 data)
2630 {
2631 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2632
2633 return dd->rcv_err_status_cnt[16];
2634 }
2635
access_rx_rbuf_lookup_des_reg_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2636 static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt(
2637 const struct cntr_entry *entry,
2638 void *context, int vl, int mode, u64 data)
2639 {
2640 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2641
2642 return dd->rcv_err_status_cnt[15];
2643 }
2644
access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2645 static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry,
2646 void *context, int vl,
2647 int mode, u64 data)
2648 {
2649 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2650
2651 return dd->rcv_err_status_cnt[14];
2652 }
2653
access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2654 static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry,
2655 void *context, int vl,
2656 int mode, u64 data)
2657 {
2658 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2659
2660 return dd->rcv_err_status_cnt[13];
2661 }
2662
access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2663 static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2664 void *context, int vl, int mode,
2665 u64 data)
2666 {
2667 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2668
2669 return dd->rcv_err_status_cnt[12];
2670 }
2671
access_rx_dma_flag_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2672 static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry,
2673 void *context, int vl, int mode,
2674 u64 data)
2675 {
2676 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2677
2678 return dd->rcv_err_status_cnt[11];
2679 }
2680
access_rx_dma_flag_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2681 static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry,
2682 void *context, int vl, int mode,
2683 u64 data)
2684 {
2685 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2686
2687 return dd->rcv_err_status_cnt[10];
2688 }
2689
access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2690 static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry,
2691 void *context, int vl, int mode,
2692 u64 data)
2693 {
2694 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2695
2696 return dd->rcv_err_status_cnt[9];
2697 }
2698
access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2699 static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry,
2700 void *context, int vl, int mode,
2701 u64 data)
2702 {
2703 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2704
2705 return dd->rcv_err_status_cnt[8];
2706 }
2707
access_rx_rcv_qp_map_table_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2708 static u64 access_rx_rcv_qp_map_table_cor_err_cnt(
2709 const struct cntr_entry *entry,
2710 void *context, int vl, int mode, u64 data)
2711 {
2712 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2713
2714 return dd->rcv_err_status_cnt[7];
2715 }
2716
access_rx_rcv_qp_map_table_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2717 static u64 access_rx_rcv_qp_map_table_unc_err_cnt(
2718 const struct cntr_entry *entry,
2719 void *context, int vl, int mode, u64 data)
2720 {
2721 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2722
2723 return dd->rcv_err_status_cnt[6];
2724 }
2725
access_rx_rcv_data_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2726 static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry,
2727 void *context, int vl, int mode,
2728 u64 data)
2729 {
2730 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2731
2732 return dd->rcv_err_status_cnt[5];
2733 }
2734
access_rx_rcv_data_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2735 static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry,
2736 void *context, int vl, int mode,
2737 u64 data)
2738 {
2739 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2740
2741 return dd->rcv_err_status_cnt[4];
2742 }
2743
access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2744 static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry,
2745 void *context, int vl, int mode,
2746 u64 data)
2747 {
2748 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2749
2750 return dd->rcv_err_status_cnt[3];
2751 }
2752
access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2753 static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry,
2754 void *context, int vl, int mode,
2755 u64 data)
2756 {
2757 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2758
2759 return dd->rcv_err_status_cnt[2];
2760 }
2761
access_rx_dc_intf_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2762 static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry,
2763 void *context, int vl, int mode,
2764 u64 data)
2765 {
2766 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2767
2768 return dd->rcv_err_status_cnt[1];
2769 }
2770
access_rx_dma_csr_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2771 static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry,
2772 void *context, int vl, int mode,
2773 u64 data)
2774 {
2775 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2776
2777 return dd->rcv_err_status_cnt[0];
2778 }
2779
2780 /*
2781 * Software counters corresponding to each of the
2782 * error status bits within SendPioErrStatus
2783 */
access_pio_pec_sop_head_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2784 static u64 access_pio_pec_sop_head_parity_err_cnt(
2785 const struct cntr_entry *entry,
2786 void *context, int vl, int mode, u64 data)
2787 {
2788 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2789
2790 return dd->send_pio_err_status_cnt[35];
2791 }
2792
access_pio_pcc_sop_head_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2793 static u64 access_pio_pcc_sop_head_parity_err_cnt(
2794 const struct cntr_entry *entry,
2795 void *context, int vl, int mode, u64 data)
2796 {
2797 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2798
2799 return dd->send_pio_err_status_cnt[34];
2800 }
2801
access_pio_last_returned_cnt_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2802 static u64 access_pio_last_returned_cnt_parity_err_cnt(
2803 const struct cntr_entry *entry,
2804 void *context, int vl, int mode, u64 data)
2805 {
2806 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2807
2808 return dd->send_pio_err_status_cnt[33];
2809 }
2810
access_pio_current_free_cnt_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2811 static u64 access_pio_current_free_cnt_parity_err_cnt(
2812 const struct cntr_entry *entry,
2813 void *context, int vl, int mode, u64 data)
2814 {
2815 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2816
2817 return dd->send_pio_err_status_cnt[32];
2818 }
2819
access_pio_reserved_31_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2820 static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry,
2821 void *context, int vl, int mode,
2822 u64 data)
2823 {
2824 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2825
2826 return dd->send_pio_err_status_cnt[31];
2827 }
2828
access_pio_reserved_30_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2829 static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry,
2830 void *context, int vl, int mode,
2831 u64 data)
2832 {
2833 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2834
2835 return dd->send_pio_err_status_cnt[30];
2836 }
2837
access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2838 static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry,
2839 void *context, int vl, int mode,
2840 u64 data)
2841 {
2842 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2843
2844 return dd->send_pio_err_status_cnt[29];
2845 }
2846
access_pio_ppmc_bqc_mem_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2847 static u64 access_pio_ppmc_bqc_mem_parity_err_cnt(
2848 const struct cntr_entry *entry,
2849 void *context, int vl, int mode, u64 data)
2850 {
2851 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2852
2853 return dd->send_pio_err_status_cnt[28];
2854 }
2855
access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2856 static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry,
2857 void *context, int vl, int mode,
2858 u64 data)
2859 {
2860 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2861
2862 return dd->send_pio_err_status_cnt[27];
2863 }
2864
access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2865 static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry,
2866 void *context, int vl, int mode,
2867 u64 data)
2868 {
2869 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2870
2871 return dd->send_pio_err_status_cnt[26];
2872 }
2873
access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2874 static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry,
2875 void *context, int vl,
2876 int mode, u64 data)
2877 {
2878 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2879
2880 return dd->send_pio_err_status_cnt[25];
2881 }
2882
access_pio_block_qw_count_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2883 static u64 access_pio_block_qw_count_parity_err_cnt(
2884 const struct cntr_entry *entry,
2885 void *context, int vl, int mode, u64 data)
2886 {
2887 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2888
2889 return dd->send_pio_err_status_cnt[24];
2890 }
2891
access_pio_write_qw_valid_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2892 static u64 access_pio_write_qw_valid_parity_err_cnt(
2893 const struct cntr_entry *entry,
2894 void *context, int vl, int mode, u64 data)
2895 {
2896 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2897
2898 return dd->send_pio_err_status_cnt[23];
2899 }
2900
access_pio_state_machine_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2901 static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry,
2902 void *context, int vl, int mode,
2903 u64 data)
2904 {
2905 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2906
2907 return dd->send_pio_err_status_cnt[22];
2908 }
2909
access_pio_write_data_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2910 static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry,
2911 void *context, int vl,
2912 int mode, u64 data)
2913 {
2914 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2915
2916 return dd->send_pio_err_status_cnt[21];
2917 }
2918
access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2919 static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry,
2920 void *context, int vl,
2921 int mode, u64 data)
2922 {
2923 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2924
2925 return dd->send_pio_err_status_cnt[20];
2926 }
2927
access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2928 static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry,
2929 void *context, int vl,
2930 int mode, u64 data)
2931 {
2932 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2933
2934 return dd->send_pio_err_status_cnt[19];
2935 }
2936
access_pio_pkt_evict_sm_or_arb_sm_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2937 static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt(
2938 const struct cntr_entry *entry,
2939 void *context, int vl, int mode, u64 data)
2940 {
2941 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2942
2943 return dd->send_pio_err_status_cnt[18];
2944 }
2945
access_pio_init_sm_in_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2946 static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry,
2947 void *context, int vl, int mode,
2948 u64 data)
2949 {
2950 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2951
2952 return dd->send_pio_err_status_cnt[17];
2953 }
2954
access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2955 static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry,
2956 void *context, int vl, int mode,
2957 u64 data)
2958 {
2959 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2960
2961 return dd->send_pio_err_status_cnt[16];
2962 }
2963
access_pio_credit_ret_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2964 static u64 access_pio_credit_ret_fifo_parity_err_cnt(
2965 const struct cntr_entry *entry,
2966 void *context, int vl, int mode, u64 data)
2967 {
2968 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2969
2970 return dd->send_pio_err_status_cnt[15];
2971 }
2972
access_pio_v1_len_mem_bank1_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2973 static u64 access_pio_v1_len_mem_bank1_cor_err_cnt(
2974 const struct cntr_entry *entry,
2975 void *context, int vl, int mode, u64 data)
2976 {
2977 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2978
2979 return dd->send_pio_err_status_cnt[14];
2980 }
2981
access_pio_v1_len_mem_bank0_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2982 static u64 access_pio_v1_len_mem_bank0_cor_err_cnt(
2983 const struct cntr_entry *entry,
2984 void *context, int vl, int mode, u64 data)
2985 {
2986 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2987
2988 return dd->send_pio_err_status_cnt[13];
2989 }
2990
access_pio_v1_len_mem_bank1_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2991 static u64 access_pio_v1_len_mem_bank1_unc_err_cnt(
2992 const struct cntr_entry *entry,
2993 void *context, int vl, int mode, u64 data)
2994 {
2995 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2996
2997 return dd->send_pio_err_status_cnt[12];
2998 }
2999
access_pio_v1_len_mem_bank0_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3000 static u64 access_pio_v1_len_mem_bank0_unc_err_cnt(
3001 const struct cntr_entry *entry,
3002 void *context, int vl, int mode, u64 data)
3003 {
3004 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3005
3006 return dd->send_pio_err_status_cnt[11];
3007 }
3008
access_pio_sm_pkt_reset_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3009 static u64 access_pio_sm_pkt_reset_parity_err_cnt(
3010 const struct cntr_entry *entry,
3011 void *context, int vl, int mode, u64 data)
3012 {
3013 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3014
3015 return dd->send_pio_err_status_cnt[10];
3016 }
3017
access_pio_pkt_evict_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3018 static u64 access_pio_pkt_evict_fifo_parity_err_cnt(
3019 const struct cntr_entry *entry,
3020 void *context, int vl, int mode, u64 data)
3021 {
3022 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3023
3024 return dd->send_pio_err_status_cnt[9];
3025 }
3026
access_pio_sbrdctrl_crrel_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3027 static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt(
3028 const struct cntr_entry *entry,
3029 void *context, int vl, int mode, u64 data)
3030 {
3031 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3032
3033 return dd->send_pio_err_status_cnt[8];
3034 }
3035
access_pio_sbrdctl_crrel_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3036 static u64 access_pio_sbrdctl_crrel_parity_err_cnt(
3037 const struct cntr_entry *entry,
3038 void *context, int vl, int mode, u64 data)
3039 {
3040 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3041
3042 return dd->send_pio_err_status_cnt[7];
3043 }
3044
access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3045 static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry,
3046 void *context, int vl, int mode,
3047 u64 data)
3048 {
3049 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3050
3051 return dd->send_pio_err_status_cnt[6];
3052 }
3053
access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3054 static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry,
3055 void *context, int vl, int mode,
3056 u64 data)
3057 {
3058 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3059
3060 return dd->send_pio_err_status_cnt[5];
3061 }
3062
access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3063 static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry,
3064 void *context, int vl, int mode,
3065 u64 data)
3066 {
3067 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3068
3069 return dd->send_pio_err_status_cnt[4];
3070 }
3071
access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3072 static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry,
3073 void *context, int vl, int mode,
3074 u64 data)
3075 {
3076 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3077
3078 return dd->send_pio_err_status_cnt[3];
3079 }
3080
access_pio_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3081 static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry,
3082 void *context, int vl, int mode,
3083 u64 data)
3084 {
3085 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3086
3087 return dd->send_pio_err_status_cnt[2];
3088 }
3089
access_pio_write_addr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3090 static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry,
3091 void *context, int vl,
3092 int mode, u64 data)
3093 {
3094 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3095
3096 return dd->send_pio_err_status_cnt[1];
3097 }
3098
access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3099 static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry,
3100 void *context, int vl, int mode,
3101 u64 data)
3102 {
3103 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3104
3105 return dd->send_pio_err_status_cnt[0];
3106 }
3107
3108 /*
3109 * Software counters corresponding to each of the
3110 * error status bits within SendDmaErrStatus
3111 */
access_sdma_pcie_req_tracking_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3112 static u64 access_sdma_pcie_req_tracking_cor_err_cnt(
3113 const struct cntr_entry *entry,
3114 void *context, int vl, int mode, u64 data)
3115 {
3116 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3117
3118 return dd->send_dma_err_status_cnt[3];
3119 }
3120
access_sdma_pcie_req_tracking_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3121 static u64 access_sdma_pcie_req_tracking_unc_err_cnt(
3122 const struct cntr_entry *entry,
3123 void *context, int vl, int mode, u64 data)
3124 {
3125 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3126
3127 return dd->send_dma_err_status_cnt[2];
3128 }
3129
access_sdma_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3130 static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry,
3131 void *context, int vl, int mode,
3132 u64 data)
3133 {
3134 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3135
3136 return dd->send_dma_err_status_cnt[1];
3137 }
3138
access_sdma_rpy_tag_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3139 static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry,
3140 void *context, int vl, int mode,
3141 u64 data)
3142 {
3143 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3144
3145 return dd->send_dma_err_status_cnt[0];
3146 }
3147
3148 /*
3149 * Software counters corresponding to each of the
3150 * error status bits within SendEgressErrStatus
3151 */
access_tx_read_pio_memory_csr_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3152 static u64 access_tx_read_pio_memory_csr_unc_err_cnt(
3153 const struct cntr_entry *entry,
3154 void *context, int vl, int mode, u64 data)
3155 {
3156 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3157
3158 return dd->send_egress_err_status_cnt[63];
3159 }
3160
access_tx_read_sdma_memory_csr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3161 static u64 access_tx_read_sdma_memory_csr_err_cnt(
3162 const struct cntr_entry *entry,
3163 void *context, int vl, int mode, u64 data)
3164 {
3165 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3166
3167 return dd->send_egress_err_status_cnt[62];
3168 }
3169
access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3170 static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry,
3171 void *context, int vl, int mode,
3172 u64 data)
3173 {
3174 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3175
3176 return dd->send_egress_err_status_cnt[61];
3177 }
3178
access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3179 static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry,
3180 void *context, int vl,
3181 int mode, u64 data)
3182 {
3183 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3184
3185 return dd->send_egress_err_status_cnt[60];
3186 }
3187
access_tx_read_sdma_memory_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3188 static u64 access_tx_read_sdma_memory_cor_err_cnt(
3189 const struct cntr_entry *entry,
3190 void *context, int vl, int mode, u64 data)
3191 {
3192 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3193
3194 return dd->send_egress_err_status_cnt[59];
3195 }
3196
access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3197 static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry,
3198 void *context, int vl, int mode,
3199 u64 data)
3200 {
3201 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3202
3203 return dd->send_egress_err_status_cnt[58];
3204 }
3205
access_tx_credit_overrun_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3206 static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry,
3207 void *context, int vl, int mode,
3208 u64 data)
3209 {
3210 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3211
3212 return dd->send_egress_err_status_cnt[57];
3213 }
3214
access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3215 static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry,
3216 void *context, int vl, int mode,
3217 u64 data)
3218 {
3219 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3220
3221 return dd->send_egress_err_status_cnt[56];
3222 }
3223
access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3224 static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry,
3225 void *context, int vl, int mode,
3226 u64 data)
3227 {
3228 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3229
3230 return dd->send_egress_err_status_cnt[55];
3231 }
3232
access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3233 static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry,
3234 void *context, int vl, int mode,
3235 u64 data)
3236 {
3237 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3238
3239 return dd->send_egress_err_status_cnt[54];
3240 }
3241
access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3242 static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry,
3243 void *context, int vl, int mode,
3244 u64 data)
3245 {
3246 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3247
3248 return dd->send_egress_err_status_cnt[53];
3249 }
3250
access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3251 static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry,
3252 void *context, int vl, int mode,
3253 u64 data)
3254 {
3255 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3256
3257 return dd->send_egress_err_status_cnt[52];
3258 }
3259
access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3260 static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry,
3261 void *context, int vl, int mode,
3262 u64 data)
3263 {
3264 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3265
3266 return dd->send_egress_err_status_cnt[51];
3267 }
3268
access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3269 static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry,
3270 void *context, int vl, int mode,
3271 u64 data)
3272 {
3273 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3274
3275 return dd->send_egress_err_status_cnt[50];
3276 }
3277
access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3278 static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry,
3279 void *context, int vl, int mode,
3280 u64 data)
3281 {
3282 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3283
3284 return dd->send_egress_err_status_cnt[49];
3285 }
3286
access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3287 static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry,
3288 void *context, int vl, int mode,
3289 u64 data)
3290 {
3291 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3292
3293 return dd->send_egress_err_status_cnt[48];
3294 }
3295
access_tx_credit_return_vl_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3296 static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry,
3297 void *context, int vl, int mode,
3298 u64 data)
3299 {
3300 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3301
3302 return dd->send_egress_err_status_cnt[47];
3303 }
3304
access_tx_hcrc_insertion_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3305 static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry,
3306 void *context, int vl, int mode,
3307 u64 data)
3308 {
3309 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3310
3311 return dd->send_egress_err_status_cnt[46];
3312 }
3313
access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3314 static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry,
3315 void *context, int vl, int mode,
3316 u64 data)
3317 {
3318 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3319
3320 return dd->send_egress_err_status_cnt[45];
3321 }
3322
access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3323 static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry,
3324 void *context, int vl,
3325 int mode, u64 data)
3326 {
3327 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3328
3329 return dd->send_egress_err_status_cnt[44];
3330 }
3331
access_tx_read_sdma_memory_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3332 static u64 access_tx_read_sdma_memory_unc_err_cnt(
3333 const struct cntr_entry *entry,
3334 void *context, int vl, int mode, u64 data)
3335 {
3336 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3337
3338 return dd->send_egress_err_status_cnt[43];
3339 }
3340
access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3341 static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry,
3342 void *context, int vl, int mode,
3343 u64 data)
3344 {
3345 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3346
3347 return dd->send_egress_err_status_cnt[42];
3348 }
3349
access_tx_credit_return_partiy_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3350 static u64 access_tx_credit_return_partiy_err_cnt(
3351 const struct cntr_entry *entry,
3352 void *context, int vl, int mode, u64 data)
3353 {
3354 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3355
3356 return dd->send_egress_err_status_cnt[41];
3357 }
3358
access_tx_launch_fifo8_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3359 static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt(
3360 const struct cntr_entry *entry,
3361 void *context, int vl, int mode, u64 data)
3362 {
3363 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3364
3365 return dd->send_egress_err_status_cnt[40];
3366 }
3367
access_tx_launch_fifo7_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3368 static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt(
3369 const struct cntr_entry *entry,
3370 void *context, int vl, int mode, u64 data)
3371 {
3372 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3373
3374 return dd->send_egress_err_status_cnt[39];
3375 }
3376
access_tx_launch_fifo6_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3377 static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt(
3378 const struct cntr_entry *entry,
3379 void *context, int vl, int mode, u64 data)
3380 {
3381 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3382
3383 return dd->send_egress_err_status_cnt[38];
3384 }
3385
access_tx_launch_fifo5_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3386 static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt(
3387 const struct cntr_entry *entry,
3388 void *context, int vl, int mode, u64 data)
3389 {
3390 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3391
3392 return dd->send_egress_err_status_cnt[37];
3393 }
3394
access_tx_launch_fifo4_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3395 static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt(
3396 const struct cntr_entry *entry,
3397 void *context, int vl, int mode, u64 data)
3398 {
3399 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3400
3401 return dd->send_egress_err_status_cnt[36];
3402 }
3403
access_tx_launch_fifo3_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3404 static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt(
3405 const struct cntr_entry *entry,
3406 void *context, int vl, int mode, u64 data)
3407 {
3408 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3409
3410 return dd->send_egress_err_status_cnt[35];
3411 }
3412
access_tx_launch_fifo2_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3413 static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt(
3414 const struct cntr_entry *entry,
3415 void *context, int vl, int mode, u64 data)
3416 {
3417 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3418
3419 return dd->send_egress_err_status_cnt[34];
3420 }
3421
access_tx_launch_fifo1_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3422 static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt(
3423 const struct cntr_entry *entry,
3424 void *context, int vl, int mode, u64 data)
3425 {
3426 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3427
3428 return dd->send_egress_err_status_cnt[33];
3429 }
3430
access_tx_launch_fifo0_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3431 static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt(
3432 const struct cntr_entry *entry,
3433 void *context, int vl, int mode, u64 data)
3434 {
3435 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3436
3437 return dd->send_egress_err_status_cnt[32];
3438 }
3439
access_tx_sdma15_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3440 static u64 access_tx_sdma15_disallowed_packet_err_cnt(
3441 const struct cntr_entry *entry,
3442 void *context, int vl, int mode, u64 data)
3443 {
3444 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3445
3446 return dd->send_egress_err_status_cnt[31];
3447 }
3448
access_tx_sdma14_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3449 static u64 access_tx_sdma14_disallowed_packet_err_cnt(
3450 const struct cntr_entry *entry,
3451 void *context, int vl, int mode, u64 data)
3452 {
3453 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3454
3455 return dd->send_egress_err_status_cnt[30];
3456 }
3457
access_tx_sdma13_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3458 static u64 access_tx_sdma13_disallowed_packet_err_cnt(
3459 const struct cntr_entry *entry,
3460 void *context, int vl, int mode, u64 data)
3461 {
3462 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3463
3464 return dd->send_egress_err_status_cnt[29];
3465 }
3466
access_tx_sdma12_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3467 static u64 access_tx_sdma12_disallowed_packet_err_cnt(
3468 const struct cntr_entry *entry,
3469 void *context, int vl, int mode, u64 data)
3470 {
3471 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3472
3473 return dd->send_egress_err_status_cnt[28];
3474 }
3475
access_tx_sdma11_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3476 static u64 access_tx_sdma11_disallowed_packet_err_cnt(
3477 const struct cntr_entry *entry,
3478 void *context, int vl, int mode, u64 data)
3479 {
3480 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3481
3482 return dd->send_egress_err_status_cnt[27];
3483 }
3484
access_tx_sdma10_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3485 static u64 access_tx_sdma10_disallowed_packet_err_cnt(
3486 const struct cntr_entry *entry,
3487 void *context, int vl, int mode, u64 data)
3488 {
3489 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3490
3491 return dd->send_egress_err_status_cnt[26];
3492 }
3493
access_tx_sdma9_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3494 static u64 access_tx_sdma9_disallowed_packet_err_cnt(
3495 const struct cntr_entry *entry,
3496 void *context, int vl, int mode, u64 data)
3497 {
3498 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3499
3500 return dd->send_egress_err_status_cnt[25];
3501 }
3502
access_tx_sdma8_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3503 static u64 access_tx_sdma8_disallowed_packet_err_cnt(
3504 const struct cntr_entry *entry,
3505 void *context, int vl, int mode, u64 data)
3506 {
3507 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3508
3509 return dd->send_egress_err_status_cnt[24];
3510 }
3511
access_tx_sdma7_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3512 static u64 access_tx_sdma7_disallowed_packet_err_cnt(
3513 const struct cntr_entry *entry,
3514 void *context, int vl, int mode, u64 data)
3515 {
3516 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3517
3518 return dd->send_egress_err_status_cnt[23];
3519 }
3520
access_tx_sdma6_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3521 static u64 access_tx_sdma6_disallowed_packet_err_cnt(
3522 const struct cntr_entry *entry,
3523 void *context, int vl, int mode, u64 data)
3524 {
3525 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3526
3527 return dd->send_egress_err_status_cnt[22];
3528 }
3529
access_tx_sdma5_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3530 static u64 access_tx_sdma5_disallowed_packet_err_cnt(
3531 const struct cntr_entry *entry,
3532 void *context, int vl, int mode, u64 data)
3533 {
3534 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3535
3536 return dd->send_egress_err_status_cnt[21];
3537 }
3538
access_tx_sdma4_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3539 static u64 access_tx_sdma4_disallowed_packet_err_cnt(
3540 const struct cntr_entry *entry,
3541 void *context, int vl, int mode, u64 data)
3542 {
3543 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3544
3545 return dd->send_egress_err_status_cnt[20];
3546 }
3547
access_tx_sdma3_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3548 static u64 access_tx_sdma3_disallowed_packet_err_cnt(
3549 const struct cntr_entry *entry,
3550 void *context, int vl, int mode, u64 data)
3551 {
3552 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3553
3554 return dd->send_egress_err_status_cnt[19];
3555 }
3556
access_tx_sdma2_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3557 static u64 access_tx_sdma2_disallowed_packet_err_cnt(
3558 const struct cntr_entry *entry,
3559 void *context, int vl, int mode, u64 data)
3560 {
3561 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3562
3563 return dd->send_egress_err_status_cnt[18];
3564 }
3565
access_tx_sdma1_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3566 static u64 access_tx_sdma1_disallowed_packet_err_cnt(
3567 const struct cntr_entry *entry,
3568 void *context, int vl, int mode, u64 data)
3569 {
3570 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3571
3572 return dd->send_egress_err_status_cnt[17];
3573 }
3574
access_tx_sdma0_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3575 static u64 access_tx_sdma0_disallowed_packet_err_cnt(
3576 const struct cntr_entry *entry,
3577 void *context, int vl, int mode, u64 data)
3578 {
3579 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3580
3581 return dd->send_egress_err_status_cnt[16];
3582 }
3583
access_tx_config_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3584 static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry,
3585 void *context, int vl, int mode,
3586 u64 data)
3587 {
3588 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3589
3590 return dd->send_egress_err_status_cnt[15];
3591 }
3592
access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3593 static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry,
3594 void *context, int vl,
3595 int mode, u64 data)
3596 {
3597 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3598
3599 return dd->send_egress_err_status_cnt[14];
3600 }
3601
access_tx_launch_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3602 static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry,
3603 void *context, int vl, int mode,
3604 u64 data)
3605 {
3606 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3607
3608 return dd->send_egress_err_status_cnt[13];
3609 }
3610
access_tx_illegal_vl_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3611 static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry,
3612 void *context, int vl, int mode,
3613 u64 data)
3614 {
3615 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3616
3617 return dd->send_egress_err_status_cnt[12];
3618 }
3619
access_tx_sbrd_ctl_state_machine_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3620 static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt(
3621 const struct cntr_entry *entry,
3622 void *context, int vl, int mode, u64 data)
3623 {
3624 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3625
3626 return dd->send_egress_err_status_cnt[11];
3627 }
3628
access_egress_reserved_10_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3629 static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry,
3630 void *context, int vl, int mode,
3631 u64 data)
3632 {
3633 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3634
3635 return dd->send_egress_err_status_cnt[10];
3636 }
3637
access_egress_reserved_9_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3638 static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry,
3639 void *context, int vl, int mode,
3640 u64 data)
3641 {
3642 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3643
3644 return dd->send_egress_err_status_cnt[9];
3645 }
3646
access_tx_sdma_launch_intf_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3647 static u64 access_tx_sdma_launch_intf_parity_err_cnt(
3648 const struct cntr_entry *entry,
3649 void *context, int vl, int mode, u64 data)
3650 {
3651 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3652
3653 return dd->send_egress_err_status_cnt[8];
3654 }
3655
access_tx_pio_launch_intf_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3656 static u64 access_tx_pio_launch_intf_parity_err_cnt(
3657 const struct cntr_entry *entry,
3658 void *context, int vl, int mode, u64 data)
3659 {
3660 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3661
3662 return dd->send_egress_err_status_cnt[7];
3663 }
3664
access_egress_reserved_6_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3665 static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry,
3666 void *context, int vl, int mode,
3667 u64 data)
3668 {
3669 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3670
3671 return dd->send_egress_err_status_cnt[6];
3672 }
3673
access_tx_incorrect_link_state_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3674 static u64 access_tx_incorrect_link_state_err_cnt(
3675 const struct cntr_entry *entry,
3676 void *context, int vl, int mode, u64 data)
3677 {
3678 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3679
3680 return dd->send_egress_err_status_cnt[5];
3681 }
3682
access_tx_linkdown_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3683 static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry,
3684 void *context, int vl, int mode,
3685 u64 data)
3686 {
3687 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3688
3689 return dd->send_egress_err_status_cnt[4];
3690 }
3691
access_tx_egress_fifi_underrun_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3692 static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt(
3693 const struct cntr_entry *entry,
3694 void *context, int vl, int mode, u64 data)
3695 {
3696 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3697
3698 return dd->send_egress_err_status_cnt[3];
3699 }
3700
access_egress_reserved_2_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3701 static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry,
3702 void *context, int vl, int mode,
3703 u64 data)
3704 {
3705 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3706
3707 return dd->send_egress_err_status_cnt[2];
3708 }
3709
access_tx_pkt_integrity_mem_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3710 static u64 access_tx_pkt_integrity_mem_unc_err_cnt(
3711 const struct cntr_entry *entry,
3712 void *context, int vl, int mode, u64 data)
3713 {
3714 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3715
3716 return dd->send_egress_err_status_cnt[1];
3717 }
3718
access_tx_pkt_integrity_mem_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3719 static u64 access_tx_pkt_integrity_mem_cor_err_cnt(
3720 const struct cntr_entry *entry,
3721 void *context, int vl, int mode, u64 data)
3722 {
3723 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3724
3725 return dd->send_egress_err_status_cnt[0];
3726 }
3727
3728 /*
3729 * Software counters corresponding to each of the
3730 * error status bits within SendErrStatus
3731 */
access_send_csr_write_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3732 static u64 access_send_csr_write_bad_addr_err_cnt(
3733 const struct cntr_entry *entry,
3734 void *context, int vl, int mode, u64 data)
3735 {
3736 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3737
3738 return dd->send_err_status_cnt[2];
3739 }
3740
access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3741 static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
3742 void *context, int vl,
3743 int mode, u64 data)
3744 {
3745 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3746
3747 return dd->send_err_status_cnt[1];
3748 }
3749
access_send_csr_parity_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3750 static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry,
3751 void *context, int vl, int mode,
3752 u64 data)
3753 {
3754 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3755
3756 return dd->send_err_status_cnt[0];
3757 }
3758
3759 /*
3760 * Software counters corresponding to each of the
3761 * error status bits within SendCtxtErrStatus
3762 */
access_pio_write_out_of_bounds_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3763 static u64 access_pio_write_out_of_bounds_err_cnt(
3764 const struct cntr_entry *entry,
3765 void *context, int vl, int mode, u64 data)
3766 {
3767 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3768
3769 return dd->sw_ctxt_err_status_cnt[4];
3770 }
3771
access_pio_write_overflow_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3772 static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry,
3773 void *context, int vl, int mode,
3774 u64 data)
3775 {
3776 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3777
3778 return dd->sw_ctxt_err_status_cnt[3];
3779 }
3780
access_pio_write_crosses_boundary_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3781 static u64 access_pio_write_crosses_boundary_err_cnt(
3782 const struct cntr_entry *entry,
3783 void *context, int vl, int mode, u64 data)
3784 {
3785 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3786
3787 return dd->sw_ctxt_err_status_cnt[2];
3788 }
3789
access_pio_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3790 static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry,
3791 void *context, int vl,
3792 int mode, u64 data)
3793 {
3794 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3795
3796 return dd->sw_ctxt_err_status_cnt[1];
3797 }
3798
access_pio_inconsistent_sop_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3799 static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry,
3800 void *context, int vl, int mode,
3801 u64 data)
3802 {
3803 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3804
3805 return dd->sw_ctxt_err_status_cnt[0];
3806 }
3807
3808 /*
3809 * Software counters corresponding to each of the
3810 * error status bits within SendDmaEngErrStatus
3811 */
access_sdma_header_request_fifo_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3812 static u64 access_sdma_header_request_fifo_cor_err_cnt(
3813 const struct cntr_entry *entry,
3814 void *context, int vl, int mode, u64 data)
3815 {
3816 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3817
3818 return dd->sw_send_dma_eng_err_status_cnt[23];
3819 }
3820
access_sdma_header_storage_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3821 static u64 access_sdma_header_storage_cor_err_cnt(
3822 const struct cntr_entry *entry,
3823 void *context, int vl, int mode, u64 data)
3824 {
3825 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3826
3827 return dd->sw_send_dma_eng_err_status_cnt[22];
3828 }
3829
access_sdma_packet_tracking_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3830 static u64 access_sdma_packet_tracking_cor_err_cnt(
3831 const struct cntr_entry *entry,
3832 void *context, int vl, int mode, u64 data)
3833 {
3834 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3835
3836 return dd->sw_send_dma_eng_err_status_cnt[21];
3837 }
3838
access_sdma_assembly_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3839 static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry,
3840 void *context, int vl, int mode,
3841 u64 data)
3842 {
3843 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3844
3845 return dd->sw_send_dma_eng_err_status_cnt[20];
3846 }
3847
access_sdma_desc_table_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3848 static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry,
3849 void *context, int vl, int mode,
3850 u64 data)
3851 {
3852 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3853
3854 return dd->sw_send_dma_eng_err_status_cnt[19];
3855 }
3856
access_sdma_header_request_fifo_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3857 static u64 access_sdma_header_request_fifo_unc_err_cnt(
3858 const struct cntr_entry *entry,
3859 void *context, int vl, int mode, u64 data)
3860 {
3861 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3862
3863 return dd->sw_send_dma_eng_err_status_cnt[18];
3864 }
3865
access_sdma_header_storage_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3866 static u64 access_sdma_header_storage_unc_err_cnt(
3867 const struct cntr_entry *entry,
3868 void *context, int vl, int mode, u64 data)
3869 {
3870 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3871
3872 return dd->sw_send_dma_eng_err_status_cnt[17];
3873 }
3874
access_sdma_packet_tracking_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3875 static u64 access_sdma_packet_tracking_unc_err_cnt(
3876 const struct cntr_entry *entry,
3877 void *context, int vl, int mode, u64 data)
3878 {
3879 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3880
3881 return dd->sw_send_dma_eng_err_status_cnt[16];
3882 }
3883
access_sdma_assembly_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3884 static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry,
3885 void *context, int vl, int mode,
3886 u64 data)
3887 {
3888 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3889
3890 return dd->sw_send_dma_eng_err_status_cnt[15];
3891 }
3892
access_sdma_desc_table_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3893 static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry,
3894 void *context, int vl, int mode,
3895 u64 data)
3896 {
3897 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3898
3899 return dd->sw_send_dma_eng_err_status_cnt[14];
3900 }
3901
access_sdma_timeout_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3902 static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry,
3903 void *context, int vl, int mode,
3904 u64 data)
3905 {
3906 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3907
3908 return dd->sw_send_dma_eng_err_status_cnt[13];
3909 }
3910
access_sdma_header_length_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3911 static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry,
3912 void *context, int vl, int mode,
3913 u64 data)
3914 {
3915 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3916
3917 return dd->sw_send_dma_eng_err_status_cnt[12];
3918 }
3919
access_sdma_header_address_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3920 static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry,
3921 void *context, int vl, int mode,
3922 u64 data)
3923 {
3924 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3925
3926 return dd->sw_send_dma_eng_err_status_cnt[11];
3927 }
3928
access_sdma_header_select_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3929 static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry,
3930 void *context, int vl, int mode,
3931 u64 data)
3932 {
3933 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3934
3935 return dd->sw_send_dma_eng_err_status_cnt[10];
3936 }
3937
access_sdma_reserved_9_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3938 static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry,
3939 void *context, int vl, int mode,
3940 u64 data)
3941 {
3942 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3943
3944 return dd->sw_send_dma_eng_err_status_cnt[9];
3945 }
3946
access_sdma_packet_desc_overflow_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3947 static u64 access_sdma_packet_desc_overflow_err_cnt(
3948 const struct cntr_entry *entry,
3949 void *context, int vl, int mode, u64 data)
3950 {
3951 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3952
3953 return dd->sw_send_dma_eng_err_status_cnt[8];
3954 }
3955
access_sdma_length_mismatch_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3956 static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry,
3957 void *context, int vl,
3958 int mode, u64 data)
3959 {
3960 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3961
3962 return dd->sw_send_dma_eng_err_status_cnt[7];
3963 }
3964
access_sdma_halt_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3965 static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry,
3966 void *context, int vl, int mode, u64 data)
3967 {
3968 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3969
3970 return dd->sw_send_dma_eng_err_status_cnt[6];
3971 }
3972
access_sdma_mem_read_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3973 static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry,
3974 void *context, int vl, int mode,
3975 u64 data)
3976 {
3977 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3978
3979 return dd->sw_send_dma_eng_err_status_cnt[5];
3980 }
3981
access_sdma_first_desc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3982 static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry,
3983 void *context, int vl, int mode,
3984 u64 data)
3985 {
3986 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3987
3988 return dd->sw_send_dma_eng_err_status_cnt[4];
3989 }
3990
access_sdma_tail_out_of_bounds_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3991 static u64 access_sdma_tail_out_of_bounds_err_cnt(
3992 const struct cntr_entry *entry,
3993 void *context, int vl, int mode, u64 data)
3994 {
3995 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3996
3997 return dd->sw_send_dma_eng_err_status_cnt[3];
3998 }
3999
access_sdma_too_long_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)4000 static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry,
4001 void *context, int vl, int mode,
4002 u64 data)
4003 {
4004 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4005
4006 return dd->sw_send_dma_eng_err_status_cnt[2];
4007 }
4008
access_sdma_gen_mismatch_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)4009 static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry,
4010 void *context, int vl, int mode,
4011 u64 data)
4012 {
4013 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4014
4015 return dd->sw_send_dma_eng_err_status_cnt[1];
4016 }
4017
access_sdma_wrong_dw_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)4018 static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry,
4019 void *context, int vl, int mode,
4020 u64 data)
4021 {
4022 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4023
4024 return dd->sw_send_dma_eng_err_status_cnt[0];
4025 }
4026
access_dc_rcv_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)4027 static u64 access_dc_rcv_err_cnt(const struct cntr_entry *entry,
4028 void *context, int vl, int mode,
4029 u64 data)
4030 {
4031 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4032
4033 u64 val = 0;
4034 u64 csr = entry->csr;
4035
4036 val = read_write_csr(dd, csr, mode, data);
4037 if (mode == CNTR_MODE_R) {
4038 val = val > CNTR_MAX - dd->sw_rcv_bypass_packet_errors ?
4039 CNTR_MAX : val + dd->sw_rcv_bypass_packet_errors;
4040 } else if (mode == CNTR_MODE_W) {
4041 dd->sw_rcv_bypass_packet_errors = 0;
4042 } else {
4043 dd_dev_err(dd, "Invalid cntr register access mode");
4044 return 0;
4045 }
4046 return val;
4047 }
4048
4049 #define def_access_sw_cpu(cntr) \
4050 static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry, \
4051 void *context, int vl, int mode, u64 data) \
4052 { \
4053 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
4054 return read_write_cpu(ppd->dd, &ppd->ibport_data.rvp.z_ ##cntr, \
4055 ppd->ibport_data.rvp.cntr, vl, \
4056 mode, data); \
4057 }
4058
4059 def_access_sw_cpu(rc_acks);
4060 def_access_sw_cpu(rc_qacks);
4061 def_access_sw_cpu(rc_delayed_comp);
4062
4063 #define def_access_ibp_counter(cntr) \
4064 static u64 access_ibp_##cntr(const struct cntr_entry *entry, \
4065 void *context, int vl, int mode, u64 data) \
4066 { \
4067 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
4068 \
4069 if (vl != CNTR_INVALID_VL) \
4070 return 0; \
4071 \
4072 return read_write_sw(ppd->dd, &ppd->ibport_data.rvp.n_ ##cntr, \
4073 mode, data); \
4074 }
4075
4076 def_access_ibp_counter(loop_pkts);
4077 def_access_ibp_counter(rc_resends);
4078 def_access_ibp_counter(rnr_naks);
4079 def_access_ibp_counter(other_naks);
4080 def_access_ibp_counter(rc_timeouts);
4081 def_access_ibp_counter(pkt_drops);
4082 def_access_ibp_counter(dmawait);
4083 def_access_ibp_counter(rc_seqnak);
4084 def_access_ibp_counter(rc_dupreq);
4085 def_access_ibp_counter(rdma_seq);
4086 def_access_ibp_counter(unaligned);
4087 def_access_ibp_counter(seq_naks);
4088 def_access_ibp_counter(rc_crwaits);
4089
4090 static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {
4091 [C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH),
4092 [C_RX_LEN_ERR] = RXE32_DEV_CNTR_ELEM(RxLenErr, RCV_LENGTH_ERR_CNT, CNTR_SYNTH),
4093 [C_RX_SHORT_ERR] = RXE32_DEV_CNTR_ELEM(RxShrErr, RCV_SHORT_ERR_CNT, CNTR_SYNTH),
4094 [C_RX_ICRC_ERR] = RXE32_DEV_CNTR_ELEM(RxICrcErr, RCV_ICRC_ERR_CNT, CNTR_SYNTH),
4095 [C_RX_EBP] = RXE32_DEV_CNTR_ELEM(RxEbpCnt, RCV_EBP_CNT, CNTR_SYNTH),
4096 [C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT,
4097 CNTR_NORMAL),
4098 [C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT,
4099 CNTR_NORMAL),
4100 [C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs,
4101 RCV_TID_FLOW_GEN_MISMATCH_CNT,
4102 CNTR_NORMAL),
4103 [C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL,
4104 CNTR_NORMAL),
4105 [C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs,
4106 RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL),
4107 [C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt,
4108 CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL),
4109 [C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT,
4110 CNTR_NORMAL),
4111 [C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT,
4112 CNTR_NORMAL),
4113 [C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT,
4114 CNTR_NORMAL),
4115 [C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT,
4116 CNTR_NORMAL),
4117 [C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT,
4118 CNTR_NORMAL),
4119 [C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT,
4120 CNTR_NORMAL),
4121 [C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt,
4122 CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL),
4123 [C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt,
4124 CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL),
4125 [C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT,
4126 CNTR_SYNTH),
4127 [C_DC_RCV_ERR] = CNTR_ELEM("DcRecvErr", DCC_ERR_PORTRCV_ERR_CNT, 0, CNTR_SYNTH,
4128 access_dc_rcv_err_cnt),
4129 [C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT,
4130 CNTR_SYNTH),
4131 [C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT,
4132 CNTR_SYNTH),
4133 [C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT,
4134 CNTR_SYNTH),
4135 [C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts,
4136 DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH),
4137 [C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts,
4138 DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT,
4139 CNTR_SYNTH),
4140 [C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr,
4141 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH),
4142 [C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT,
4143 CNTR_SYNTH),
4144 [C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT,
4145 CNTR_SYNTH),
4146 [C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT,
4147 CNTR_SYNTH),
4148 [C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT,
4149 CNTR_SYNTH),
4150 [C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT,
4151 CNTR_SYNTH),
4152 [C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT,
4153 CNTR_SYNTH),
4154 [C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT,
4155 CNTR_SYNTH),
4156 [C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT,
4157 CNTR_SYNTH | CNTR_VL),
4158 [C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT,
4159 CNTR_SYNTH | CNTR_VL),
4160 [C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH),
4161 [C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT,
4162 CNTR_SYNTH | CNTR_VL),
4163 [C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH),
4164 [C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT,
4165 CNTR_SYNTH | CNTR_VL),
4166 [C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT,
4167 CNTR_SYNTH),
4168 [C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT,
4169 CNTR_SYNTH | CNTR_VL),
4170 [C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT,
4171 CNTR_SYNTH),
4172 [C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT,
4173 CNTR_SYNTH | CNTR_VL),
4174 [C_DC_TOTAL_CRC] =
4175 DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR,
4176 CNTR_SYNTH),
4177 [C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0,
4178 CNTR_SYNTH),
4179 [C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1,
4180 CNTR_SYNTH),
4181 [C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2,
4182 CNTR_SYNTH),
4183 [C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3,
4184 CNTR_SYNTH),
4185 [C_DC_CRC_MULT_LN] =
4186 DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN,
4187 CNTR_SYNTH),
4188 [C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT,
4189 CNTR_SYNTH),
4190 [C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT,
4191 CNTR_SYNTH),
4192 [C_DC_SEQ_CRC_CNT] =
4193 DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT,
4194 CNTR_SYNTH),
4195 [C_DC_ESC0_ONLY_CNT] =
4196 DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT,
4197 CNTR_SYNTH),
4198 [C_DC_ESC0_PLUS1_CNT] =
4199 DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT,
4200 CNTR_SYNTH),
4201 [C_DC_ESC0_PLUS2_CNT] =
4202 DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT,
4203 CNTR_SYNTH),
4204 [C_DC_REINIT_FROM_PEER_CNT] =
4205 DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT,
4206 CNTR_SYNTH),
4207 [C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT,
4208 CNTR_SYNTH),
4209 [C_DC_MISC_FLG_CNT] =
4210 DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT,
4211 CNTR_SYNTH),
4212 [C_DC_PRF_GOOD_LTP_CNT] =
4213 DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH),
4214 [C_DC_PRF_ACCEPTED_LTP_CNT] =
4215 DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT,
4216 CNTR_SYNTH),
4217 [C_DC_PRF_RX_FLIT_CNT] =
4218 DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH),
4219 [C_DC_PRF_TX_FLIT_CNT] =
4220 DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH),
4221 [C_DC_PRF_CLK_CNTR] =
4222 DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH),
4223 [C_DC_PG_DBG_FLIT_CRDTS_CNT] =
4224 DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH),
4225 [C_DC_PG_STS_PAUSE_COMPLETE_CNT] =
4226 DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT,
4227 CNTR_SYNTH),
4228 [C_DC_PG_STS_TX_SBE_CNT] =
4229 DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH),
4230 [C_DC_PG_STS_TX_MBE_CNT] =
4231 DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT,
4232 CNTR_SYNTH),
4233 [C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL,
4234 access_sw_cpu_intr),
4235 [C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL,
4236 access_sw_cpu_rcv_limit),
4237 [C_SW_CTX0_SEQ_DROP] = CNTR_ELEM("SeqDrop0", 0, 0, CNTR_NORMAL,
4238 access_sw_ctx0_seq_drop),
4239 [C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL,
4240 access_sw_vtx_wait),
4241 [C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL,
4242 access_sw_pio_wait),
4243 [C_SW_PIO_DRAIN] = CNTR_ELEM("PioDrain", 0, 0, CNTR_NORMAL,
4244 access_sw_pio_drain),
4245 [C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL,
4246 access_sw_kmem_wait),
4247 [C_SW_TID_WAIT] = CNTR_ELEM("TidWait", 0, 0, CNTR_NORMAL,
4248 hfi1_access_sw_tid_wait),
4249 [C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL,
4250 access_sw_send_schedule),
4251 [C_SDMA_DESC_FETCHED_CNT] = CNTR_ELEM("SDEDscFdCn",
4252 SEND_DMA_DESC_FETCHED_CNT, 0,
4253 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4254 dev_access_u32_csr),
4255 [C_SDMA_INT_CNT] = CNTR_ELEM("SDMAInt", 0, 0,
4256 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4257 access_sde_int_cnt),
4258 [C_SDMA_ERR_CNT] = CNTR_ELEM("SDMAErrCt", 0, 0,
4259 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4260 access_sde_err_cnt),
4261 [C_SDMA_IDLE_INT_CNT] = CNTR_ELEM("SDMAIdInt", 0, 0,
4262 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4263 access_sde_idle_int_cnt),
4264 [C_SDMA_PROGRESS_INT_CNT] = CNTR_ELEM("SDMAPrIntCn", 0, 0,
4265 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4266 access_sde_progress_int_cnt),
4267 /* MISC_ERR_STATUS */
4268 [C_MISC_PLL_LOCK_FAIL_ERR] = CNTR_ELEM("MISC_PLL_LOCK_FAIL_ERR", 0, 0,
4269 CNTR_NORMAL,
4270 access_misc_pll_lock_fail_err_cnt),
4271 [C_MISC_MBIST_FAIL_ERR] = CNTR_ELEM("MISC_MBIST_FAIL_ERR", 0, 0,
4272 CNTR_NORMAL,
4273 access_misc_mbist_fail_err_cnt),
4274 [C_MISC_INVALID_EEP_CMD_ERR] = CNTR_ELEM("MISC_INVALID_EEP_CMD_ERR", 0, 0,
4275 CNTR_NORMAL,
4276 access_misc_invalid_eep_cmd_err_cnt),
4277 [C_MISC_EFUSE_DONE_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_DONE_PARITY_ERR", 0, 0,
4278 CNTR_NORMAL,
4279 access_misc_efuse_done_parity_err_cnt),
4280 [C_MISC_EFUSE_WRITE_ERR] = CNTR_ELEM("MISC_EFUSE_WRITE_ERR", 0, 0,
4281 CNTR_NORMAL,
4282 access_misc_efuse_write_err_cnt),
4283 [C_MISC_EFUSE_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_EFUSE_READ_BAD_ADDR_ERR", 0,
4284 0, CNTR_NORMAL,
4285 access_misc_efuse_read_bad_addr_err_cnt),
4286 [C_MISC_EFUSE_CSR_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_CSR_PARITY_ERR", 0, 0,
4287 CNTR_NORMAL,
4288 access_misc_efuse_csr_parity_err_cnt),
4289 [C_MISC_FW_AUTH_FAILED_ERR] = CNTR_ELEM("MISC_FW_AUTH_FAILED_ERR", 0, 0,
4290 CNTR_NORMAL,
4291 access_misc_fw_auth_failed_err_cnt),
4292 [C_MISC_KEY_MISMATCH_ERR] = CNTR_ELEM("MISC_KEY_MISMATCH_ERR", 0, 0,
4293 CNTR_NORMAL,
4294 access_misc_key_mismatch_err_cnt),
4295 [C_MISC_SBUS_WRITE_FAILED_ERR] = CNTR_ELEM("MISC_SBUS_WRITE_FAILED_ERR", 0, 0,
4296 CNTR_NORMAL,
4297 access_misc_sbus_write_failed_err_cnt),
4298 [C_MISC_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_WRITE_BAD_ADDR_ERR", 0, 0,
4299 CNTR_NORMAL,
4300 access_misc_csr_write_bad_addr_err_cnt),
4301 [C_MISC_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_READ_BAD_ADDR_ERR", 0, 0,
4302 CNTR_NORMAL,
4303 access_misc_csr_read_bad_addr_err_cnt),
4304 [C_MISC_CSR_PARITY_ERR] = CNTR_ELEM("MISC_CSR_PARITY_ERR", 0, 0,
4305 CNTR_NORMAL,
4306 access_misc_csr_parity_err_cnt),
4307 /* CceErrStatus */
4308 [C_CCE_ERR_STATUS_AGGREGATED_CNT] = CNTR_ELEM("CceErrStatusAggregatedCnt", 0, 0,
4309 CNTR_NORMAL,
4310 access_sw_cce_err_status_aggregated_cnt),
4311 [C_CCE_MSIX_CSR_PARITY_ERR] = CNTR_ELEM("CceMsixCsrParityErr", 0, 0,
4312 CNTR_NORMAL,
4313 access_cce_msix_csr_parity_err_cnt),
4314 [C_CCE_INT_MAP_UNC_ERR] = CNTR_ELEM("CceIntMapUncErr", 0, 0,
4315 CNTR_NORMAL,
4316 access_cce_int_map_unc_err_cnt),
4317 [C_CCE_INT_MAP_COR_ERR] = CNTR_ELEM("CceIntMapCorErr", 0, 0,
4318 CNTR_NORMAL,
4319 access_cce_int_map_cor_err_cnt),
4320 [C_CCE_MSIX_TABLE_UNC_ERR] = CNTR_ELEM("CceMsixTableUncErr", 0, 0,
4321 CNTR_NORMAL,
4322 access_cce_msix_table_unc_err_cnt),
4323 [C_CCE_MSIX_TABLE_COR_ERR] = CNTR_ELEM("CceMsixTableCorErr", 0, 0,
4324 CNTR_NORMAL,
4325 access_cce_msix_table_cor_err_cnt),
4326 [C_CCE_RXDMA_CONV_FIFO_PARITY_ERR] = CNTR_ELEM("CceRxdmaConvFifoParityErr", 0,
4327 0, CNTR_NORMAL,
4328 access_cce_rxdma_conv_fifo_parity_err_cnt),
4329 [C_CCE_RCPL_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceRcplAsyncFifoParityErr", 0,
4330 0, CNTR_NORMAL,
4331 access_cce_rcpl_async_fifo_parity_err_cnt),
4332 [C_CCE_SEG_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceSegWriteBadAddrErr", 0, 0,
4333 CNTR_NORMAL,
4334 access_cce_seg_write_bad_addr_err_cnt),
4335 [C_CCE_SEG_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceSegReadBadAddrErr", 0, 0,
4336 CNTR_NORMAL,
4337 access_cce_seg_read_bad_addr_err_cnt),
4338 [C_LA_TRIGGERED] = CNTR_ELEM("Cce LATriggered", 0, 0,
4339 CNTR_NORMAL,
4340 access_la_triggered_cnt),
4341 [C_CCE_TRGT_CPL_TIMEOUT_ERR] = CNTR_ELEM("CceTrgtCplTimeoutErr", 0, 0,
4342 CNTR_NORMAL,
4343 access_cce_trgt_cpl_timeout_err_cnt),
4344 [C_PCIC_RECEIVE_PARITY_ERR] = CNTR_ELEM("PcicReceiveParityErr", 0, 0,
4345 CNTR_NORMAL,
4346 access_pcic_receive_parity_err_cnt),
4347 [C_PCIC_TRANSMIT_BACK_PARITY_ERR] = CNTR_ELEM("PcicTransmitBackParityErr", 0, 0,
4348 CNTR_NORMAL,
4349 access_pcic_transmit_back_parity_err_cnt),
4350 [C_PCIC_TRANSMIT_FRONT_PARITY_ERR] = CNTR_ELEM("PcicTransmitFrontParityErr", 0,
4351 0, CNTR_NORMAL,
4352 access_pcic_transmit_front_parity_err_cnt),
4353 [C_PCIC_CPL_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicCplDatQUncErr", 0, 0,
4354 CNTR_NORMAL,
4355 access_pcic_cpl_dat_q_unc_err_cnt),
4356 [C_PCIC_CPL_HD_Q_UNC_ERR] = CNTR_ELEM("PcicCplHdQUncErr", 0, 0,
4357 CNTR_NORMAL,
4358 access_pcic_cpl_hd_q_unc_err_cnt),
4359 [C_PCIC_POST_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicPostDatQUncErr", 0, 0,
4360 CNTR_NORMAL,
4361 access_pcic_post_dat_q_unc_err_cnt),
4362 [C_PCIC_POST_HD_Q_UNC_ERR] = CNTR_ELEM("PcicPostHdQUncErr", 0, 0,
4363 CNTR_NORMAL,
4364 access_pcic_post_hd_q_unc_err_cnt),
4365 [C_PCIC_RETRY_SOT_MEM_UNC_ERR] = CNTR_ELEM("PcicRetrySotMemUncErr", 0, 0,
4366 CNTR_NORMAL,
4367 access_pcic_retry_sot_mem_unc_err_cnt),
4368 [C_PCIC_RETRY_MEM_UNC_ERR] = CNTR_ELEM("PcicRetryMemUncErr", 0, 0,
4369 CNTR_NORMAL,
4370 access_pcic_retry_mem_unc_err),
4371 [C_PCIC_N_POST_DAT_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostDatQParityErr", 0, 0,
4372 CNTR_NORMAL,
4373 access_pcic_n_post_dat_q_parity_err_cnt),
4374 [C_PCIC_N_POST_H_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostHQParityErr", 0, 0,
4375 CNTR_NORMAL,
4376 access_pcic_n_post_h_q_parity_err_cnt),
4377 [C_PCIC_CPL_DAT_Q_COR_ERR] = CNTR_ELEM("PcicCplDatQCorErr", 0, 0,
4378 CNTR_NORMAL,
4379 access_pcic_cpl_dat_q_cor_err_cnt),
4380 [C_PCIC_CPL_HD_Q_COR_ERR] = CNTR_ELEM("PcicCplHdQCorErr", 0, 0,
4381 CNTR_NORMAL,
4382 access_pcic_cpl_hd_q_cor_err_cnt),
4383 [C_PCIC_POST_DAT_Q_COR_ERR] = CNTR_ELEM("PcicPostDatQCorErr", 0, 0,
4384 CNTR_NORMAL,
4385 access_pcic_post_dat_q_cor_err_cnt),
4386 [C_PCIC_POST_HD_Q_COR_ERR] = CNTR_ELEM("PcicPostHdQCorErr", 0, 0,
4387 CNTR_NORMAL,
4388 access_pcic_post_hd_q_cor_err_cnt),
4389 [C_PCIC_RETRY_SOT_MEM_COR_ERR] = CNTR_ELEM("PcicRetrySotMemCorErr", 0, 0,
4390 CNTR_NORMAL,
4391 access_pcic_retry_sot_mem_cor_err_cnt),
4392 [C_PCIC_RETRY_MEM_COR_ERR] = CNTR_ELEM("PcicRetryMemCorErr", 0, 0,
4393 CNTR_NORMAL,
4394 access_pcic_retry_mem_cor_err_cnt),
4395 [C_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERR] = CNTR_ELEM(
4396 "CceCli1AsyncFifoDbgParityError", 0, 0,
4397 CNTR_NORMAL,
4398 access_cce_cli1_async_fifo_dbg_parity_err_cnt),
4399 [C_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERR] = CNTR_ELEM(
4400 "CceCli1AsyncFifoRxdmaParityError", 0, 0,
4401 CNTR_NORMAL,
4402 access_cce_cli1_async_fifo_rxdma_parity_err_cnt
4403 ),
4404 [C_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR] = CNTR_ELEM(
4405 "CceCli1AsyncFifoSdmaHdParityErr", 0, 0,
4406 CNTR_NORMAL,
4407 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt),
4408 [C_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR] = CNTR_ELEM(
4409 "CceCli1AsyncFifoPioCrdtParityErr", 0, 0,
4410 CNTR_NORMAL,
4411 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt),
4412 [C_CCE_CLI2_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceCli2AsyncFifoParityErr", 0,
4413 0, CNTR_NORMAL,
4414 access_cce_cli2_async_fifo_parity_err_cnt),
4415 [C_CCE_CSR_CFG_BUS_PARITY_ERR] = CNTR_ELEM("CceCsrCfgBusParityErr", 0, 0,
4416 CNTR_NORMAL,
4417 access_cce_csr_cfg_bus_parity_err_cnt),
4418 [C_CCE_CLI0_ASYNC_FIFO_PARTIY_ERR] = CNTR_ELEM("CceCli0AsyncFifoParityErr", 0,
4419 0, CNTR_NORMAL,
4420 access_cce_cli0_async_fifo_parity_err_cnt),
4421 [C_CCE_RSPD_DATA_PARITY_ERR] = CNTR_ELEM("CceRspdDataParityErr", 0, 0,
4422 CNTR_NORMAL,
4423 access_cce_rspd_data_parity_err_cnt),
4424 [C_CCE_TRGT_ACCESS_ERR] = CNTR_ELEM("CceTrgtAccessErr", 0, 0,
4425 CNTR_NORMAL,
4426 access_cce_trgt_access_err_cnt),
4427 [C_CCE_TRGT_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceTrgtAsyncFifoParityErr", 0,
4428 0, CNTR_NORMAL,
4429 access_cce_trgt_async_fifo_parity_err_cnt),
4430 [C_CCE_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrWriteBadAddrErr", 0, 0,
4431 CNTR_NORMAL,
4432 access_cce_csr_write_bad_addr_err_cnt),
4433 [C_CCE_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrReadBadAddrErr", 0, 0,
4434 CNTR_NORMAL,
4435 access_cce_csr_read_bad_addr_err_cnt),
4436 [C_CCE_CSR_PARITY_ERR] = CNTR_ELEM("CceCsrParityErr", 0, 0,
4437 CNTR_NORMAL,
4438 access_ccs_csr_parity_err_cnt),
4439
4440 /* RcvErrStatus */
4441 [C_RX_CSR_PARITY_ERR] = CNTR_ELEM("RxCsrParityErr", 0, 0,
4442 CNTR_NORMAL,
4443 access_rx_csr_parity_err_cnt),
4444 [C_RX_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrWriteBadAddrErr", 0, 0,
4445 CNTR_NORMAL,
4446 access_rx_csr_write_bad_addr_err_cnt),
4447 [C_RX_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrReadBadAddrErr", 0, 0,
4448 CNTR_NORMAL,
4449 access_rx_csr_read_bad_addr_err_cnt),
4450 [C_RX_DMA_CSR_UNC_ERR] = CNTR_ELEM("RxDmaCsrUncErr", 0, 0,
4451 CNTR_NORMAL,
4452 access_rx_dma_csr_unc_err_cnt),
4453 [C_RX_DMA_DQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaDqFsmEncodingErr", 0, 0,
4454 CNTR_NORMAL,
4455 access_rx_dma_dq_fsm_encoding_err_cnt),
4456 [C_RX_DMA_EQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaEqFsmEncodingErr", 0, 0,
4457 CNTR_NORMAL,
4458 access_rx_dma_eq_fsm_encoding_err_cnt),
4459 [C_RX_DMA_CSR_PARITY_ERR] = CNTR_ELEM("RxDmaCsrParityErr", 0, 0,
4460 CNTR_NORMAL,
4461 access_rx_dma_csr_parity_err_cnt),
4462 [C_RX_RBUF_DATA_COR_ERR] = CNTR_ELEM("RxRbufDataCorErr", 0, 0,
4463 CNTR_NORMAL,
4464 access_rx_rbuf_data_cor_err_cnt),
4465 [C_RX_RBUF_DATA_UNC_ERR] = CNTR_ELEM("RxRbufDataUncErr", 0, 0,
4466 CNTR_NORMAL,
4467 access_rx_rbuf_data_unc_err_cnt),
4468 [C_RX_DMA_DATA_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaDataFifoRdCorErr", 0, 0,
4469 CNTR_NORMAL,
4470 access_rx_dma_data_fifo_rd_cor_err_cnt),
4471 [C_RX_DMA_DATA_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaDataFifoRdUncErr", 0, 0,
4472 CNTR_NORMAL,
4473 access_rx_dma_data_fifo_rd_unc_err_cnt),
4474 [C_RX_DMA_HDR_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaHdrFifoRdCorErr", 0, 0,
4475 CNTR_NORMAL,
4476 access_rx_dma_hdr_fifo_rd_cor_err_cnt),
4477 [C_RX_DMA_HDR_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaHdrFifoRdUncErr", 0, 0,
4478 CNTR_NORMAL,
4479 access_rx_dma_hdr_fifo_rd_unc_err_cnt),
4480 [C_RX_RBUF_DESC_PART2_COR_ERR] = CNTR_ELEM("RxRbufDescPart2CorErr", 0, 0,
4481 CNTR_NORMAL,
4482 access_rx_rbuf_desc_part2_cor_err_cnt),
4483 [C_RX_RBUF_DESC_PART2_UNC_ERR] = CNTR_ELEM("RxRbufDescPart2UncErr", 0, 0,
4484 CNTR_NORMAL,
4485 access_rx_rbuf_desc_part2_unc_err_cnt),
4486 [C_RX_RBUF_DESC_PART1_COR_ERR] = CNTR_ELEM("RxRbufDescPart1CorErr", 0, 0,
4487 CNTR_NORMAL,
4488 access_rx_rbuf_desc_part1_cor_err_cnt),
4489 [C_RX_RBUF_DESC_PART1_UNC_ERR] = CNTR_ELEM("RxRbufDescPart1UncErr", 0, 0,
4490 CNTR_NORMAL,
4491 access_rx_rbuf_desc_part1_unc_err_cnt),
4492 [C_RX_HQ_INTR_FSM_ERR] = CNTR_ELEM("RxHqIntrFsmErr", 0, 0,
4493 CNTR_NORMAL,
4494 access_rx_hq_intr_fsm_err_cnt),
4495 [C_RX_HQ_INTR_CSR_PARITY_ERR] = CNTR_ELEM("RxHqIntrCsrParityErr", 0, 0,
4496 CNTR_NORMAL,
4497 access_rx_hq_intr_csr_parity_err_cnt),
4498 [C_RX_LOOKUP_CSR_PARITY_ERR] = CNTR_ELEM("RxLookupCsrParityErr", 0, 0,
4499 CNTR_NORMAL,
4500 access_rx_lookup_csr_parity_err_cnt),
4501 [C_RX_LOOKUP_RCV_ARRAY_COR_ERR] = CNTR_ELEM("RxLookupRcvArrayCorErr", 0, 0,
4502 CNTR_NORMAL,
4503 access_rx_lookup_rcv_array_cor_err_cnt),
4504 [C_RX_LOOKUP_RCV_ARRAY_UNC_ERR] = CNTR_ELEM("RxLookupRcvArrayUncErr", 0, 0,
4505 CNTR_NORMAL,
4506 access_rx_lookup_rcv_array_unc_err_cnt),
4507 [C_RX_LOOKUP_DES_PART2_PARITY_ERR] = CNTR_ELEM("RxLookupDesPart2ParityErr", 0,
4508 0, CNTR_NORMAL,
4509 access_rx_lookup_des_part2_parity_err_cnt),
4510 [C_RX_LOOKUP_DES_PART1_UNC_COR_ERR] = CNTR_ELEM("RxLookupDesPart1UncCorErr", 0,
4511 0, CNTR_NORMAL,
4512 access_rx_lookup_des_part1_unc_cor_err_cnt),
4513 [C_RX_LOOKUP_DES_PART1_UNC_ERR] = CNTR_ELEM("RxLookupDesPart1UncErr", 0, 0,
4514 CNTR_NORMAL,
4515 access_rx_lookup_des_part1_unc_err_cnt),
4516 [C_RX_RBUF_NEXT_FREE_BUF_COR_ERR] = CNTR_ELEM("RxRbufNextFreeBufCorErr", 0, 0,
4517 CNTR_NORMAL,
4518 access_rx_rbuf_next_free_buf_cor_err_cnt),
4519 [C_RX_RBUF_NEXT_FREE_BUF_UNC_ERR] = CNTR_ELEM("RxRbufNextFreeBufUncErr", 0, 0,
4520 CNTR_NORMAL,
4521 access_rx_rbuf_next_free_buf_unc_err_cnt),
4522 [C_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR] = CNTR_ELEM(
4523 "RxRbufFlInitWrAddrParityErr", 0, 0,
4524 CNTR_NORMAL,
4525 access_rbuf_fl_init_wr_addr_parity_err_cnt),
4526 [C_RX_RBUF_FL_INITDONE_PARITY_ERR] = CNTR_ELEM("RxRbufFlInitdoneParityErr", 0,
4527 0, CNTR_NORMAL,
4528 access_rx_rbuf_fl_initdone_parity_err_cnt),
4529 [C_RX_RBUF_FL_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlWrAddrParityErr", 0,
4530 0, CNTR_NORMAL,
4531 access_rx_rbuf_fl_write_addr_parity_err_cnt),
4532 [C_RX_RBUF_FL_RD_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlRdAddrParityErr", 0, 0,
4533 CNTR_NORMAL,
4534 access_rx_rbuf_fl_rd_addr_parity_err_cnt),
4535 [C_RX_RBUF_EMPTY_ERR] = CNTR_ELEM("RxRbufEmptyErr", 0, 0,
4536 CNTR_NORMAL,
4537 access_rx_rbuf_empty_err_cnt),
4538 [C_RX_RBUF_FULL_ERR] = CNTR_ELEM("RxRbufFullErr", 0, 0,
4539 CNTR_NORMAL,
4540 access_rx_rbuf_full_err_cnt),
4541 [C_RX_RBUF_BAD_LOOKUP_ERR] = CNTR_ELEM("RxRBufBadLookupErr", 0, 0,
4542 CNTR_NORMAL,
4543 access_rbuf_bad_lookup_err_cnt),
4544 [C_RX_RBUF_CTX_ID_PARITY_ERR] = CNTR_ELEM("RxRbufCtxIdParityErr", 0, 0,
4545 CNTR_NORMAL,
4546 access_rbuf_ctx_id_parity_err_cnt),
4547 [C_RX_RBUF_CSR_QEOPDW_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEOPDWParityErr", 0, 0,
4548 CNTR_NORMAL,
4549 access_rbuf_csr_qeopdw_parity_err_cnt),
4550 [C_RX_RBUF_CSR_Q_NUM_OF_PKT_PARITY_ERR] = CNTR_ELEM(
4551 "RxRbufCsrQNumOfPktParityErr", 0, 0,
4552 CNTR_NORMAL,
4553 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt),
4554 [C_RX_RBUF_CSR_Q_T1_PTR_PARITY_ERR] = CNTR_ELEM(
4555 "RxRbufCsrQTlPtrParityErr", 0, 0,
4556 CNTR_NORMAL,
4557 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt),
4558 [C_RX_RBUF_CSR_Q_HD_PTR_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQHdPtrParityErr", 0,
4559 0, CNTR_NORMAL,
4560 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt),
4561 [C_RX_RBUF_CSR_Q_VLD_BIT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQVldBitParityErr", 0,
4562 0, CNTR_NORMAL,
4563 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt),
4564 [C_RX_RBUF_CSR_Q_NEXT_BUF_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQNextBufParityErr",
4565 0, 0, CNTR_NORMAL,
4566 access_rx_rbuf_csr_q_next_buf_parity_err_cnt),
4567 [C_RX_RBUF_CSR_Q_ENT_CNT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEntCntParityErr", 0,
4568 0, CNTR_NORMAL,
4569 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt),
4570 [C_RX_RBUF_CSR_Q_HEAD_BUF_NUM_PARITY_ERR] = CNTR_ELEM(
4571 "RxRbufCsrQHeadBufNumParityErr", 0, 0,
4572 CNTR_NORMAL,
4573 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt),
4574 [C_RX_RBUF_BLOCK_LIST_READ_COR_ERR] = CNTR_ELEM("RxRbufBlockListReadCorErr", 0,
4575 0, CNTR_NORMAL,
4576 access_rx_rbuf_block_list_read_cor_err_cnt),
4577 [C_RX_RBUF_BLOCK_LIST_READ_UNC_ERR] = CNTR_ELEM("RxRbufBlockListReadUncErr", 0,
4578 0, CNTR_NORMAL,
4579 access_rx_rbuf_block_list_read_unc_err_cnt),
4580 [C_RX_RBUF_LOOKUP_DES_COR_ERR] = CNTR_ELEM("RxRbufLookupDesCorErr", 0, 0,
4581 CNTR_NORMAL,
4582 access_rx_rbuf_lookup_des_cor_err_cnt),
4583 [C_RX_RBUF_LOOKUP_DES_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesUncErr", 0, 0,
4584 CNTR_NORMAL,
4585 access_rx_rbuf_lookup_des_unc_err_cnt),
4586 [C_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR] = CNTR_ELEM(
4587 "RxRbufLookupDesRegUncCorErr", 0, 0,
4588 CNTR_NORMAL,
4589 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt),
4590 [C_RX_RBUF_LOOKUP_DES_REG_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesRegUncErr", 0, 0,
4591 CNTR_NORMAL,
4592 access_rx_rbuf_lookup_des_reg_unc_err_cnt),
4593 [C_RX_RBUF_FREE_LIST_COR_ERR] = CNTR_ELEM("RxRbufFreeListCorErr", 0, 0,
4594 CNTR_NORMAL,
4595 access_rx_rbuf_free_list_cor_err_cnt),
4596 [C_RX_RBUF_FREE_LIST_UNC_ERR] = CNTR_ELEM("RxRbufFreeListUncErr", 0, 0,
4597 CNTR_NORMAL,
4598 access_rx_rbuf_free_list_unc_err_cnt),
4599 [C_RX_RCV_FSM_ENCODING_ERR] = CNTR_ELEM("RxRcvFsmEncodingErr", 0, 0,
4600 CNTR_NORMAL,
4601 access_rx_rcv_fsm_encoding_err_cnt),
4602 [C_RX_DMA_FLAG_COR_ERR] = CNTR_ELEM("RxDmaFlagCorErr", 0, 0,
4603 CNTR_NORMAL,
4604 access_rx_dma_flag_cor_err_cnt),
4605 [C_RX_DMA_FLAG_UNC_ERR] = CNTR_ELEM("RxDmaFlagUncErr", 0, 0,
4606 CNTR_NORMAL,
4607 access_rx_dma_flag_unc_err_cnt),
4608 [C_RX_DC_SOP_EOP_PARITY_ERR] = CNTR_ELEM("RxDcSopEopParityErr", 0, 0,
4609 CNTR_NORMAL,
4610 access_rx_dc_sop_eop_parity_err_cnt),
4611 [C_RX_RCV_CSR_PARITY_ERR] = CNTR_ELEM("RxRcvCsrParityErr", 0, 0,
4612 CNTR_NORMAL,
4613 access_rx_rcv_csr_parity_err_cnt),
4614 [C_RX_RCV_QP_MAP_TABLE_COR_ERR] = CNTR_ELEM("RxRcvQpMapTableCorErr", 0, 0,
4615 CNTR_NORMAL,
4616 access_rx_rcv_qp_map_table_cor_err_cnt),
4617 [C_RX_RCV_QP_MAP_TABLE_UNC_ERR] = CNTR_ELEM("RxRcvQpMapTableUncErr", 0, 0,
4618 CNTR_NORMAL,
4619 access_rx_rcv_qp_map_table_unc_err_cnt),
4620 [C_RX_RCV_DATA_COR_ERR] = CNTR_ELEM("RxRcvDataCorErr", 0, 0,
4621 CNTR_NORMAL,
4622 access_rx_rcv_data_cor_err_cnt),
4623 [C_RX_RCV_DATA_UNC_ERR] = CNTR_ELEM("RxRcvDataUncErr", 0, 0,
4624 CNTR_NORMAL,
4625 access_rx_rcv_data_unc_err_cnt),
4626 [C_RX_RCV_HDR_COR_ERR] = CNTR_ELEM("RxRcvHdrCorErr", 0, 0,
4627 CNTR_NORMAL,
4628 access_rx_rcv_hdr_cor_err_cnt),
4629 [C_RX_RCV_HDR_UNC_ERR] = CNTR_ELEM("RxRcvHdrUncErr", 0, 0,
4630 CNTR_NORMAL,
4631 access_rx_rcv_hdr_unc_err_cnt),
4632 [C_RX_DC_INTF_PARITY_ERR] = CNTR_ELEM("RxDcIntfParityErr", 0, 0,
4633 CNTR_NORMAL,
4634 access_rx_dc_intf_parity_err_cnt),
4635 [C_RX_DMA_CSR_COR_ERR] = CNTR_ELEM("RxDmaCsrCorErr", 0, 0,
4636 CNTR_NORMAL,
4637 access_rx_dma_csr_cor_err_cnt),
4638 /* SendPioErrStatus */
4639 [C_PIO_PEC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPecSopHeadParityErr", 0, 0,
4640 CNTR_NORMAL,
4641 access_pio_pec_sop_head_parity_err_cnt),
4642 [C_PIO_PCC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPccSopHeadParityErr", 0, 0,
4643 CNTR_NORMAL,
4644 access_pio_pcc_sop_head_parity_err_cnt),
4645 [C_PIO_LAST_RETURNED_CNT_PARITY_ERR] = CNTR_ELEM("PioLastReturnedCntParityErr",
4646 0, 0, CNTR_NORMAL,
4647 access_pio_last_returned_cnt_parity_err_cnt),
4648 [C_PIO_CURRENT_FREE_CNT_PARITY_ERR] = CNTR_ELEM("PioCurrentFreeCntParityErr", 0,
4649 0, CNTR_NORMAL,
4650 access_pio_current_free_cnt_parity_err_cnt),
4651 [C_PIO_RSVD_31_ERR] = CNTR_ELEM("Pio Reserved 31", 0, 0,
4652 CNTR_NORMAL,
4653 access_pio_reserved_31_err_cnt),
4654 [C_PIO_RSVD_30_ERR] = CNTR_ELEM("Pio Reserved 30", 0, 0,
4655 CNTR_NORMAL,
4656 access_pio_reserved_30_err_cnt),
4657 [C_PIO_PPMC_SOP_LEN_ERR] = CNTR_ELEM("PioPpmcSopLenErr", 0, 0,
4658 CNTR_NORMAL,
4659 access_pio_ppmc_sop_len_err_cnt),
4660 [C_PIO_PPMC_BQC_MEM_PARITY_ERR] = CNTR_ELEM("PioPpmcBqcMemParityErr", 0, 0,
4661 CNTR_NORMAL,
4662 access_pio_ppmc_bqc_mem_parity_err_cnt),
4663 [C_PIO_VL_FIFO_PARITY_ERR] = CNTR_ELEM("PioVlFifoParityErr", 0, 0,
4664 CNTR_NORMAL,
4665 access_pio_vl_fifo_parity_err_cnt),
4666 [C_PIO_VLF_SOP_PARITY_ERR] = CNTR_ELEM("PioVlfSopParityErr", 0, 0,
4667 CNTR_NORMAL,
4668 access_pio_vlf_sop_parity_err_cnt),
4669 [C_PIO_VLF_V1_LEN_PARITY_ERR] = CNTR_ELEM("PioVlfVlLenParityErr", 0, 0,
4670 CNTR_NORMAL,
4671 access_pio_vlf_v1_len_parity_err_cnt),
4672 [C_PIO_BLOCK_QW_COUNT_PARITY_ERR] = CNTR_ELEM("PioBlockQwCountParityErr", 0, 0,
4673 CNTR_NORMAL,
4674 access_pio_block_qw_count_parity_err_cnt),
4675 [C_PIO_WRITE_QW_VALID_PARITY_ERR] = CNTR_ELEM("PioWriteQwValidParityErr", 0, 0,
4676 CNTR_NORMAL,
4677 access_pio_write_qw_valid_parity_err_cnt),
4678 [C_PIO_STATE_MACHINE_ERR] = CNTR_ELEM("PioStateMachineErr", 0, 0,
4679 CNTR_NORMAL,
4680 access_pio_state_machine_err_cnt),
4681 [C_PIO_WRITE_DATA_PARITY_ERR] = CNTR_ELEM("PioWriteDataParityErr", 0, 0,
4682 CNTR_NORMAL,
4683 access_pio_write_data_parity_err_cnt),
4684 [C_PIO_HOST_ADDR_MEM_COR_ERR] = CNTR_ELEM("PioHostAddrMemCorErr", 0, 0,
4685 CNTR_NORMAL,
4686 access_pio_host_addr_mem_cor_err_cnt),
4687 [C_PIO_HOST_ADDR_MEM_UNC_ERR] = CNTR_ELEM("PioHostAddrMemUncErr", 0, 0,
4688 CNTR_NORMAL,
4689 access_pio_host_addr_mem_unc_err_cnt),
4690 [C_PIO_PKT_EVICT_SM_OR_ARM_SM_ERR] = CNTR_ELEM("PioPktEvictSmOrArbSmErr", 0, 0,
4691 CNTR_NORMAL,
4692 access_pio_pkt_evict_sm_or_arb_sm_err_cnt),
4693 [C_PIO_INIT_SM_IN_ERR] = CNTR_ELEM("PioInitSmInErr", 0, 0,
4694 CNTR_NORMAL,
4695 access_pio_init_sm_in_err_cnt),
4696 [C_PIO_PPMC_PBL_FIFO_ERR] = CNTR_ELEM("PioPpmcPblFifoErr", 0, 0,
4697 CNTR_NORMAL,
4698 access_pio_ppmc_pbl_fifo_err_cnt),
4699 [C_PIO_CREDIT_RET_FIFO_PARITY_ERR] = CNTR_ELEM("PioCreditRetFifoParityErr", 0,
4700 0, CNTR_NORMAL,
4701 access_pio_credit_ret_fifo_parity_err_cnt),
4702 [C_PIO_V1_LEN_MEM_BANK1_COR_ERR] = CNTR_ELEM("PioVlLenMemBank1CorErr", 0, 0,
4703 CNTR_NORMAL,
4704 access_pio_v1_len_mem_bank1_cor_err_cnt),
4705 [C_PIO_V1_LEN_MEM_BANK0_COR_ERR] = CNTR_ELEM("PioVlLenMemBank0CorErr", 0, 0,
4706 CNTR_NORMAL,
4707 access_pio_v1_len_mem_bank0_cor_err_cnt),
4708 [C_PIO_V1_LEN_MEM_BANK1_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank1UncErr", 0, 0,
4709 CNTR_NORMAL,
4710 access_pio_v1_len_mem_bank1_unc_err_cnt),
4711 [C_PIO_V1_LEN_MEM_BANK0_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank0UncErr", 0, 0,
4712 CNTR_NORMAL,
4713 access_pio_v1_len_mem_bank0_unc_err_cnt),
4714 [C_PIO_SM_PKT_RESET_PARITY_ERR] = CNTR_ELEM("PioSmPktResetParityErr", 0, 0,
4715 CNTR_NORMAL,
4716 access_pio_sm_pkt_reset_parity_err_cnt),
4717 [C_PIO_PKT_EVICT_FIFO_PARITY_ERR] = CNTR_ELEM("PioPktEvictFifoParityErr", 0, 0,
4718 CNTR_NORMAL,
4719 access_pio_pkt_evict_fifo_parity_err_cnt),
4720 [C_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR] = CNTR_ELEM(
4721 "PioSbrdctrlCrrelFifoParityErr", 0, 0,
4722 CNTR_NORMAL,
4723 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt),
4724 [C_PIO_SBRDCTL_CRREL_PARITY_ERR] = CNTR_ELEM("PioSbrdctlCrrelParityErr", 0, 0,
4725 CNTR_NORMAL,
4726 access_pio_sbrdctl_crrel_parity_err_cnt),
4727 [C_PIO_PEC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPecFifoParityErr", 0, 0,
4728 CNTR_NORMAL,
4729 access_pio_pec_fifo_parity_err_cnt),
4730 [C_PIO_PCC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPccFifoParityErr", 0, 0,
4731 CNTR_NORMAL,
4732 access_pio_pcc_fifo_parity_err_cnt),
4733 [C_PIO_SB_MEM_FIFO1_ERR] = CNTR_ELEM("PioSbMemFifo1Err", 0, 0,
4734 CNTR_NORMAL,
4735 access_pio_sb_mem_fifo1_err_cnt),
4736 [C_PIO_SB_MEM_FIFO0_ERR] = CNTR_ELEM("PioSbMemFifo0Err", 0, 0,
4737 CNTR_NORMAL,
4738 access_pio_sb_mem_fifo0_err_cnt),
4739 [C_PIO_CSR_PARITY_ERR] = CNTR_ELEM("PioCsrParityErr", 0, 0,
4740 CNTR_NORMAL,
4741 access_pio_csr_parity_err_cnt),
4742 [C_PIO_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("PioWriteAddrParityErr", 0, 0,
4743 CNTR_NORMAL,
4744 access_pio_write_addr_parity_err_cnt),
4745 [C_PIO_WRITE_BAD_CTXT_ERR] = CNTR_ELEM("PioWriteBadCtxtErr", 0, 0,
4746 CNTR_NORMAL,
4747 access_pio_write_bad_ctxt_err_cnt),
4748 /* SendDmaErrStatus */
4749 [C_SDMA_PCIE_REQ_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPcieReqTrackingCorErr", 0,
4750 0, CNTR_NORMAL,
4751 access_sdma_pcie_req_tracking_cor_err_cnt),
4752 [C_SDMA_PCIE_REQ_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPcieReqTrackingUncErr", 0,
4753 0, CNTR_NORMAL,
4754 access_sdma_pcie_req_tracking_unc_err_cnt),
4755 [C_SDMA_CSR_PARITY_ERR] = CNTR_ELEM("SDmaCsrParityErr", 0, 0,
4756 CNTR_NORMAL,
4757 access_sdma_csr_parity_err_cnt),
4758 [C_SDMA_RPY_TAG_ERR] = CNTR_ELEM("SDmaRpyTagErr", 0, 0,
4759 CNTR_NORMAL,
4760 access_sdma_rpy_tag_err_cnt),
4761 /* SendEgressErrStatus */
4762 [C_TX_READ_PIO_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryCsrUncErr", 0, 0,
4763 CNTR_NORMAL,
4764 access_tx_read_pio_memory_csr_unc_err_cnt),
4765 [C_TX_READ_SDMA_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryCsrUncErr", 0,
4766 0, CNTR_NORMAL,
4767 access_tx_read_sdma_memory_csr_err_cnt),
4768 [C_TX_EGRESS_FIFO_COR_ERR] = CNTR_ELEM("TxEgressFifoCorErr", 0, 0,
4769 CNTR_NORMAL,
4770 access_tx_egress_fifo_cor_err_cnt),
4771 [C_TX_READ_PIO_MEMORY_COR_ERR] = CNTR_ELEM("TxReadPioMemoryCorErr", 0, 0,
4772 CNTR_NORMAL,
4773 access_tx_read_pio_memory_cor_err_cnt),
4774 [C_TX_READ_SDMA_MEMORY_COR_ERR] = CNTR_ELEM("TxReadSdmaMemoryCorErr", 0, 0,
4775 CNTR_NORMAL,
4776 access_tx_read_sdma_memory_cor_err_cnt),
4777 [C_TX_SB_HDR_COR_ERR] = CNTR_ELEM("TxSbHdrCorErr", 0, 0,
4778 CNTR_NORMAL,
4779 access_tx_sb_hdr_cor_err_cnt),
4780 [C_TX_CREDIT_OVERRUN_ERR] = CNTR_ELEM("TxCreditOverrunErr", 0, 0,
4781 CNTR_NORMAL,
4782 access_tx_credit_overrun_err_cnt),
4783 [C_TX_LAUNCH_FIFO8_COR_ERR] = CNTR_ELEM("TxLaunchFifo8CorErr", 0, 0,
4784 CNTR_NORMAL,
4785 access_tx_launch_fifo8_cor_err_cnt),
4786 [C_TX_LAUNCH_FIFO7_COR_ERR] = CNTR_ELEM("TxLaunchFifo7CorErr", 0, 0,
4787 CNTR_NORMAL,
4788 access_tx_launch_fifo7_cor_err_cnt),
4789 [C_TX_LAUNCH_FIFO6_COR_ERR] = CNTR_ELEM("TxLaunchFifo6CorErr", 0, 0,
4790 CNTR_NORMAL,
4791 access_tx_launch_fifo6_cor_err_cnt),
4792 [C_TX_LAUNCH_FIFO5_COR_ERR] = CNTR_ELEM("TxLaunchFifo5CorErr", 0, 0,
4793 CNTR_NORMAL,
4794 access_tx_launch_fifo5_cor_err_cnt),
4795 [C_TX_LAUNCH_FIFO4_COR_ERR] = CNTR_ELEM("TxLaunchFifo4CorErr", 0, 0,
4796 CNTR_NORMAL,
4797 access_tx_launch_fifo4_cor_err_cnt),
4798 [C_TX_LAUNCH_FIFO3_COR_ERR] = CNTR_ELEM("TxLaunchFifo3CorErr", 0, 0,
4799 CNTR_NORMAL,
4800 access_tx_launch_fifo3_cor_err_cnt),
4801 [C_TX_LAUNCH_FIFO2_COR_ERR] = CNTR_ELEM("TxLaunchFifo2CorErr", 0, 0,
4802 CNTR_NORMAL,
4803 access_tx_launch_fifo2_cor_err_cnt),
4804 [C_TX_LAUNCH_FIFO1_COR_ERR] = CNTR_ELEM("TxLaunchFifo1CorErr", 0, 0,
4805 CNTR_NORMAL,
4806 access_tx_launch_fifo1_cor_err_cnt),
4807 [C_TX_LAUNCH_FIFO0_COR_ERR] = CNTR_ELEM("TxLaunchFifo0CorErr", 0, 0,
4808 CNTR_NORMAL,
4809 access_tx_launch_fifo0_cor_err_cnt),
4810 [C_TX_CREDIT_RETURN_VL_ERR] = CNTR_ELEM("TxCreditReturnVLErr", 0, 0,
4811 CNTR_NORMAL,
4812 access_tx_credit_return_vl_err_cnt),
4813 [C_TX_HCRC_INSERTION_ERR] = CNTR_ELEM("TxHcrcInsertionErr", 0, 0,
4814 CNTR_NORMAL,
4815 access_tx_hcrc_insertion_err_cnt),
4816 [C_TX_EGRESS_FIFI_UNC_ERR] = CNTR_ELEM("TxEgressFifoUncErr", 0, 0,
4817 CNTR_NORMAL,
4818 access_tx_egress_fifo_unc_err_cnt),
4819 [C_TX_READ_PIO_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryUncErr", 0, 0,
4820 CNTR_NORMAL,
4821 access_tx_read_pio_memory_unc_err_cnt),
4822 [C_TX_READ_SDMA_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryUncErr", 0, 0,
4823 CNTR_NORMAL,
4824 access_tx_read_sdma_memory_unc_err_cnt),
4825 [C_TX_SB_HDR_UNC_ERR] = CNTR_ELEM("TxSbHdrUncErr", 0, 0,
4826 CNTR_NORMAL,
4827 access_tx_sb_hdr_unc_err_cnt),
4828 [C_TX_CREDIT_RETURN_PARITY_ERR] = CNTR_ELEM("TxCreditReturnParityErr", 0, 0,
4829 CNTR_NORMAL,
4830 access_tx_credit_return_partiy_err_cnt),
4831 [C_TX_LAUNCH_FIFO8_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo8UncOrParityErr",
4832 0, 0, CNTR_NORMAL,
4833 access_tx_launch_fifo8_unc_or_parity_err_cnt),
4834 [C_TX_LAUNCH_FIFO7_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo7UncOrParityErr",
4835 0, 0, CNTR_NORMAL,
4836 access_tx_launch_fifo7_unc_or_parity_err_cnt),
4837 [C_TX_LAUNCH_FIFO6_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo6UncOrParityErr",
4838 0, 0, CNTR_NORMAL,
4839 access_tx_launch_fifo6_unc_or_parity_err_cnt),
4840 [C_TX_LAUNCH_FIFO5_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo5UncOrParityErr",
4841 0, 0, CNTR_NORMAL,
4842 access_tx_launch_fifo5_unc_or_parity_err_cnt),
4843 [C_TX_LAUNCH_FIFO4_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo4UncOrParityErr",
4844 0, 0, CNTR_NORMAL,
4845 access_tx_launch_fifo4_unc_or_parity_err_cnt),
4846 [C_TX_LAUNCH_FIFO3_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo3UncOrParityErr",
4847 0, 0, CNTR_NORMAL,
4848 access_tx_launch_fifo3_unc_or_parity_err_cnt),
4849 [C_TX_LAUNCH_FIFO2_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo2UncOrParityErr",
4850 0, 0, CNTR_NORMAL,
4851 access_tx_launch_fifo2_unc_or_parity_err_cnt),
4852 [C_TX_LAUNCH_FIFO1_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo1UncOrParityErr",
4853 0, 0, CNTR_NORMAL,
4854 access_tx_launch_fifo1_unc_or_parity_err_cnt),
4855 [C_TX_LAUNCH_FIFO0_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo0UncOrParityErr",
4856 0, 0, CNTR_NORMAL,
4857 access_tx_launch_fifo0_unc_or_parity_err_cnt),
4858 [C_TX_SDMA15_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma15DisallowedPacketErr",
4859 0, 0, CNTR_NORMAL,
4860 access_tx_sdma15_disallowed_packet_err_cnt),
4861 [C_TX_SDMA14_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma14DisallowedPacketErr",
4862 0, 0, CNTR_NORMAL,
4863 access_tx_sdma14_disallowed_packet_err_cnt),
4864 [C_TX_SDMA13_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma13DisallowedPacketErr",
4865 0, 0, CNTR_NORMAL,
4866 access_tx_sdma13_disallowed_packet_err_cnt),
4867 [C_TX_SDMA12_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma12DisallowedPacketErr",
4868 0, 0, CNTR_NORMAL,
4869 access_tx_sdma12_disallowed_packet_err_cnt),
4870 [C_TX_SDMA11_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma11DisallowedPacketErr",
4871 0, 0, CNTR_NORMAL,
4872 access_tx_sdma11_disallowed_packet_err_cnt),
4873 [C_TX_SDMA10_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma10DisallowedPacketErr",
4874 0, 0, CNTR_NORMAL,
4875 access_tx_sdma10_disallowed_packet_err_cnt),
4876 [C_TX_SDMA9_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma9DisallowedPacketErr",
4877 0, 0, CNTR_NORMAL,
4878 access_tx_sdma9_disallowed_packet_err_cnt),
4879 [C_TX_SDMA8_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma8DisallowedPacketErr",
4880 0, 0, CNTR_NORMAL,
4881 access_tx_sdma8_disallowed_packet_err_cnt),
4882 [C_TX_SDMA7_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma7DisallowedPacketErr",
4883 0, 0, CNTR_NORMAL,
4884 access_tx_sdma7_disallowed_packet_err_cnt),
4885 [C_TX_SDMA6_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma6DisallowedPacketErr",
4886 0, 0, CNTR_NORMAL,
4887 access_tx_sdma6_disallowed_packet_err_cnt),
4888 [C_TX_SDMA5_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma5DisallowedPacketErr",
4889 0, 0, CNTR_NORMAL,
4890 access_tx_sdma5_disallowed_packet_err_cnt),
4891 [C_TX_SDMA4_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma4DisallowedPacketErr",
4892 0, 0, CNTR_NORMAL,
4893 access_tx_sdma4_disallowed_packet_err_cnt),
4894 [C_TX_SDMA3_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma3DisallowedPacketErr",
4895 0, 0, CNTR_NORMAL,
4896 access_tx_sdma3_disallowed_packet_err_cnt),
4897 [C_TX_SDMA2_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma2DisallowedPacketErr",
4898 0, 0, CNTR_NORMAL,
4899 access_tx_sdma2_disallowed_packet_err_cnt),
4900 [C_TX_SDMA1_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma1DisallowedPacketErr",
4901 0, 0, CNTR_NORMAL,
4902 access_tx_sdma1_disallowed_packet_err_cnt),
4903 [C_TX_SDMA0_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma0DisallowedPacketErr",
4904 0, 0, CNTR_NORMAL,
4905 access_tx_sdma0_disallowed_packet_err_cnt),
4906 [C_TX_CONFIG_PARITY_ERR] = CNTR_ELEM("TxConfigParityErr", 0, 0,
4907 CNTR_NORMAL,
4908 access_tx_config_parity_err_cnt),
4909 [C_TX_SBRD_CTL_CSR_PARITY_ERR] = CNTR_ELEM("TxSbrdCtlCsrParityErr", 0, 0,
4910 CNTR_NORMAL,
4911 access_tx_sbrd_ctl_csr_parity_err_cnt),
4912 [C_TX_LAUNCH_CSR_PARITY_ERR] = CNTR_ELEM("TxLaunchCsrParityErr", 0, 0,
4913 CNTR_NORMAL,
4914 access_tx_launch_csr_parity_err_cnt),
4915 [C_TX_ILLEGAL_CL_ERR] = CNTR_ELEM("TxIllegalVLErr", 0, 0,
4916 CNTR_NORMAL,
4917 access_tx_illegal_vl_err_cnt),
4918 [C_TX_SBRD_CTL_STATE_MACHINE_PARITY_ERR] = CNTR_ELEM(
4919 "TxSbrdCtlStateMachineParityErr", 0, 0,
4920 CNTR_NORMAL,
4921 access_tx_sbrd_ctl_state_machine_parity_err_cnt),
4922 [C_TX_RESERVED_10] = CNTR_ELEM("Tx Egress Reserved 10", 0, 0,
4923 CNTR_NORMAL,
4924 access_egress_reserved_10_err_cnt),
4925 [C_TX_RESERVED_9] = CNTR_ELEM("Tx Egress Reserved 9", 0, 0,
4926 CNTR_NORMAL,
4927 access_egress_reserved_9_err_cnt),
4928 [C_TX_SDMA_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxSdmaLaunchIntfParityErr",
4929 0, 0, CNTR_NORMAL,
4930 access_tx_sdma_launch_intf_parity_err_cnt),
4931 [C_TX_PIO_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxPioLaunchIntfParityErr", 0, 0,
4932 CNTR_NORMAL,
4933 access_tx_pio_launch_intf_parity_err_cnt),
4934 [C_TX_RESERVED_6] = CNTR_ELEM("Tx Egress Reserved 6", 0, 0,
4935 CNTR_NORMAL,
4936 access_egress_reserved_6_err_cnt),
4937 [C_TX_INCORRECT_LINK_STATE_ERR] = CNTR_ELEM("TxIncorrectLinkStateErr", 0, 0,
4938 CNTR_NORMAL,
4939 access_tx_incorrect_link_state_err_cnt),
4940 [C_TX_LINK_DOWN_ERR] = CNTR_ELEM("TxLinkdownErr", 0, 0,
4941 CNTR_NORMAL,
4942 access_tx_linkdown_err_cnt),
4943 [C_TX_EGRESS_FIFO_UNDERRUN_OR_PARITY_ERR] = CNTR_ELEM(
4944 "EgressFifoUnderrunOrParityErr", 0, 0,
4945 CNTR_NORMAL,
4946 access_tx_egress_fifi_underrun_or_parity_err_cnt),
4947 [C_TX_RESERVED_2] = CNTR_ELEM("Tx Egress Reserved 2", 0, 0,
4948 CNTR_NORMAL,
4949 access_egress_reserved_2_err_cnt),
4950 [C_TX_PKT_INTEGRITY_MEM_UNC_ERR] = CNTR_ELEM("TxPktIntegrityMemUncErr", 0, 0,
4951 CNTR_NORMAL,
4952 access_tx_pkt_integrity_mem_unc_err_cnt),
4953 [C_TX_PKT_INTEGRITY_MEM_COR_ERR] = CNTR_ELEM("TxPktIntegrityMemCorErr", 0, 0,
4954 CNTR_NORMAL,
4955 access_tx_pkt_integrity_mem_cor_err_cnt),
4956 /* SendErrStatus */
4957 [C_SEND_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("SendCsrWriteBadAddrErr", 0, 0,
4958 CNTR_NORMAL,
4959 access_send_csr_write_bad_addr_err_cnt),
4960 [C_SEND_CSR_READ_BAD_ADD_ERR] = CNTR_ELEM("SendCsrReadBadAddrErr", 0, 0,
4961 CNTR_NORMAL,
4962 access_send_csr_read_bad_addr_err_cnt),
4963 [C_SEND_CSR_PARITY_ERR] = CNTR_ELEM("SendCsrParityErr", 0, 0,
4964 CNTR_NORMAL,
4965 access_send_csr_parity_cnt),
4966 /* SendCtxtErrStatus */
4967 [C_PIO_WRITE_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("PioWriteOutOfBoundsErr", 0, 0,
4968 CNTR_NORMAL,
4969 access_pio_write_out_of_bounds_err_cnt),
4970 [C_PIO_WRITE_OVERFLOW_ERR] = CNTR_ELEM("PioWriteOverflowErr", 0, 0,
4971 CNTR_NORMAL,
4972 access_pio_write_overflow_err_cnt),
4973 [C_PIO_WRITE_CROSSES_BOUNDARY_ERR] = CNTR_ELEM("PioWriteCrossesBoundaryErr",
4974 0, 0, CNTR_NORMAL,
4975 access_pio_write_crosses_boundary_err_cnt),
4976 [C_PIO_DISALLOWED_PACKET_ERR] = CNTR_ELEM("PioDisallowedPacketErr", 0, 0,
4977 CNTR_NORMAL,
4978 access_pio_disallowed_packet_err_cnt),
4979 [C_PIO_INCONSISTENT_SOP_ERR] = CNTR_ELEM("PioInconsistentSopErr", 0, 0,
4980 CNTR_NORMAL,
4981 access_pio_inconsistent_sop_err_cnt),
4982 /* SendDmaEngErrStatus */
4983 [C_SDMA_HEADER_REQUEST_FIFO_COR_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoCorErr",
4984 0, 0, CNTR_NORMAL,
4985 access_sdma_header_request_fifo_cor_err_cnt),
4986 [C_SDMA_HEADER_STORAGE_COR_ERR] = CNTR_ELEM("SDmaHeaderStorageCorErr", 0, 0,
4987 CNTR_NORMAL,
4988 access_sdma_header_storage_cor_err_cnt),
4989 [C_SDMA_PACKET_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPacketTrackingCorErr", 0, 0,
4990 CNTR_NORMAL,
4991 access_sdma_packet_tracking_cor_err_cnt),
4992 [C_SDMA_ASSEMBLY_COR_ERR] = CNTR_ELEM("SDmaAssemblyCorErr", 0, 0,
4993 CNTR_NORMAL,
4994 access_sdma_assembly_cor_err_cnt),
4995 [C_SDMA_DESC_TABLE_COR_ERR] = CNTR_ELEM("SDmaDescTableCorErr", 0, 0,
4996 CNTR_NORMAL,
4997 access_sdma_desc_table_cor_err_cnt),
4998 [C_SDMA_HEADER_REQUEST_FIFO_UNC_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoUncErr",
4999 0, 0, CNTR_NORMAL,
5000 access_sdma_header_request_fifo_unc_err_cnt),
5001 [C_SDMA_HEADER_STORAGE_UNC_ERR] = CNTR_ELEM("SDmaHeaderStorageUncErr", 0, 0,
5002 CNTR_NORMAL,
5003 access_sdma_header_storage_unc_err_cnt),
5004 [C_SDMA_PACKET_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPacketTrackingUncErr", 0, 0,
5005 CNTR_NORMAL,
5006 access_sdma_packet_tracking_unc_err_cnt),
5007 [C_SDMA_ASSEMBLY_UNC_ERR] = CNTR_ELEM("SDmaAssemblyUncErr", 0, 0,
5008 CNTR_NORMAL,
5009 access_sdma_assembly_unc_err_cnt),
5010 [C_SDMA_DESC_TABLE_UNC_ERR] = CNTR_ELEM("SDmaDescTableUncErr", 0, 0,
5011 CNTR_NORMAL,
5012 access_sdma_desc_table_unc_err_cnt),
5013 [C_SDMA_TIMEOUT_ERR] = CNTR_ELEM("SDmaTimeoutErr", 0, 0,
5014 CNTR_NORMAL,
5015 access_sdma_timeout_err_cnt),
5016 [C_SDMA_HEADER_LENGTH_ERR] = CNTR_ELEM("SDmaHeaderLengthErr", 0, 0,
5017 CNTR_NORMAL,
5018 access_sdma_header_length_err_cnt),
5019 [C_SDMA_HEADER_ADDRESS_ERR] = CNTR_ELEM("SDmaHeaderAddressErr", 0, 0,
5020 CNTR_NORMAL,
5021 access_sdma_header_address_err_cnt),
5022 [C_SDMA_HEADER_SELECT_ERR] = CNTR_ELEM("SDmaHeaderSelectErr", 0, 0,
5023 CNTR_NORMAL,
5024 access_sdma_header_select_err_cnt),
5025 [C_SMDA_RESERVED_9] = CNTR_ELEM("SDma Reserved 9", 0, 0,
5026 CNTR_NORMAL,
5027 access_sdma_reserved_9_err_cnt),
5028 [C_SDMA_PACKET_DESC_OVERFLOW_ERR] = CNTR_ELEM("SDmaPacketDescOverflowErr", 0, 0,
5029 CNTR_NORMAL,
5030 access_sdma_packet_desc_overflow_err_cnt),
5031 [C_SDMA_LENGTH_MISMATCH_ERR] = CNTR_ELEM("SDmaLengthMismatchErr", 0, 0,
5032 CNTR_NORMAL,
5033 access_sdma_length_mismatch_err_cnt),
5034 [C_SDMA_HALT_ERR] = CNTR_ELEM("SDmaHaltErr", 0, 0,
5035 CNTR_NORMAL,
5036 access_sdma_halt_err_cnt),
5037 [C_SDMA_MEM_READ_ERR] = CNTR_ELEM("SDmaMemReadErr", 0, 0,
5038 CNTR_NORMAL,
5039 access_sdma_mem_read_err_cnt),
5040 [C_SDMA_FIRST_DESC_ERR] = CNTR_ELEM("SDmaFirstDescErr", 0, 0,
5041 CNTR_NORMAL,
5042 access_sdma_first_desc_err_cnt),
5043 [C_SDMA_TAIL_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("SDmaTailOutOfBoundsErr", 0, 0,
5044 CNTR_NORMAL,
5045 access_sdma_tail_out_of_bounds_err_cnt),
5046 [C_SDMA_TOO_LONG_ERR] = CNTR_ELEM("SDmaTooLongErr", 0, 0,
5047 CNTR_NORMAL,
5048 access_sdma_too_long_err_cnt),
5049 [C_SDMA_GEN_MISMATCH_ERR] = CNTR_ELEM("SDmaGenMismatchErr", 0, 0,
5050 CNTR_NORMAL,
5051 access_sdma_gen_mismatch_err_cnt),
5052 [C_SDMA_WRONG_DW_ERR] = CNTR_ELEM("SDmaWrongDwErr", 0, 0,
5053 CNTR_NORMAL,
5054 access_sdma_wrong_dw_err_cnt),
5055 };
5056
5057 static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = {
5058 [C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT,
5059 CNTR_NORMAL),
5060 [C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT,
5061 CNTR_NORMAL),
5062 [C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT,
5063 CNTR_NORMAL),
5064 [C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT,
5065 CNTR_NORMAL),
5066 [C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT,
5067 CNTR_NORMAL),
5068 [C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT,
5069 CNTR_NORMAL),
5070 [C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT,
5071 CNTR_NORMAL),
5072 [C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL),
5073 [C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL),
5074 [C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH),
5075 [C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT,
5076 CNTR_SYNTH | CNTR_VL),
5077 [C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT,
5078 CNTR_SYNTH | CNTR_VL),
5079 [C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT,
5080 CNTR_SYNTH | CNTR_VL),
5081 [C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL),
5082 [C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL),
5083 [C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5084 access_sw_link_dn_cnt),
5085 [C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5086 access_sw_link_up_cnt),
5087 [C_SW_UNKNOWN_FRAME] = CNTR_ELEM("UnknownFrame", 0, 0, CNTR_NORMAL,
5088 access_sw_unknown_frame_cnt),
5089 [C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5090 access_sw_xmit_discards),
5091 [C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0,
5092 CNTR_SYNTH | CNTR_32BIT | CNTR_VL,
5093 access_sw_xmit_discards),
5094 [C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH,
5095 access_xmit_constraint_errs),
5096 [C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH,
5097 access_rcv_constraint_errs),
5098 [C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts),
5099 [C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends),
5100 [C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks),
5101 [C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks),
5102 [C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts),
5103 [C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops),
5104 [C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait),
5105 [C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak),
5106 [C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq),
5107 [C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq),
5108 [C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned),
5109 [C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks),
5110 [C_SW_IBP_RC_CRWAITS] = SW_IBP_CNTR(RcCrWait, rc_crwaits),
5111 [C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL,
5112 access_sw_cpu_rc_acks),
5113 [C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL,
5114 access_sw_cpu_rc_qacks),
5115 [C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL,
5116 access_sw_cpu_rc_delayed_comp),
5117 [OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1),
5118 [OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3),
5119 [OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5),
5120 [OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7),
5121 [OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9),
5122 [OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11),
5123 [OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13),
5124 [OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15),
5125 [OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17),
5126 [OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19),
5127 [OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21),
5128 [OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23),
5129 [OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25),
5130 [OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27),
5131 [OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29),
5132 [OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31),
5133 [OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33),
5134 [OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35),
5135 [OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37),
5136 [OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39),
5137 [OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41),
5138 [OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43),
5139 [OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45),
5140 [OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47),
5141 [OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49),
5142 [OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51),
5143 [OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53),
5144 [OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55),
5145 [OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57),
5146 [OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59),
5147 [OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61),
5148 [OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63),
5149 [OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65),
5150 [OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67),
5151 [OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69),
5152 [OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71),
5153 [OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73),
5154 [OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75),
5155 [OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77),
5156 [OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79),
5157 [OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81),
5158 [OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83),
5159 [OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85),
5160 [OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87),
5161 [OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89),
5162 [OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91),
5163 [OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93),
5164 [OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95),
5165 [OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97),
5166 [OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99),
5167 [OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101),
5168 [OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103),
5169 [OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105),
5170 [OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107),
5171 [OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109),
5172 [OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111),
5173 [OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113),
5174 [OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115),
5175 [OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117),
5176 [OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119),
5177 [OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121),
5178 [OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123),
5179 [OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125),
5180 [OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127),
5181 [OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129),
5182 [OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131),
5183 [OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133),
5184 [OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135),
5185 [OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137),
5186 [OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139),
5187 [OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141),
5188 [OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143),
5189 [OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145),
5190 [OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147),
5191 [OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149),
5192 [OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151),
5193 [OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153),
5194 [OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155),
5195 [OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157),
5196 [OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159),
5197 };
5198
5199 /* ======================================================================== */
5200
5201 /* return true if this is chip revision revision a */
is_ax(struct hfi1_devdata * dd)5202 int is_ax(struct hfi1_devdata *dd)
5203 {
5204 u8 chip_rev_minor =
5205 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5206 & CCE_REVISION_CHIP_REV_MINOR_MASK;
5207 return (chip_rev_minor & 0xf0) == 0;
5208 }
5209
5210 /* return true if this is chip revision revision b */
is_bx(struct hfi1_devdata * dd)5211 int is_bx(struct hfi1_devdata *dd)
5212 {
5213 u8 chip_rev_minor =
5214 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5215 & CCE_REVISION_CHIP_REV_MINOR_MASK;
5216 return (chip_rev_minor & 0xF0) == 0x10;
5217 }
5218
5219 /* return true is kernel urg disabled for rcd */
is_urg_masked(struct hfi1_ctxtdata * rcd)5220 bool is_urg_masked(struct hfi1_ctxtdata *rcd)
5221 {
5222 u64 mask;
5223 u32 is = IS_RCVURGENT_START + rcd->ctxt;
5224 u8 bit = is % 64;
5225
5226 mask = read_csr(rcd->dd, CCE_INT_MASK + (8 * (is / 64)));
5227 return !(mask & BIT_ULL(bit));
5228 }
5229
5230 /*
5231 * Append string s to buffer buf. Arguments curp and len are the current
5232 * position and remaining length, respectively.
5233 *
5234 * return 0 on success, 1 on out of room
5235 */
append_str(char * buf,char ** curp,int * lenp,const char * s)5236 static int append_str(char *buf, char **curp, int *lenp, const char *s)
5237 {
5238 char *p = *curp;
5239 int len = *lenp;
5240 int result = 0; /* success */
5241 char c;
5242
5243 /* add a comma, if first in the buffer */
5244 if (p != buf) {
5245 if (len == 0) {
5246 result = 1; /* out of room */
5247 goto done;
5248 }
5249 *p++ = ',';
5250 len--;
5251 }
5252
5253 /* copy the string */
5254 while ((c = *s++) != 0) {
5255 if (len == 0) {
5256 result = 1; /* out of room */
5257 goto done;
5258 }
5259 *p++ = c;
5260 len--;
5261 }
5262
5263 done:
5264 /* write return values */
5265 *curp = p;
5266 *lenp = len;
5267
5268 return result;
5269 }
5270
5271 /*
5272 * Using the given flag table, print a comma separated string into
5273 * the buffer. End in '*' if the buffer is too short.
5274 */
flag_string(char * buf,int buf_len,u64 flags,struct flag_table * table,int table_size)5275 static char *flag_string(char *buf, int buf_len, u64 flags,
5276 struct flag_table *table, int table_size)
5277 {
5278 char extra[32];
5279 char *p = buf;
5280 int len = buf_len;
5281 int no_room = 0;
5282 int i;
5283
5284 /* make sure there is at least 2 so we can form "*" */
5285 if (len < 2)
5286 return "";
5287
5288 len--; /* leave room for a nul */
5289 for (i = 0; i < table_size; i++) {
5290 if (flags & table[i].flag) {
5291 no_room = append_str(buf, &p, &len, table[i].str);
5292 if (no_room)
5293 break;
5294 flags &= ~table[i].flag;
5295 }
5296 }
5297
5298 /* any undocumented bits left? */
5299 if (!no_room && flags) {
5300 snprintf(extra, sizeof(extra), "bits 0x%llx", flags);
5301 no_room = append_str(buf, &p, &len, extra);
5302 }
5303
5304 /* add * if ran out of room */
5305 if (no_room) {
5306 /* may need to back up to add space for a '*' */
5307 if (len == 0)
5308 --p;
5309 *p++ = '*';
5310 }
5311
5312 /* add final nul - space already allocated above */
5313 *p = 0;
5314 return buf;
5315 }
5316
5317 /* first 8 CCE error interrupt source names */
5318 static const char * const cce_misc_names[] = {
5319 "CceErrInt", /* 0 */
5320 "RxeErrInt", /* 1 */
5321 "MiscErrInt", /* 2 */
5322 "Reserved3", /* 3 */
5323 "PioErrInt", /* 4 */
5324 "SDmaErrInt", /* 5 */
5325 "EgressErrInt", /* 6 */
5326 "TxeErrInt" /* 7 */
5327 };
5328
5329 /*
5330 * Return the miscellaneous error interrupt name.
5331 */
is_misc_err_name(char * buf,size_t bsize,unsigned int source)5332 static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source)
5333 {
5334 if (source < ARRAY_SIZE(cce_misc_names))
5335 strncpy(buf, cce_misc_names[source], bsize);
5336 else
5337 snprintf(buf, bsize, "Reserved%u",
5338 source + IS_GENERAL_ERR_START);
5339
5340 return buf;
5341 }
5342
5343 /*
5344 * Return the SDMA engine error interrupt name.
5345 */
is_sdma_eng_err_name(char * buf,size_t bsize,unsigned int source)5346 static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source)
5347 {
5348 snprintf(buf, bsize, "SDmaEngErrInt%u", source);
5349 return buf;
5350 }
5351
5352 /*
5353 * Return the send context error interrupt name.
5354 */
is_sendctxt_err_name(char * buf,size_t bsize,unsigned int source)5355 static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source)
5356 {
5357 snprintf(buf, bsize, "SendCtxtErrInt%u", source);
5358 return buf;
5359 }
5360
5361 static const char * const various_names[] = {
5362 "PbcInt",
5363 "GpioAssertInt",
5364 "Qsfp1Int",
5365 "Qsfp2Int",
5366 "TCritInt"
5367 };
5368
5369 /*
5370 * Return the various interrupt name.
5371 */
is_various_name(char * buf,size_t bsize,unsigned int source)5372 static char *is_various_name(char *buf, size_t bsize, unsigned int source)
5373 {
5374 if (source < ARRAY_SIZE(various_names))
5375 strncpy(buf, various_names[source], bsize);
5376 else
5377 snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START);
5378 return buf;
5379 }
5380
5381 /*
5382 * Return the DC interrupt name.
5383 */
is_dc_name(char * buf,size_t bsize,unsigned int source)5384 static char *is_dc_name(char *buf, size_t bsize, unsigned int source)
5385 {
5386 static const char * const dc_int_names[] = {
5387 "common",
5388 "lcb",
5389 "8051",
5390 "lbm" /* local block merge */
5391 };
5392
5393 if (source < ARRAY_SIZE(dc_int_names))
5394 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]);
5395 else
5396 snprintf(buf, bsize, "DCInt%u", source);
5397 return buf;
5398 }
5399
5400 static const char * const sdma_int_names[] = {
5401 "SDmaInt",
5402 "SdmaIdleInt",
5403 "SdmaProgressInt",
5404 };
5405
5406 /*
5407 * Return the SDMA engine interrupt name.
5408 */
is_sdma_eng_name(char * buf,size_t bsize,unsigned int source)5409 static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source)
5410 {
5411 /* what interrupt */
5412 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
5413 /* which engine */
5414 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
5415
5416 if (likely(what < 3))
5417 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which);
5418 else
5419 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source);
5420 return buf;
5421 }
5422
5423 /*
5424 * Return the receive available interrupt name.
5425 */
is_rcv_avail_name(char * buf,size_t bsize,unsigned int source)5426 static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source)
5427 {
5428 snprintf(buf, bsize, "RcvAvailInt%u", source);
5429 return buf;
5430 }
5431
5432 /*
5433 * Return the receive urgent interrupt name.
5434 */
is_rcv_urgent_name(char * buf,size_t bsize,unsigned int source)5435 static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source)
5436 {
5437 snprintf(buf, bsize, "RcvUrgentInt%u", source);
5438 return buf;
5439 }
5440
5441 /*
5442 * Return the send credit interrupt name.
5443 */
is_send_credit_name(char * buf,size_t bsize,unsigned int source)5444 static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source)
5445 {
5446 snprintf(buf, bsize, "SendCreditInt%u", source);
5447 return buf;
5448 }
5449
5450 /*
5451 * Return the reserved interrupt name.
5452 */
is_reserved_name(char * buf,size_t bsize,unsigned int source)5453 static char *is_reserved_name(char *buf, size_t bsize, unsigned int source)
5454 {
5455 snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START);
5456 return buf;
5457 }
5458
cce_err_status_string(char * buf,int buf_len,u64 flags)5459 static char *cce_err_status_string(char *buf, int buf_len, u64 flags)
5460 {
5461 return flag_string(buf, buf_len, flags,
5462 cce_err_status_flags,
5463 ARRAY_SIZE(cce_err_status_flags));
5464 }
5465
rxe_err_status_string(char * buf,int buf_len,u64 flags)5466 static char *rxe_err_status_string(char *buf, int buf_len, u64 flags)
5467 {
5468 return flag_string(buf, buf_len, flags,
5469 rxe_err_status_flags,
5470 ARRAY_SIZE(rxe_err_status_flags));
5471 }
5472
misc_err_status_string(char * buf,int buf_len,u64 flags)5473 static char *misc_err_status_string(char *buf, int buf_len, u64 flags)
5474 {
5475 return flag_string(buf, buf_len, flags, misc_err_status_flags,
5476 ARRAY_SIZE(misc_err_status_flags));
5477 }
5478
pio_err_status_string(char * buf,int buf_len,u64 flags)5479 static char *pio_err_status_string(char *buf, int buf_len, u64 flags)
5480 {
5481 return flag_string(buf, buf_len, flags,
5482 pio_err_status_flags,
5483 ARRAY_SIZE(pio_err_status_flags));
5484 }
5485
sdma_err_status_string(char * buf,int buf_len,u64 flags)5486 static char *sdma_err_status_string(char *buf, int buf_len, u64 flags)
5487 {
5488 return flag_string(buf, buf_len, flags,
5489 sdma_err_status_flags,
5490 ARRAY_SIZE(sdma_err_status_flags));
5491 }
5492
egress_err_status_string(char * buf,int buf_len,u64 flags)5493 static char *egress_err_status_string(char *buf, int buf_len, u64 flags)
5494 {
5495 return flag_string(buf, buf_len, flags,
5496 egress_err_status_flags,
5497 ARRAY_SIZE(egress_err_status_flags));
5498 }
5499
egress_err_info_string(char * buf,int buf_len,u64 flags)5500 static char *egress_err_info_string(char *buf, int buf_len, u64 flags)
5501 {
5502 return flag_string(buf, buf_len, flags,
5503 egress_err_info_flags,
5504 ARRAY_SIZE(egress_err_info_flags));
5505 }
5506
send_err_status_string(char * buf,int buf_len,u64 flags)5507 static char *send_err_status_string(char *buf, int buf_len, u64 flags)
5508 {
5509 return flag_string(buf, buf_len, flags,
5510 send_err_status_flags,
5511 ARRAY_SIZE(send_err_status_flags));
5512 }
5513
handle_cce_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5514 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5515 {
5516 char buf[96];
5517 int i = 0;
5518
5519 /*
5520 * For most these errors, there is nothing that can be done except
5521 * report or record it.
5522 */
5523 dd_dev_info(dd, "CCE Error: %s\n",
5524 cce_err_status_string(buf, sizeof(buf), reg));
5525
5526 if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) &&
5527 is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) {
5528 /* this error requires a manual drop into SPC freeze mode */
5529 /* then a fix up */
5530 start_freeze_handling(dd->pport, FREEZE_SELF);
5531 }
5532
5533 for (i = 0; i < NUM_CCE_ERR_STATUS_COUNTERS; i++) {
5534 if (reg & (1ull << i)) {
5535 incr_cntr64(&dd->cce_err_status_cnt[i]);
5536 /* maintain a counter over all cce_err_status errors */
5537 incr_cntr64(&dd->sw_cce_err_status_aggregate);
5538 }
5539 }
5540 }
5541
5542 /*
5543 * Check counters for receive errors that do not have an interrupt
5544 * associated with them.
5545 */
5546 #define RCVERR_CHECK_TIME 10
update_rcverr_timer(struct timer_list * t)5547 static void update_rcverr_timer(struct timer_list *t)
5548 {
5549 struct hfi1_devdata *dd = from_timer(dd, t, rcverr_timer);
5550 struct hfi1_pportdata *ppd = dd->pport;
5551 u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL);
5552
5553 if (dd->rcv_ovfl_cnt < cur_ovfl_cnt &&
5554 ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) {
5555 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
5556 set_link_down_reason(
5557 ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0,
5558 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN);
5559 queue_work(ppd->link_wq, &ppd->link_bounce_work);
5560 }
5561 dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt;
5562
5563 mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5564 }
5565
init_rcverr(struct hfi1_devdata * dd)5566 static int init_rcverr(struct hfi1_devdata *dd)
5567 {
5568 timer_setup(&dd->rcverr_timer, update_rcverr_timer, 0);
5569 /* Assume the hardware counter has been reset */
5570 dd->rcv_ovfl_cnt = 0;
5571 return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5572 }
5573
free_rcverr(struct hfi1_devdata * dd)5574 static void free_rcverr(struct hfi1_devdata *dd)
5575 {
5576 if (dd->rcverr_timer.function)
5577 del_timer_sync(&dd->rcverr_timer);
5578 }
5579
handle_rxe_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5580 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5581 {
5582 char buf[96];
5583 int i = 0;
5584
5585 dd_dev_info(dd, "Receive Error: %s\n",
5586 rxe_err_status_string(buf, sizeof(buf), reg));
5587
5588 if (reg & ALL_RXE_FREEZE_ERR) {
5589 int flags = 0;
5590
5591 /*
5592 * Freeze mode recovery is disabled for the errors
5593 * in RXE_FREEZE_ABORT_MASK
5594 */
5595 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK))
5596 flags = FREEZE_ABORT;
5597
5598 start_freeze_handling(dd->pport, flags);
5599 }
5600
5601 for (i = 0; i < NUM_RCV_ERR_STATUS_COUNTERS; i++) {
5602 if (reg & (1ull << i))
5603 incr_cntr64(&dd->rcv_err_status_cnt[i]);
5604 }
5605 }
5606
handle_misc_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5607 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5608 {
5609 char buf[96];
5610 int i = 0;
5611
5612 dd_dev_info(dd, "Misc Error: %s",
5613 misc_err_status_string(buf, sizeof(buf), reg));
5614 for (i = 0; i < NUM_MISC_ERR_STATUS_COUNTERS; i++) {
5615 if (reg & (1ull << i))
5616 incr_cntr64(&dd->misc_err_status_cnt[i]);
5617 }
5618 }
5619
handle_pio_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5620 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5621 {
5622 char buf[96];
5623 int i = 0;
5624
5625 dd_dev_info(dd, "PIO Error: %s\n",
5626 pio_err_status_string(buf, sizeof(buf), reg));
5627
5628 if (reg & ALL_PIO_FREEZE_ERR)
5629 start_freeze_handling(dd->pport, 0);
5630
5631 for (i = 0; i < NUM_SEND_PIO_ERR_STATUS_COUNTERS; i++) {
5632 if (reg & (1ull << i))
5633 incr_cntr64(&dd->send_pio_err_status_cnt[i]);
5634 }
5635 }
5636
handle_sdma_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5637 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5638 {
5639 char buf[96];
5640 int i = 0;
5641
5642 dd_dev_info(dd, "SDMA Error: %s\n",
5643 sdma_err_status_string(buf, sizeof(buf), reg));
5644
5645 if (reg & ALL_SDMA_FREEZE_ERR)
5646 start_freeze_handling(dd->pport, 0);
5647
5648 for (i = 0; i < NUM_SEND_DMA_ERR_STATUS_COUNTERS; i++) {
5649 if (reg & (1ull << i))
5650 incr_cntr64(&dd->send_dma_err_status_cnt[i]);
5651 }
5652 }
5653
__count_port_discards(struct hfi1_pportdata * ppd)5654 static inline void __count_port_discards(struct hfi1_pportdata *ppd)
5655 {
5656 incr_cntr64(&ppd->port_xmit_discards);
5657 }
5658
count_port_inactive(struct hfi1_devdata * dd)5659 static void count_port_inactive(struct hfi1_devdata *dd)
5660 {
5661 __count_port_discards(dd->pport);
5662 }
5663
5664 /*
5665 * We have had a "disallowed packet" error during egress. Determine the
5666 * integrity check which failed, and update relevant error counter, etc.
5667 *
5668 * Note that the SEND_EGRESS_ERR_INFO register has only a single
5669 * bit of state per integrity check, and so we can miss the reason for an
5670 * egress error if more than one packet fails the same integrity check
5671 * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO.
5672 */
handle_send_egress_err_info(struct hfi1_devdata * dd,int vl)5673 static void handle_send_egress_err_info(struct hfi1_devdata *dd,
5674 int vl)
5675 {
5676 struct hfi1_pportdata *ppd = dd->pport;
5677 u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */
5678 u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO);
5679 char buf[96];
5680
5681 /* clear down all observed info as quickly as possible after read */
5682 write_csr(dd, SEND_EGRESS_ERR_INFO, info);
5683
5684 dd_dev_info(dd,
5685 "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n",
5686 info, egress_err_info_string(buf, sizeof(buf), info), src);
5687
5688 /* Eventually add other counters for each bit */
5689 if (info & PORT_DISCARD_EGRESS_ERRS) {
5690 int weight, i;
5691
5692 /*
5693 * Count all applicable bits as individual errors and
5694 * attribute them to the packet that triggered this handler.
5695 * This may not be completely accurate due to limitations
5696 * on the available hardware error information. There is
5697 * a single information register and any number of error
5698 * packets may have occurred and contributed to it before
5699 * this routine is called. This means that:
5700 * a) If multiple packets with the same error occur before
5701 * this routine is called, earlier packets are missed.
5702 * There is only a single bit for each error type.
5703 * b) Errors may not be attributed to the correct VL.
5704 * The driver is attributing all bits in the info register
5705 * to the packet that triggered this call, but bits
5706 * could be an accumulation of different packets with
5707 * different VLs.
5708 * c) A single error packet may have multiple counts attached
5709 * to it. There is no way for the driver to know if
5710 * multiple bits set in the info register are due to a
5711 * single packet or multiple packets. The driver assumes
5712 * multiple packets.
5713 */
5714 weight = hweight64(info & PORT_DISCARD_EGRESS_ERRS);
5715 for (i = 0; i < weight; i++) {
5716 __count_port_discards(ppd);
5717 if (vl >= 0 && vl < TXE_NUM_DATA_VL)
5718 incr_cntr64(&ppd->port_xmit_discards_vl[vl]);
5719 else if (vl == 15)
5720 incr_cntr64(&ppd->port_xmit_discards_vl
5721 [C_VL_15]);
5722 }
5723 }
5724 }
5725
5726 /*
5727 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5728 * register. Does it represent a 'port inactive' error?
5729 */
port_inactive_err(u64 posn)5730 static inline int port_inactive_err(u64 posn)
5731 {
5732 return (posn >= SEES(TX_LINKDOWN) &&
5733 posn <= SEES(TX_INCORRECT_LINK_STATE));
5734 }
5735
5736 /*
5737 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5738 * register. Does it represent a 'disallowed packet' error?
5739 */
disallowed_pkt_err(int posn)5740 static inline int disallowed_pkt_err(int posn)
5741 {
5742 return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) &&
5743 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET));
5744 }
5745
5746 /*
5747 * Input value is a bit position of one of the SDMA engine disallowed
5748 * packet errors. Return which engine. Use of this must be guarded by
5749 * disallowed_pkt_err().
5750 */
disallowed_pkt_engine(int posn)5751 static inline int disallowed_pkt_engine(int posn)
5752 {
5753 return posn - SEES(TX_SDMA0_DISALLOWED_PACKET);
5754 }
5755
5756 /*
5757 * Translate an SDMA engine to a VL. Return -1 if the tranlation cannot
5758 * be done.
5759 */
engine_to_vl(struct hfi1_devdata * dd,int engine)5760 static int engine_to_vl(struct hfi1_devdata *dd, int engine)
5761 {
5762 struct sdma_vl_map *m;
5763 int vl;
5764
5765 /* range check */
5766 if (engine < 0 || engine >= TXE_NUM_SDMA_ENGINES)
5767 return -1;
5768
5769 rcu_read_lock();
5770 m = rcu_dereference(dd->sdma_map);
5771 vl = m->engine_to_vl[engine];
5772 rcu_read_unlock();
5773
5774 return vl;
5775 }
5776
5777 /*
5778 * Translate the send context (sofware index) into a VL. Return -1 if the
5779 * translation cannot be done.
5780 */
sc_to_vl(struct hfi1_devdata * dd,int sw_index)5781 static int sc_to_vl(struct hfi1_devdata *dd, int sw_index)
5782 {
5783 struct send_context_info *sci;
5784 struct send_context *sc;
5785 int i;
5786
5787 sci = &dd->send_contexts[sw_index];
5788
5789 /* there is no information for user (PSM) and ack contexts */
5790 if ((sci->type != SC_KERNEL) && (sci->type != SC_VL15))
5791 return -1;
5792
5793 sc = sci->sc;
5794 if (!sc)
5795 return -1;
5796 if (dd->vld[15].sc == sc)
5797 return 15;
5798 for (i = 0; i < num_vls; i++)
5799 if (dd->vld[i].sc == sc)
5800 return i;
5801
5802 return -1;
5803 }
5804
handle_egress_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5805 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5806 {
5807 u64 reg_copy = reg, handled = 0;
5808 char buf[96];
5809 int i = 0;
5810
5811 if (reg & ALL_TXE_EGRESS_FREEZE_ERR)
5812 start_freeze_handling(dd->pport, 0);
5813 else if (is_ax(dd) &&
5814 (reg & SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK) &&
5815 (dd->icode != ICODE_FUNCTIONAL_SIMULATOR))
5816 start_freeze_handling(dd->pport, 0);
5817
5818 while (reg_copy) {
5819 int posn = fls64(reg_copy);
5820 /* fls64() returns a 1-based offset, we want it zero based */
5821 int shift = posn - 1;
5822 u64 mask = 1ULL << shift;
5823
5824 if (port_inactive_err(shift)) {
5825 count_port_inactive(dd);
5826 handled |= mask;
5827 } else if (disallowed_pkt_err(shift)) {
5828 int vl = engine_to_vl(dd, disallowed_pkt_engine(shift));
5829
5830 handle_send_egress_err_info(dd, vl);
5831 handled |= mask;
5832 }
5833 reg_copy &= ~mask;
5834 }
5835
5836 reg &= ~handled;
5837
5838 if (reg)
5839 dd_dev_info(dd, "Egress Error: %s\n",
5840 egress_err_status_string(buf, sizeof(buf), reg));
5841
5842 for (i = 0; i < NUM_SEND_EGRESS_ERR_STATUS_COUNTERS; i++) {
5843 if (reg & (1ull << i))
5844 incr_cntr64(&dd->send_egress_err_status_cnt[i]);
5845 }
5846 }
5847
handle_txe_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5848 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5849 {
5850 char buf[96];
5851 int i = 0;
5852
5853 dd_dev_info(dd, "Send Error: %s\n",
5854 send_err_status_string(buf, sizeof(buf), reg));
5855
5856 for (i = 0; i < NUM_SEND_ERR_STATUS_COUNTERS; i++) {
5857 if (reg & (1ull << i))
5858 incr_cntr64(&dd->send_err_status_cnt[i]);
5859 }
5860 }
5861
5862 /*
5863 * The maximum number of times the error clear down will loop before
5864 * blocking a repeating error. This value is arbitrary.
5865 */
5866 #define MAX_CLEAR_COUNT 20
5867
5868 /*
5869 * Clear and handle an error register. All error interrupts are funneled
5870 * through here to have a central location to correctly handle single-
5871 * or multi-shot errors.
5872 *
5873 * For non per-context registers, call this routine with a context value
5874 * of 0 so the per-context offset is zero.
5875 *
5876 * If the handler loops too many times, assume that something is wrong
5877 * and can't be fixed, so mask the error bits.
5878 */
interrupt_clear_down(struct hfi1_devdata * dd,u32 context,const struct err_reg_info * eri)5879 static void interrupt_clear_down(struct hfi1_devdata *dd,
5880 u32 context,
5881 const struct err_reg_info *eri)
5882 {
5883 u64 reg;
5884 u32 count;
5885
5886 /* read in a loop until no more errors are seen */
5887 count = 0;
5888 while (1) {
5889 reg = read_kctxt_csr(dd, context, eri->status);
5890 if (reg == 0)
5891 break;
5892 write_kctxt_csr(dd, context, eri->clear, reg);
5893 if (likely(eri->handler))
5894 eri->handler(dd, context, reg);
5895 count++;
5896 if (count > MAX_CLEAR_COUNT) {
5897 u64 mask;
5898
5899 dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n",
5900 eri->desc, reg);
5901 /*
5902 * Read-modify-write so any other masked bits
5903 * remain masked.
5904 */
5905 mask = read_kctxt_csr(dd, context, eri->mask);
5906 mask &= ~reg;
5907 write_kctxt_csr(dd, context, eri->mask, mask);
5908 break;
5909 }
5910 }
5911 }
5912
5913 /*
5914 * CCE block "misc" interrupt. Source is < 16.
5915 */
is_misc_err_int(struct hfi1_devdata * dd,unsigned int source)5916 static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source)
5917 {
5918 const struct err_reg_info *eri = &misc_errs[source];
5919
5920 if (eri->handler) {
5921 interrupt_clear_down(dd, 0, eri);
5922 } else {
5923 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n",
5924 source);
5925 }
5926 }
5927
send_context_err_status_string(char * buf,int buf_len,u64 flags)5928 static char *send_context_err_status_string(char *buf, int buf_len, u64 flags)
5929 {
5930 return flag_string(buf, buf_len, flags,
5931 sc_err_status_flags,
5932 ARRAY_SIZE(sc_err_status_flags));
5933 }
5934
5935 /*
5936 * Send context error interrupt. Source (hw_context) is < 160.
5937 *
5938 * All send context errors cause the send context to halt. The normal
5939 * clear-down mechanism cannot be used because we cannot clear the
5940 * error bits until several other long-running items are done first.
5941 * This is OK because with the context halted, nothing else is going
5942 * to happen on it anyway.
5943 */
is_sendctxt_err_int(struct hfi1_devdata * dd,unsigned int hw_context)5944 static void is_sendctxt_err_int(struct hfi1_devdata *dd,
5945 unsigned int hw_context)
5946 {
5947 struct send_context_info *sci;
5948 struct send_context *sc;
5949 char flags[96];
5950 u64 status;
5951 u32 sw_index;
5952 int i = 0;
5953 unsigned long irq_flags;
5954
5955 sw_index = dd->hw_to_sw[hw_context];
5956 if (sw_index >= dd->num_send_contexts) {
5957 dd_dev_err(dd,
5958 "out of range sw index %u for send context %u\n",
5959 sw_index, hw_context);
5960 return;
5961 }
5962 sci = &dd->send_contexts[sw_index];
5963 spin_lock_irqsave(&dd->sc_lock, irq_flags);
5964 sc = sci->sc;
5965 if (!sc) {
5966 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__,
5967 sw_index, hw_context);
5968 spin_unlock_irqrestore(&dd->sc_lock, irq_flags);
5969 return;
5970 }
5971
5972 /* tell the software that a halt has begun */
5973 sc_stop(sc, SCF_HALTED);
5974
5975 status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS);
5976
5977 dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context,
5978 send_context_err_status_string(flags, sizeof(flags),
5979 status));
5980
5981 if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK)
5982 handle_send_egress_err_info(dd, sc_to_vl(dd, sw_index));
5983
5984 /*
5985 * Automatically restart halted kernel contexts out of interrupt
5986 * context. User contexts must ask the driver to restart the context.
5987 */
5988 if (sc->type != SC_USER)
5989 queue_work(dd->pport->hfi1_wq, &sc->halt_work);
5990 spin_unlock_irqrestore(&dd->sc_lock, irq_flags);
5991
5992 /*
5993 * Update the counters for the corresponding status bits.
5994 * Note that these particular counters are aggregated over all
5995 * 160 contexts.
5996 */
5997 for (i = 0; i < NUM_SEND_CTXT_ERR_STATUS_COUNTERS; i++) {
5998 if (status & (1ull << i))
5999 incr_cntr64(&dd->sw_ctxt_err_status_cnt[i]);
6000 }
6001 }
6002
handle_sdma_eng_err(struct hfi1_devdata * dd,unsigned int source,u64 status)6003 static void handle_sdma_eng_err(struct hfi1_devdata *dd,
6004 unsigned int source, u64 status)
6005 {
6006 struct sdma_engine *sde;
6007 int i = 0;
6008
6009 sde = &dd->per_sdma[source];
6010 #ifdef CONFIG_SDMA_VERBOSITY
6011 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
6012 slashstrip(__FILE__), __LINE__, __func__);
6013 dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n",
6014 sde->this_idx, source, (unsigned long long)status);
6015 #endif
6016 sde->err_cnt++;
6017 sdma_engine_error(sde, status);
6018
6019 /*
6020 * Update the counters for the corresponding status bits.
6021 * Note that these particular counters are aggregated over
6022 * all 16 DMA engines.
6023 */
6024 for (i = 0; i < NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS; i++) {
6025 if (status & (1ull << i))
6026 incr_cntr64(&dd->sw_send_dma_eng_err_status_cnt[i]);
6027 }
6028 }
6029
6030 /*
6031 * CCE block SDMA error interrupt. Source is < 16.
6032 */
is_sdma_eng_err_int(struct hfi1_devdata * dd,unsigned int source)6033 static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source)
6034 {
6035 #ifdef CONFIG_SDMA_VERBOSITY
6036 struct sdma_engine *sde = &dd->per_sdma[source];
6037
6038 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
6039 slashstrip(__FILE__), __LINE__, __func__);
6040 dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx,
6041 source);
6042 sdma_dumpstate(sde);
6043 #endif
6044 interrupt_clear_down(dd, source, &sdma_eng_err);
6045 }
6046
6047 /*
6048 * CCE block "various" interrupt. Source is < 8.
6049 */
is_various_int(struct hfi1_devdata * dd,unsigned int source)6050 static void is_various_int(struct hfi1_devdata *dd, unsigned int source)
6051 {
6052 const struct err_reg_info *eri = &various_err[source];
6053
6054 /*
6055 * TCritInt cannot go through interrupt_clear_down()
6056 * because it is not a second tier interrupt. The handler
6057 * should be called directly.
6058 */
6059 if (source == TCRIT_INT_SOURCE)
6060 handle_temp_err(dd);
6061 else if (eri->handler)
6062 interrupt_clear_down(dd, 0, eri);
6063 else
6064 dd_dev_info(dd,
6065 "%s: Unimplemented/reserved interrupt %d\n",
6066 __func__, source);
6067 }
6068
handle_qsfp_int(struct hfi1_devdata * dd,u32 src_ctx,u64 reg)6069 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg)
6070 {
6071 /* src_ctx is always zero */
6072 struct hfi1_pportdata *ppd = dd->pport;
6073 unsigned long flags;
6074 u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
6075
6076 if (reg & QSFP_HFI0_MODPRST_N) {
6077 if (!qsfp_mod_present(ppd)) {
6078 dd_dev_info(dd, "%s: QSFP module removed\n",
6079 __func__);
6080
6081 ppd->driver_link_ready = 0;
6082 /*
6083 * Cable removed, reset all our information about the
6084 * cache and cable capabilities
6085 */
6086
6087 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6088 /*
6089 * We don't set cache_refresh_required here as we expect
6090 * an interrupt when a cable is inserted
6091 */
6092 ppd->qsfp_info.cache_valid = 0;
6093 ppd->qsfp_info.reset_needed = 0;
6094 ppd->qsfp_info.limiting_active = 0;
6095 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
6096 flags);
6097 /* Invert the ModPresent pin now to detect plug-in */
6098 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
6099 ASIC_QSFP1_INVERT, qsfp_int_mgmt);
6100
6101 if ((ppd->offline_disabled_reason >
6102 HFI1_ODR_MASK(
6103 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED)) ||
6104 (ppd->offline_disabled_reason ==
6105 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE)))
6106 ppd->offline_disabled_reason =
6107 HFI1_ODR_MASK(
6108 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED);
6109
6110 if (ppd->host_link_state == HLS_DN_POLL) {
6111 /*
6112 * The link is still in POLL. This means
6113 * that the normal link down processing
6114 * will not happen. We have to do it here
6115 * before turning the DC off.
6116 */
6117 queue_work(ppd->link_wq, &ppd->link_down_work);
6118 }
6119 } else {
6120 dd_dev_info(dd, "%s: QSFP module inserted\n",
6121 __func__);
6122
6123 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6124 ppd->qsfp_info.cache_valid = 0;
6125 ppd->qsfp_info.cache_refresh_required = 1;
6126 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
6127 flags);
6128
6129 /*
6130 * Stop inversion of ModPresent pin to detect
6131 * removal of the cable
6132 */
6133 qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N;
6134 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
6135 ASIC_QSFP1_INVERT, qsfp_int_mgmt);
6136
6137 ppd->offline_disabled_reason =
6138 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
6139 }
6140 }
6141
6142 if (reg & QSFP_HFI0_INT_N) {
6143 dd_dev_info(dd, "%s: Interrupt received from QSFP module\n",
6144 __func__);
6145 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6146 ppd->qsfp_info.check_interrupt_flags = 1;
6147 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags);
6148 }
6149
6150 /* Schedule the QSFP work only if there is a cable attached. */
6151 if (qsfp_mod_present(ppd))
6152 queue_work(ppd->link_wq, &ppd->qsfp_info.qsfp_work);
6153 }
6154
request_host_lcb_access(struct hfi1_devdata * dd)6155 static int request_host_lcb_access(struct hfi1_devdata *dd)
6156 {
6157 int ret;
6158
6159 ret = do_8051_command(dd, HCMD_MISC,
6160 (u64)HCMD_MISC_REQUEST_LCB_ACCESS <<
6161 LOAD_DATA_FIELD_ID_SHIFT, NULL);
6162 if (ret != HCMD_SUCCESS) {
6163 dd_dev_err(dd, "%s: command failed with error %d\n",
6164 __func__, ret);
6165 }
6166 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6167 }
6168
request_8051_lcb_access(struct hfi1_devdata * dd)6169 static int request_8051_lcb_access(struct hfi1_devdata *dd)
6170 {
6171 int ret;
6172
6173 ret = do_8051_command(dd, HCMD_MISC,
6174 (u64)HCMD_MISC_GRANT_LCB_ACCESS <<
6175 LOAD_DATA_FIELD_ID_SHIFT, NULL);
6176 if (ret != HCMD_SUCCESS) {
6177 dd_dev_err(dd, "%s: command failed with error %d\n",
6178 __func__, ret);
6179 }
6180 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6181 }
6182
6183 /*
6184 * Set the LCB selector - allow host access. The DCC selector always
6185 * points to the host.
6186 */
set_host_lcb_access(struct hfi1_devdata * dd)6187 static inline void set_host_lcb_access(struct hfi1_devdata *dd)
6188 {
6189 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
6190 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK |
6191 DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK);
6192 }
6193
6194 /*
6195 * Clear the LCB selector - allow 8051 access. The DCC selector always
6196 * points to the host.
6197 */
set_8051_lcb_access(struct hfi1_devdata * dd)6198 static inline void set_8051_lcb_access(struct hfi1_devdata *dd)
6199 {
6200 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
6201 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK);
6202 }
6203
6204 /*
6205 * Acquire LCB access from the 8051. If the host already has access,
6206 * just increment a counter. Otherwise, inform the 8051 that the
6207 * host is taking access.
6208 *
6209 * Returns:
6210 * 0 on success
6211 * -EBUSY if the 8051 has control and cannot be disturbed
6212 * -errno if unable to acquire access from the 8051
6213 */
acquire_lcb_access(struct hfi1_devdata * dd,int sleep_ok)6214 int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6215 {
6216 struct hfi1_pportdata *ppd = dd->pport;
6217 int ret = 0;
6218
6219 /*
6220 * Use the host link state lock so the operation of this routine
6221 * { link state check, selector change, count increment } can occur
6222 * as a unit against a link state change. Otherwise there is a
6223 * race between the state change and the count increment.
6224 */
6225 if (sleep_ok) {
6226 mutex_lock(&ppd->hls_lock);
6227 } else {
6228 while (!mutex_trylock(&ppd->hls_lock))
6229 udelay(1);
6230 }
6231
6232 /* this access is valid only when the link is up */
6233 if (ppd->host_link_state & HLS_DOWN) {
6234 dd_dev_info(dd, "%s: link state %s not up\n",
6235 __func__, link_state_name(ppd->host_link_state));
6236 ret = -EBUSY;
6237 goto done;
6238 }
6239
6240 if (dd->lcb_access_count == 0) {
6241 ret = request_host_lcb_access(dd);
6242 if (ret) {
6243 dd_dev_err(dd,
6244 "%s: unable to acquire LCB access, err %d\n",
6245 __func__, ret);
6246 goto done;
6247 }
6248 set_host_lcb_access(dd);
6249 }
6250 dd->lcb_access_count++;
6251 done:
6252 mutex_unlock(&ppd->hls_lock);
6253 return ret;
6254 }
6255
6256 /*
6257 * Release LCB access by decrementing the use count. If the count is moving
6258 * from 1 to 0, inform 8051 that it has control back.
6259 *
6260 * Returns:
6261 * 0 on success
6262 * -errno if unable to release access to the 8051
6263 */
release_lcb_access(struct hfi1_devdata * dd,int sleep_ok)6264 int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6265 {
6266 int ret = 0;
6267
6268 /*
6269 * Use the host link state lock because the acquire needed it.
6270 * Here, we only need to keep { selector change, count decrement }
6271 * as a unit.
6272 */
6273 if (sleep_ok) {
6274 mutex_lock(&dd->pport->hls_lock);
6275 } else {
6276 while (!mutex_trylock(&dd->pport->hls_lock))
6277 udelay(1);
6278 }
6279
6280 if (dd->lcb_access_count == 0) {
6281 dd_dev_err(dd, "%s: LCB access count is zero. Skipping.\n",
6282 __func__);
6283 goto done;
6284 }
6285
6286 if (dd->lcb_access_count == 1) {
6287 set_8051_lcb_access(dd);
6288 ret = request_8051_lcb_access(dd);
6289 if (ret) {
6290 dd_dev_err(dd,
6291 "%s: unable to release LCB access, err %d\n",
6292 __func__, ret);
6293 /* restore host access if the grant didn't work */
6294 set_host_lcb_access(dd);
6295 goto done;
6296 }
6297 }
6298 dd->lcb_access_count--;
6299 done:
6300 mutex_unlock(&dd->pport->hls_lock);
6301 return ret;
6302 }
6303
6304 /*
6305 * Initialize LCB access variables and state. Called during driver load,
6306 * after most of the initialization is finished.
6307 *
6308 * The DC default is LCB access on for the host. The driver defaults to
6309 * leaving access to the 8051. Assign access now - this constrains the call
6310 * to this routine to be after all LCB set-up is done. In particular, after
6311 * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts()
6312 */
init_lcb_access(struct hfi1_devdata * dd)6313 static void init_lcb_access(struct hfi1_devdata *dd)
6314 {
6315 dd->lcb_access_count = 0;
6316 }
6317
6318 /*
6319 * Write a response back to a 8051 request.
6320 */
hreq_response(struct hfi1_devdata * dd,u8 return_code,u16 rsp_data)6321 static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data)
6322 {
6323 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0,
6324 DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK |
6325 (u64)return_code <<
6326 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT |
6327 (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
6328 }
6329
6330 /*
6331 * Handle host requests from the 8051.
6332 */
handle_8051_request(struct hfi1_pportdata * ppd)6333 static void handle_8051_request(struct hfi1_pportdata *ppd)
6334 {
6335 struct hfi1_devdata *dd = ppd->dd;
6336 u64 reg;
6337 u16 data = 0;
6338 u8 type;
6339
6340 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1);
6341 if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0)
6342 return; /* no request */
6343
6344 /* zero out COMPLETED so the response is seen */
6345 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0);
6346
6347 /* extract request details */
6348 type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT)
6349 & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK;
6350 data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT)
6351 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK;
6352
6353 switch (type) {
6354 case HREQ_LOAD_CONFIG:
6355 case HREQ_SAVE_CONFIG:
6356 case HREQ_READ_CONFIG:
6357 case HREQ_SET_TX_EQ_ABS:
6358 case HREQ_SET_TX_EQ_REL:
6359 case HREQ_ENABLE:
6360 dd_dev_info(dd, "8051 request: request 0x%x not supported\n",
6361 type);
6362 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6363 break;
6364 case HREQ_LCB_RESET:
6365 /* Put the LCB, RX FPE and TX FPE into reset */
6366 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_INTO_RESET);
6367 /* Make sure the write completed */
6368 (void)read_csr(dd, DCC_CFG_RESET);
6369 /* Hold the reset long enough to take effect */
6370 udelay(1);
6371 /* Take the LCB, RX FPE and TX FPE out of reset */
6372 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET);
6373 hreq_response(dd, HREQ_SUCCESS, 0);
6374
6375 break;
6376 case HREQ_CONFIG_DONE:
6377 hreq_response(dd, HREQ_SUCCESS, 0);
6378 break;
6379
6380 case HREQ_INTERFACE_TEST:
6381 hreq_response(dd, HREQ_SUCCESS, data);
6382 break;
6383 default:
6384 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type);
6385 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6386 break;
6387 }
6388 }
6389
6390 /*
6391 * Set up allocation unit vaulue.
6392 */
set_up_vau(struct hfi1_devdata * dd,u8 vau)6393 void set_up_vau(struct hfi1_devdata *dd, u8 vau)
6394 {
6395 u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
6396
6397 /* do not modify other values in the register */
6398 reg &= ~SEND_CM_GLOBAL_CREDIT_AU_SMASK;
6399 reg |= (u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT;
6400 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
6401 }
6402
6403 /*
6404 * Set up initial VL15 credits of the remote. Assumes the rest of
6405 * the CM credit registers are zero from a previous global or credit reset.
6406 * Shared limit for VL15 will always be 0.
6407 */
set_up_vl15(struct hfi1_devdata * dd,u16 vl15buf)6408 void set_up_vl15(struct hfi1_devdata *dd, u16 vl15buf)
6409 {
6410 u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
6411
6412 /* set initial values for total and shared credit limit */
6413 reg &= ~(SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK |
6414 SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK);
6415
6416 /*
6417 * Set total limit to be equal to VL15 credits.
6418 * Leave shared limit at 0.
6419 */
6420 reg |= (u64)vl15buf << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
6421 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
6422
6423 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf
6424 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
6425 }
6426
6427 /*
6428 * Zero all credit details from the previous connection and
6429 * reset the CM manager's internal counters.
6430 */
reset_link_credits(struct hfi1_devdata * dd)6431 void reset_link_credits(struct hfi1_devdata *dd)
6432 {
6433 int i;
6434
6435 /* remove all previous VL credit limits */
6436 for (i = 0; i < TXE_NUM_DATA_VL; i++)
6437 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
6438 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
6439 write_csr(dd, SEND_CM_GLOBAL_CREDIT, 0);
6440 /* reset the CM block */
6441 pio_send_control(dd, PSC_CM_RESET);
6442 /* reset cached value */
6443 dd->vl15buf_cached = 0;
6444 }
6445
6446 /* convert a vCU to a CU */
vcu_to_cu(u8 vcu)6447 static u32 vcu_to_cu(u8 vcu)
6448 {
6449 return 1 << vcu;
6450 }
6451
6452 /* convert a CU to a vCU */
cu_to_vcu(u32 cu)6453 static u8 cu_to_vcu(u32 cu)
6454 {
6455 return ilog2(cu);
6456 }
6457
6458 /* convert a vAU to an AU */
vau_to_au(u8 vau)6459 static u32 vau_to_au(u8 vau)
6460 {
6461 return 8 * (1 << vau);
6462 }
6463
set_linkup_defaults(struct hfi1_pportdata * ppd)6464 static void set_linkup_defaults(struct hfi1_pportdata *ppd)
6465 {
6466 ppd->sm_trap_qp = 0x0;
6467 ppd->sa_qp = 0x1;
6468 }
6469
6470 /*
6471 * Graceful LCB shutdown. This leaves the LCB FIFOs in reset.
6472 */
lcb_shutdown(struct hfi1_devdata * dd,int abort)6473 static void lcb_shutdown(struct hfi1_devdata *dd, int abort)
6474 {
6475 u64 reg;
6476
6477 /* clear lcb run: LCB_CFG_RUN.EN = 0 */
6478 write_csr(dd, DC_LCB_CFG_RUN, 0);
6479 /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */
6480 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET,
6481 1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT);
6482 /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */
6483 dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN);
6484 reg = read_csr(dd, DCC_CFG_RESET);
6485 write_csr(dd, DCC_CFG_RESET, reg |
6486 DCC_CFG_RESET_RESET_LCB | DCC_CFG_RESET_RESET_RX_FPE);
6487 (void)read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */
6488 if (!abort) {
6489 udelay(1); /* must hold for the longer of 16cclks or 20ns */
6490 write_csr(dd, DCC_CFG_RESET, reg);
6491 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6492 }
6493 }
6494
6495 /*
6496 * This routine should be called after the link has been transitioned to
6497 * OFFLINE (OFFLINE state has the side effect of putting the SerDes into
6498 * reset).
6499 *
6500 * The expectation is that the caller of this routine would have taken
6501 * care of properly transitioning the link into the correct state.
6502 * NOTE: the caller needs to acquire the dd->dc8051_lock lock
6503 * before calling this function.
6504 */
_dc_shutdown(struct hfi1_devdata * dd)6505 static void _dc_shutdown(struct hfi1_devdata *dd)
6506 {
6507 lockdep_assert_held(&dd->dc8051_lock);
6508
6509 if (dd->dc_shutdown)
6510 return;
6511
6512 dd->dc_shutdown = 1;
6513 /* Shutdown the LCB */
6514 lcb_shutdown(dd, 1);
6515 /*
6516 * Going to OFFLINE would have causes the 8051 to put the
6517 * SerDes into reset already. Just need to shut down the 8051,
6518 * itself.
6519 */
6520 write_csr(dd, DC_DC8051_CFG_RST, 0x1);
6521 }
6522
dc_shutdown(struct hfi1_devdata * dd)6523 static void dc_shutdown(struct hfi1_devdata *dd)
6524 {
6525 mutex_lock(&dd->dc8051_lock);
6526 _dc_shutdown(dd);
6527 mutex_unlock(&dd->dc8051_lock);
6528 }
6529
6530 /*
6531 * Calling this after the DC has been brought out of reset should not
6532 * do any damage.
6533 * NOTE: the caller needs to acquire the dd->dc8051_lock lock
6534 * before calling this function.
6535 */
_dc_start(struct hfi1_devdata * dd)6536 static void _dc_start(struct hfi1_devdata *dd)
6537 {
6538 lockdep_assert_held(&dd->dc8051_lock);
6539
6540 if (!dd->dc_shutdown)
6541 return;
6542
6543 /* Take the 8051 out of reset */
6544 write_csr(dd, DC_DC8051_CFG_RST, 0ull);
6545 /* Wait until 8051 is ready */
6546 if (wait_fm_ready(dd, TIMEOUT_8051_START))
6547 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n",
6548 __func__);
6549
6550 /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */
6551 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET);
6552 /* lcb_shutdown() with abort=1 does not restore these */
6553 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6554 dd->dc_shutdown = 0;
6555 }
6556
dc_start(struct hfi1_devdata * dd)6557 static void dc_start(struct hfi1_devdata *dd)
6558 {
6559 mutex_lock(&dd->dc8051_lock);
6560 _dc_start(dd);
6561 mutex_unlock(&dd->dc8051_lock);
6562 }
6563
6564 /*
6565 * These LCB adjustments are for the Aurora SerDes core in the FPGA.
6566 */
adjust_lcb_for_fpga_serdes(struct hfi1_devdata * dd)6567 static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd)
6568 {
6569 u64 rx_radr, tx_radr;
6570 u32 version;
6571
6572 if (dd->icode != ICODE_FPGA_EMULATION)
6573 return;
6574
6575 /*
6576 * These LCB defaults on emulator _s are good, nothing to do here:
6577 * LCB_CFG_TX_FIFOS_RADR
6578 * LCB_CFG_RX_FIFOS_RADR
6579 * LCB_CFG_LN_DCLK
6580 * LCB_CFG_IGNORE_LOST_RCLK
6581 */
6582 if (is_emulator_s(dd))
6583 return;
6584 /* else this is _p */
6585
6586 version = emulator_rev(dd);
6587 if (!is_ax(dd))
6588 version = 0x2d; /* all B0 use 0x2d or higher settings */
6589
6590 if (version <= 0x12) {
6591 /* release 0x12 and below */
6592
6593 /*
6594 * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9
6595 * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9
6596 * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa
6597 */
6598 rx_radr =
6599 0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6600 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6601 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6602 /*
6603 * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default)
6604 * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6
6605 */
6606 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6607 } else if (version <= 0x18) {
6608 /* release 0x13 up to 0x18 */
6609 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6610 rx_radr =
6611 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6612 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6613 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6614 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6615 } else if (version == 0x19) {
6616 /* release 0x19 */
6617 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */
6618 rx_radr =
6619 0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6620 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6621 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6622 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6623 } else if (version == 0x1a) {
6624 /* release 0x1a */
6625 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6626 rx_radr =
6627 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6628 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6629 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6630 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6631 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull);
6632 } else {
6633 /* release 0x1b and higher */
6634 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */
6635 rx_radr =
6636 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6637 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6638 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6639 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6640 }
6641
6642 write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr);
6643 /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */
6644 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
6645 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
6646 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr);
6647 }
6648
6649 /*
6650 * Handle a SMA idle message
6651 *
6652 * This is a work-queue function outside of the interrupt.
6653 */
handle_sma_message(struct work_struct * work)6654 void handle_sma_message(struct work_struct *work)
6655 {
6656 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6657 sma_message_work);
6658 struct hfi1_devdata *dd = ppd->dd;
6659 u64 msg;
6660 int ret;
6661
6662 /*
6663 * msg is bytes 1-4 of the 40-bit idle message - the command code
6664 * is stripped off
6665 */
6666 ret = read_idle_sma(dd, &msg);
6667 if (ret)
6668 return;
6669 dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg);
6670 /*
6671 * React to the SMA message. Byte[1] (0 for us) is the command.
6672 */
6673 switch (msg & 0xff) {
6674 case SMA_IDLE_ARM:
6675 /*
6676 * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6677 * State Transitions
6678 *
6679 * Only expected in INIT or ARMED, discard otherwise.
6680 */
6681 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED))
6682 ppd->neighbor_normal = 1;
6683 break;
6684 case SMA_IDLE_ACTIVE:
6685 /*
6686 * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6687 * State Transitions
6688 *
6689 * Can activate the node. Discard otherwise.
6690 */
6691 if (ppd->host_link_state == HLS_UP_ARMED &&
6692 ppd->is_active_optimize_enabled) {
6693 ppd->neighbor_normal = 1;
6694 ret = set_link_state(ppd, HLS_UP_ACTIVE);
6695 if (ret)
6696 dd_dev_err(
6697 dd,
6698 "%s: received Active SMA idle message, couldn't set link to Active\n",
6699 __func__);
6700 }
6701 break;
6702 default:
6703 dd_dev_err(dd,
6704 "%s: received unexpected SMA idle message 0x%llx\n",
6705 __func__, msg);
6706 break;
6707 }
6708 }
6709
adjust_rcvctrl(struct hfi1_devdata * dd,u64 add,u64 clear)6710 static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear)
6711 {
6712 u64 rcvctrl;
6713 unsigned long flags;
6714
6715 spin_lock_irqsave(&dd->rcvctrl_lock, flags);
6716 rcvctrl = read_csr(dd, RCV_CTRL);
6717 rcvctrl |= add;
6718 rcvctrl &= ~clear;
6719 write_csr(dd, RCV_CTRL, rcvctrl);
6720 spin_unlock_irqrestore(&dd->rcvctrl_lock, flags);
6721 }
6722
add_rcvctrl(struct hfi1_devdata * dd,u64 add)6723 static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add)
6724 {
6725 adjust_rcvctrl(dd, add, 0);
6726 }
6727
clear_rcvctrl(struct hfi1_devdata * dd,u64 clear)6728 static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear)
6729 {
6730 adjust_rcvctrl(dd, 0, clear);
6731 }
6732
6733 /*
6734 * Called from all interrupt handlers to start handling an SPC freeze.
6735 */
start_freeze_handling(struct hfi1_pportdata * ppd,int flags)6736 void start_freeze_handling(struct hfi1_pportdata *ppd, int flags)
6737 {
6738 struct hfi1_devdata *dd = ppd->dd;
6739 struct send_context *sc;
6740 int i;
6741 int sc_flags;
6742
6743 if (flags & FREEZE_SELF)
6744 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6745
6746 /* enter frozen mode */
6747 dd->flags |= HFI1_FROZEN;
6748
6749 /* notify all SDMA engines that they are going into a freeze */
6750 sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN));
6751
6752 sc_flags = SCF_FROZEN | SCF_HALTED | (flags & FREEZE_LINK_DOWN ?
6753 SCF_LINK_DOWN : 0);
6754 /* do halt pre-handling on all enabled send contexts */
6755 for (i = 0; i < dd->num_send_contexts; i++) {
6756 sc = dd->send_contexts[i].sc;
6757 if (sc && (sc->flags & SCF_ENABLED))
6758 sc_stop(sc, sc_flags);
6759 }
6760
6761 /* Send context are frozen. Notify user space */
6762 hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT);
6763
6764 if (flags & FREEZE_ABORT) {
6765 dd_dev_err(dd,
6766 "Aborted freeze recovery. Please REBOOT system\n");
6767 return;
6768 }
6769 /* queue non-interrupt handler */
6770 queue_work(ppd->hfi1_wq, &ppd->freeze_work);
6771 }
6772
6773 /*
6774 * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen,
6775 * depending on the "freeze" parameter.
6776 *
6777 * No need to return an error if it times out, our only option
6778 * is to proceed anyway.
6779 */
wait_for_freeze_status(struct hfi1_devdata * dd,int freeze)6780 static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze)
6781 {
6782 unsigned long timeout;
6783 u64 reg;
6784
6785 timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT);
6786 while (1) {
6787 reg = read_csr(dd, CCE_STATUS);
6788 if (freeze) {
6789 /* waiting until all indicators are set */
6790 if ((reg & ALL_FROZE) == ALL_FROZE)
6791 return; /* all done */
6792 } else {
6793 /* waiting until all indicators are clear */
6794 if ((reg & ALL_FROZE) == 0)
6795 return; /* all done */
6796 }
6797
6798 if (time_after(jiffies, timeout)) {
6799 dd_dev_err(dd,
6800 "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing",
6801 freeze ? "" : "un", reg & ALL_FROZE,
6802 freeze ? ALL_FROZE : 0ull);
6803 return;
6804 }
6805 usleep_range(80, 120);
6806 }
6807 }
6808
6809 /*
6810 * Do all freeze handling for the RXE block.
6811 */
rxe_freeze(struct hfi1_devdata * dd)6812 static void rxe_freeze(struct hfi1_devdata *dd)
6813 {
6814 int i;
6815 struct hfi1_ctxtdata *rcd;
6816
6817 /* disable port */
6818 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6819
6820 /* disable all receive contexts */
6821 for (i = 0; i < dd->num_rcv_contexts; i++) {
6822 rcd = hfi1_rcd_get_by_index(dd, i);
6823 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, rcd);
6824 hfi1_rcd_put(rcd);
6825 }
6826 }
6827
6828 /*
6829 * Unfreeze handling for the RXE block - kernel contexts only.
6830 * This will also enable the port. User contexts will do unfreeze
6831 * handling on a per-context basis as they call into the driver.
6832 *
6833 */
rxe_kernel_unfreeze(struct hfi1_devdata * dd)6834 static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
6835 {
6836 u32 rcvmask;
6837 u16 i;
6838 struct hfi1_ctxtdata *rcd;
6839
6840 /* enable all kernel contexts */
6841 for (i = 0; i < dd->num_rcv_contexts; i++) {
6842 rcd = hfi1_rcd_get_by_index(dd, i);
6843
6844 /* Ensure all non-user contexts(including vnic) are enabled */
6845 if (!rcd ||
6846 (i >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic)) {
6847 hfi1_rcd_put(rcd);
6848 continue;
6849 }
6850 rcvmask = HFI1_RCVCTRL_CTXT_ENB;
6851 /* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */
6852 rcvmask |= hfi1_rcvhdrtail_kvaddr(rcd) ?
6853 HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
6854 hfi1_rcvctrl(dd, rcvmask, rcd);
6855 hfi1_rcd_put(rcd);
6856 }
6857
6858 /* enable port */
6859 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6860 }
6861
6862 /*
6863 * Non-interrupt SPC freeze handling.
6864 *
6865 * This is a work-queue function outside of the triggering interrupt.
6866 */
handle_freeze(struct work_struct * work)6867 void handle_freeze(struct work_struct *work)
6868 {
6869 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6870 freeze_work);
6871 struct hfi1_devdata *dd = ppd->dd;
6872
6873 /* wait for freeze indicators on all affected blocks */
6874 wait_for_freeze_status(dd, 1);
6875
6876 /* SPC is now frozen */
6877
6878 /* do send PIO freeze steps */
6879 pio_freeze(dd);
6880
6881 /* do send DMA freeze steps */
6882 sdma_freeze(dd);
6883
6884 /* do send egress freeze steps - nothing to do */
6885
6886 /* do receive freeze steps */
6887 rxe_freeze(dd);
6888
6889 /*
6890 * Unfreeze the hardware - clear the freeze, wait for each
6891 * block's frozen bit to clear, then clear the frozen flag.
6892 */
6893 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6894 wait_for_freeze_status(dd, 0);
6895
6896 if (is_ax(dd)) {
6897 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6898 wait_for_freeze_status(dd, 1);
6899 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6900 wait_for_freeze_status(dd, 0);
6901 }
6902
6903 /* do send PIO unfreeze steps for kernel contexts */
6904 pio_kernel_unfreeze(dd);
6905
6906 /* do send DMA unfreeze steps */
6907 sdma_unfreeze(dd);
6908
6909 /* do send egress unfreeze steps - nothing to do */
6910
6911 /* do receive unfreeze steps for kernel contexts */
6912 rxe_kernel_unfreeze(dd);
6913
6914 /*
6915 * The unfreeze procedure touches global device registers when
6916 * it disables and re-enables RXE. Mark the device unfrozen
6917 * after all that is done so other parts of the driver waiting
6918 * for the device to unfreeze don't do things out of order.
6919 *
6920 * The above implies that the meaning of HFI1_FROZEN flag is
6921 * "Device has gone into freeze mode and freeze mode handling
6922 * is still in progress."
6923 *
6924 * The flag will be removed when freeze mode processing has
6925 * completed.
6926 */
6927 dd->flags &= ~HFI1_FROZEN;
6928 wake_up(&dd->event_queue);
6929
6930 /* no longer frozen */
6931 }
6932
6933 /**
6934 * update_xmit_counters - update PortXmitWait/PortVlXmitWait
6935 * counters.
6936 * @ppd: info of physical Hfi port
6937 * @link_width: new link width after link up or downgrade
6938 *
6939 * Update the PortXmitWait and PortVlXmitWait counters after
6940 * a link up or downgrade event to reflect a link width change.
6941 */
update_xmit_counters(struct hfi1_pportdata * ppd,u16 link_width)6942 static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width)
6943 {
6944 int i;
6945 u16 tx_width;
6946 u16 link_speed;
6947
6948 tx_width = tx_link_width(link_width);
6949 link_speed = get_link_speed(ppd->link_speed_active);
6950
6951 /*
6952 * There are C_VL_COUNT number of PortVLXmitWait counters.
6953 * Adding 1 to C_VL_COUNT to include the PortXmitWait counter.
6954 */
6955 for (i = 0; i < C_VL_COUNT + 1; i++)
6956 get_xmit_wait_counters(ppd, tx_width, link_speed, i);
6957 }
6958
6959 /*
6960 * Handle a link up interrupt from the 8051.
6961 *
6962 * This is a work-queue function outside of the interrupt.
6963 */
handle_link_up(struct work_struct * work)6964 void handle_link_up(struct work_struct *work)
6965 {
6966 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6967 link_up_work);
6968 struct hfi1_devdata *dd = ppd->dd;
6969
6970 set_link_state(ppd, HLS_UP_INIT);
6971
6972 /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */
6973 read_ltp_rtt(dd);
6974 /*
6975 * OPA specifies that certain counters are cleared on a transition
6976 * to link up, so do that.
6977 */
6978 clear_linkup_counters(dd);
6979 /*
6980 * And (re)set link up default values.
6981 */
6982 set_linkup_defaults(ppd);
6983
6984 /*
6985 * Set VL15 credits. Use cached value from verify cap interrupt.
6986 * In case of quick linkup or simulator, vl15 value will be set by
6987 * handle_linkup_change. VerifyCap interrupt handler will not be
6988 * called in those scenarios.
6989 */
6990 if (!(quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR))
6991 set_up_vl15(dd, dd->vl15buf_cached);
6992
6993 /* enforce link speed enabled */
6994 if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) {
6995 /* oops - current speed is not enabled, bounce */
6996 dd_dev_err(dd,
6997 "Link speed active 0x%x is outside enabled 0x%x, downing link\n",
6998 ppd->link_speed_active, ppd->link_speed_enabled);
6999 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0,
7000 OPA_LINKDOWN_REASON_SPEED_POLICY);
7001 set_link_state(ppd, HLS_DN_OFFLINE);
7002 start_link(ppd);
7003 }
7004 }
7005
7006 /*
7007 * Several pieces of LNI information were cached for SMA in ppd.
7008 * Reset these on link down
7009 */
reset_neighbor_info(struct hfi1_pportdata * ppd)7010 static void reset_neighbor_info(struct hfi1_pportdata *ppd)
7011 {
7012 ppd->neighbor_guid = 0;
7013 ppd->neighbor_port_number = 0;
7014 ppd->neighbor_type = 0;
7015 ppd->neighbor_fm_security = 0;
7016 }
7017
7018 static const char * const link_down_reason_strs[] = {
7019 [OPA_LINKDOWN_REASON_NONE] = "None",
7020 [OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Receive error 0",
7021 [OPA_LINKDOWN_REASON_BAD_PKT_LEN] = "Bad packet length",
7022 [OPA_LINKDOWN_REASON_PKT_TOO_LONG] = "Packet too long",
7023 [OPA_LINKDOWN_REASON_PKT_TOO_SHORT] = "Packet too short",
7024 [OPA_LINKDOWN_REASON_BAD_SLID] = "Bad SLID",
7025 [OPA_LINKDOWN_REASON_BAD_DLID] = "Bad DLID",
7026 [OPA_LINKDOWN_REASON_BAD_L2] = "Bad L2",
7027 [OPA_LINKDOWN_REASON_BAD_SC] = "Bad SC",
7028 [OPA_LINKDOWN_REASON_RCV_ERROR_8] = "Receive error 8",
7029 [OPA_LINKDOWN_REASON_BAD_MID_TAIL] = "Bad mid tail",
7030 [OPA_LINKDOWN_REASON_RCV_ERROR_10] = "Receive error 10",
7031 [OPA_LINKDOWN_REASON_PREEMPT_ERROR] = "Preempt error",
7032 [OPA_LINKDOWN_REASON_PREEMPT_VL15] = "Preempt vl15",
7033 [OPA_LINKDOWN_REASON_BAD_VL_MARKER] = "Bad VL marker",
7034 [OPA_LINKDOWN_REASON_RCV_ERROR_14] = "Receive error 14",
7035 [OPA_LINKDOWN_REASON_RCV_ERROR_15] = "Receive error 15",
7036 [OPA_LINKDOWN_REASON_BAD_HEAD_DIST] = "Bad head distance",
7037 [OPA_LINKDOWN_REASON_BAD_TAIL_DIST] = "Bad tail distance",
7038 [OPA_LINKDOWN_REASON_BAD_CTRL_DIST] = "Bad control distance",
7039 [OPA_LINKDOWN_REASON_BAD_CREDIT_ACK] = "Bad credit ack",
7040 [OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER] = "Unsupported VL marker",
7041 [OPA_LINKDOWN_REASON_BAD_PREEMPT] = "Bad preempt",
7042 [OPA_LINKDOWN_REASON_BAD_CONTROL_FLIT] = "Bad control flit",
7043 [OPA_LINKDOWN_REASON_EXCEED_MULTICAST_LIMIT] = "Exceed multicast limit",
7044 [OPA_LINKDOWN_REASON_RCV_ERROR_24] = "Receive error 24",
7045 [OPA_LINKDOWN_REASON_RCV_ERROR_25] = "Receive error 25",
7046 [OPA_LINKDOWN_REASON_RCV_ERROR_26] = "Receive error 26",
7047 [OPA_LINKDOWN_REASON_RCV_ERROR_27] = "Receive error 27",
7048 [OPA_LINKDOWN_REASON_RCV_ERROR_28] = "Receive error 28",
7049 [OPA_LINKDOWN_REASON_RCV_ERROR_29] = "Receive error 29",
7050 [OPA_LINKDOWN_REASON_RCV_ERROR_30] = "Receive error 30",
7051 [OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN] =
7052 "Excessive buffer overrun",
7053 [OPA_LINKDOWN_REASON_UNKNOWN] = "Unknown",
7054 [OPA_LINKDOWN_REASON_REBOOT] = "Reboot",
7055 [OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN] = "Neighbor unknown",
7056 [OPA_LINKDOWN_REASON_FM_BOUNCE] = "FM bounce",
7057 [OPA_LINKDOWN_REASON_SPEED_POLICY] = "Speed policy",
7058 [OPA_LINKDOWN_REASON_WIDTH_POLICY] = "Width policy",
7059 [OPA_LINKDOWN_REASON_DISCONNECTED] = "Disconnected",
7060 [OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED] =
7061 "Local media not installed",
7062 [OPA_LINKDOWN_REASON_NOT_INSTALLED] = "Not installed",
7063 [OPA_LINKDOWN_REASON_CHASSIS_CONFIG] = "Chassis config",
7064 [OPA_LINKDOWN_REASON_END_TO_END_NOT_INSTALLED] =
7065 "End to end not installed",
7066 [OPA_LINKDOWN_REASON_POWER_POLICY] = "Power policy",
7067 [OPA_LINKDOWN_REASON_LINKSPEED_POLICY] = "Link speed policy",
7068 [OPA_LINKDOWN_REASON_LINKWIDTH_POLICY] = "Link width policy",
7069 [OPA_LINKDOWN_REASON_SWITCH_MGMT] = "Switch management",
7070 [OPA_LINKDOWN_REASON_SMA_DISABLED] = "SMA disabled",
7071 [OPA_LINKDOWN_REASON_TRANSIENT] = "Transient"
7072 };
7073
7074 /* return the neighbor link down reason string */
link_down_reason_str(u8 reason)7075 static const char *link_down_reason_str(u8 reason)
7076 {
7077 const char *str = NULL;
7078
7079 if (reason < ARRAY_SIZE(link_down_reason_strs))
7080 str = link_down_reason_strs[reason];
7081 if (!str)
7082 str = "(invalid)";
7083
7084 return str;
7085 }
7086
7087 /*
7088 * Handle a link down interrupt from the 8051.
7089 *
7090 * This is a work-queue function outside of the interrupt.
7091 */
handle_link_down(struct work_struct * work)7092 void handle_link_down(struct work_struct *work)
7093 {
7094 u8 lcl_reason, neigh_reason = 0;
7095 u8 link_down_reason;
7096 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7097 link_down_work);
7098 int was_up;
7099 static const char ldr_str[] = "Link down reason: ";
7100
7101 if ((ppd->host_link_state &
7102 (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) &&
7103 ppd->port_type == PORT_TYPE_FIXED)
7104 ppd->offline_disabled_reason =
7105 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NOT_INSTALLED);
7106
7107 /* Go offline first, then deal with reading/writing through 8051 */
7108 was_up = !!(ppd->host_link_state & HLS_UP);
7109 set_link_state(ppd, HLS_DN_OFFLINE);
7110 xchg(&ppd->is_link_down_queued, 0);
7111
7112 if (was_up) {
7113 lcl_reason = 0;
7114 /* link down reason is only valid if the link was up */
7115 read_link_down_reason(ppd->dd, &link_down_reason);
7116 switch (link_down_reason) {
7117 case LDR_LINK_TRANSFER_ACTIVE_LOW:
7118 /* the link went down, no idle message reason */
7119 dd_dev_info(ppd->dd, "%sUnexpected link down\n",
7120 ldr_str);
7121 break;
7122 case LDR_RECEIVED_LINKDOWN_IDLE_MSG:
7123 /*
7124 * The neighbor reason is only valid if an idle message
7125 * was received for it.
7126 */
7127 read_planned_down_reason_code(ppd->dd, &neigh_reason);
7128 dd_dev_info(ppd->dd,
7129 "%sNeighbor link down message %d, %s\n",
7130 ldr_str, neigh_reason,
7131 link_down_reason_str(neigh_reason));
7132 break;
7133 case LDR_RECEIVED_HOST_OFFLINE_REQ:
7134 dd_dev_info(ppd->dd,
7135 "%sHost requested link to go offline\n",
7136 ldr_str);
7137 break;
7138 default:
7139 dd_dev_info(ppd->dd, "%sUnknown reason 0x%x\n",
7140 ldr_str, link_down_reason);
7141 break;
7142 }
7143
7144 /*
7145 * If no reason, assume peer-initiated but missed
7146 * LinkGoingDown idle flits.
7147 */
7148 if (neigh_reason == 0)
7149 lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN;
7150 } else {
7151 /* went down while polling or going up */
7152 lcl_reason = OPA_LINKDOWN_REASON_TRANSIENT;
7153 }
7154
7155 set_link_down_reason(ppd, lcl_reason, neigh_reason, 0);
7156
7157 /* inform the SMA when the link transitions from up to down */
7158 if (was_up && ppd->local_link_down_reason.sma == 0 &&
7159 ppd->neigh_link_down_reason.sma == 0) {
7160 ppd->local_link_down_reason.sma =
7161 ppd->local_link_down_reason.latest;
7162 ppd->neigh_link_down_reason.sma =
7163 ppd->neigh_link_down_reason.latest;
7164 }
7165
7166 reset_neighbor_info(ppd);
7167
7168 /* disable the port */
7169 clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
7170
7171 /*
7172 * If there is no cable attached, turn the DC off. Otherwise,
7173 * start the link bring up.
7174 */
7175 if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd))
7176 dc_shutdown(ppd->dd);
7177 else
7178 start_link(ppd);
7179 }
7180
handle_link_bounce(struct work_struct * work)7181 void handle_link_bounce(struct work_struct *work)
7182 {
7183 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7184 link_bounce_work);
7185
7186 /*
7187 * Only do something if the link is currently up.
7188 */
7189 if (ppd->host_link_state & HLS_UP) {
7190 set_link_state(ppd, HLS_DN_OFFLINE);
7191 start_link(ppd);
7192 } else {
7193 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n",
7194 __func__, link_state_name(ppd->host_link_state));
7195 }
7196 }
7197
7198 /*
7199 * Mask conversion: Capability exchange to Port LTP. The capability
7200 * exchange has an implicit 16b CRC that is mandatory.
7201 */
cap_to_port_ltp(int cap)7202 static int cap_to_port_ltp(int cap)
7203 {
7204 int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */
7205
7206 if (cap & CAP_CRC_14B)
7207 port_ltp |= PORT_LTP_CRC_MODE_14;
7208 if (cap & CAP_CRC_48B)
7209 port_ltp |= PORT_LTP_CRC_MODE_48;
7210 if (cap & CAP_CRC_12B_16B_PER_LANE)
7211 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE;
7212
7213 return port_ltp;
7214 }
7215
7216 /*
7217 * Convert an OPA Port LTP mask to capability mask
7218 */
port_ltp_to_cap(int port_ltp)7219 int port_ltp_to_cap(int port_ltp)
7220 {
7221 int cap_mask = 0;
7222
7223 if (port_ltp & PORT_LTP_CRC_MODE_14)
7224 cap_mask |= CAP_CRC_14B;
7225 if (port_ltp & PORT_LTP_CRC_MODE_48)
7226 cap_mask |= CAP_CRC_48B;
7227 if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE)
7228 cap_mask |= CAP_CRC_12B_16B_PER_LANE;
7229
7230 return cap_mask;
7231 }
7232
7233 /*
7234 * Convert a single DC LCB CRC mode to an OPA Port LTP mask.
7235 */
lcb_to_port_ltp(int lcb_crc)7236 static int lcb_to_port_ltp(int lcb_crc)
7237 {
7238 int port_ltp = 0;
7239
7240 if (lcb_crc == LCB_CRC_12B_16B_PER_LANE)
7241 port_ltp = PORT_LTP_CRC_MODE_PER_LANE;
7242 else if (lcb_crc == LCB_CRC_48B)
7243 port_ltp = PORT_LTP_CRC_MODE_48;
7244 else if (lcb_crc == LCB_CRC_14B)
7245 port_ltp = PORT_LTP_CRC_MODE_14;
7246 else
7247 port_ltp = PORT_LTP_CRC_MODE_16;
7248
7249 return port_ltp;
7250 }
7251
clear_full_mgmt_pkey(struct hfi1_pportdata * ppd)7252 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd)
7253 {
7254 if (ppd->pkeys[2] != 0) {
7255 ppd->pkeys[2] = 0;
7256 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
7257 hfi1_event_pkey_change(ppd->dd, ppd->port);
7258 }
7259 }
7260
7261 /*
7262 * Convert the given link width to the OPA link width bitmask.
7263 */
link_width_to_bits(struct hfi1_devdata * dd,u16 width)7264 static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width)
7265 {
7266 switch (width) {
7267 case 0:
7268 /*
7269 * Simulator and quick linkup do not set the width.
7270 * Just set it to 4x without complaint.
7271 */
7272 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup)
7273 return OPA_LINK_WIDTH_4X;
7274 return 0; /* no lanes up */
7275 case 1: return OPA_LINK_WIDTH_1X;
7276 case 2: return OPA_LINK_WIDTH_2X;
7277 case 3: return OPA_LINK_WIDTH_3X;
7278 case 4: return OPA_LINK_WIDTH_4X;
7279 default:
7280 dd_dev_info(dd, "%s: invalid width %d, using 4\n",
7281 __func__, width);
7282 return OPA_LINK_WIDTH_4X;
7283 }
7284 }
7285
7286 /*
7287 * Do a population count on the bottom nibble.
7288 */
7289 static const u8 bit_counts[16] = {
7290 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4
7291 };
7292
nibble_to_count(u8 nibble)7293 static inline u8 nibble_to_count(u8 nibble)
7294 {
7295 return bit_counts[nibble & 0xf];
7296 }
7297
7298 /*
7299 * Read the active lane information from the 8051 registers and return
7300 * their widths.
7301 *
7302 * Active lane information is found in these 8051 registers:
7303 * enable_lane_tx
7304 * enable_lane_rx
7305 */
get_link_widths(struct hfi1_devdata * dd,u16 * tx_width,u16 * rx_width)7306 static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width,
7307 u16 *rx_width)
7308 {
7309 u16 tx, rx;
7310 u8 enable_lane_rx;
7311 u8 enable_lane_tx;
7312 u8 tx_polarity_inversion;
7313 u8 rx_polarity_inversion;
7314 u8 max_rate;
7315
7316 /* read the active lanes */
7317 read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
7318 &rx_polarity_inversion, &max_rate);
7319 read_local_lni(dd, &enable_lane_rx);
7320
7321 /* convert to counts */
7322 tx = nibble_to_count(enable_lane_tx);
7323 rx = nibble_to_count(enable_lane_rx);
7324
7325 /*
7326 * Set link_speed_active here, overriding what was set in
7327 * handle_verify_cap(). The ASIC 8051 firmware does not correctly
7328 * set the max_rate field in handle_verify_cap until v0.19.
7329 */
7330 if ((dd->icode == ICODE_RTL_SILICON) &&
7331 (dd->dc8051_ver < dc8051_ver(0, 19, 0))) {
7332 /* max_rate: 0 = 12.5G, 1 = 25G */
7333 switch (max_rate) {
7334 case 0:
7335 dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;
7336 break;
7337 case 1:
7338 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
7339 break;
7340 default:
7341 dd_dev_err(dd,
7342 "%s: unexpected max rate %d, using 25Gb\n",
7343 __func__, (int)max_rate);
7344 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
7345 break;
7346 }
7347 }
7348
7349 dd_dev_info(dd,
7350 "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n",
7351 enable_lane_tx, tx, enable_lane_rx, rx);
7352 *tx_width = link_width_to_bits(dd, tx);
7353 *rx_width = link_width_to_bits(dd, rx);
7354 }
7355
7356 /*
7357 * Read verify_cap_local_fm_link_width[1] to obtain the link widths.
7358 * Valid after the end of VerifyCap and during LinkUp. Does not change
7359 * after link up. I.e. look elsewhere for downgrade information.
7360 *
7361 * Bits are:
7362 * + bits [7:4] contain the number of active transmitters
7363 * + bits [3:0] contain the number of active receivers
7364 * These are numbers 1 through 4 and can be different values if the
7365 * link is asymmetric.
7366 *
7367 * verify_cap_local_fm_link_width[0] retains its original value.
7368 */
get_linkup_widths(struct hfi1_devdata * dd,u16 * tx_width,u16 * rx_width)7369 static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width,
7370 u16 *rx_width)
7371 {
7372 u16 widths, tx, rx;
7373 u8 misc_bits, local_flags;
7374 u16 active_tx, active_rx;
7375
7376 read_vc_local_link_mode(dd, &misc_bits, &local_flags, &widths);
7377 tx = widths >> 12;
7378 rx = (widths >> 8) & 0xf;
7379
7380 *tx_width = link_width_to_bits(dd, tx);
7381 *rx_width = link_width_to_bits(dd, rx);
7382
7383 /* print the active widths */
7384 get_link_widths(dd, &active_tx, &active_rx);
7385 }
7386
7387 /*
7388 * Set ppd->link_width_active and ppd->link_width_downgrade_active using
7389 * hardware information when the link first comes up.
7390 *
7391 * The link width is not available until after VerifyCap.AllFramesReceived
7392 * (the trigger for handle_verify_cap), so this is outside that routine
7393 * and should be called when the 8051 signals linkup.
7394 */
get_linkup_link_widths(struct hfi1_pportdata * ppd)7395 void get_linkup_link_widths(struct hfi1_pportdata *ppd)
7396 {
7397 u16 tx_width, rx_width;
7398
7399 /* get end-of-LNI link widths */
7400 get_linkup_widths(ppd->dd, &tx_width, &rx_width);
7401
7402 /* use tx_width as the link is supposed to be symmetric on link up */
7403 ppd->link_width_active = tx_width;
7404 /* link width downgrade active (LWD.A) starts out matching LW.A */
7405 ppd->link_width_downgrade_tx_active = ppd->link_width_active;
7406 ppd->link_width_downgrade_rx_active = ppd->link_width_active;
7407 /* per OPA spec, on link up LWD.E resets to LWD.S */
7408 ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported;
7409 /* cache the active egress rate (units {10^6 bits/sec]) */
7410 ppd->current_egress_rate = active_egress_rate(ppd);
7411 }
7412
7413 /*
7414 * Handle a verify capabilities interrupt from the 8051.
7415 *
7416 * This is a work-queue function outside of the interrupt.
7417 */
handle_verify_cap(struct work_struct * work)7418 void handle_verify_cap(struct work_struct *work)
7419 {
7420 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7421 link_vc_work);
7422 struct hfi1_devdata *dd = ppd->dd;
7423 u64 reg;
7424 u8 power_management;
7425 u8 continuous;
7426 u8 vcu;
7427 u8 vau;
7428 u8 z;
7429 u16 vl15buf;
7430 u16 link_widths;
7431 u16 crc_mask;
7432 u16 crc_val;
7433 u16 device_id;
7434 u16 active_tx, active_rx;
7435 u8 partner_supported_crc;
7436 u8 remote_tx_rate;
7437 u8 device_rev;
7438
7439 set_link_state(ppd, HLS_VERIFY_CAP);
7440
7441 lcb_shutdown(dd, 0);
7442 adjust_lcb_for_fpga_serdes(dd);
7443
7444 read_vc_remote_phy(dd, &power_management, &continuous);
7445 read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf,
7446 &partner_supported_crc);
7447 read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths);
7448 read_remote_device_id(dd, &device_id, &device_rev);
7449
7450 /* print the active widths */
7451 get_link_widths(dd, &active_tx, &active_rx);
7452 dd_dev_info(dd,
7453 "Peer PHY: power management 0x%x, continuous updates 0x%x\n",
7454 (int)power_management, (int)continuous);
7455 dd_dev_info(dd,
7456 "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n",
7457 (int)vau, (int)z, (int)vcu, (int)vl15buf,
7458 (int)partner_supported_crc);
7459 dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n",
7460 (u32)remote_tx_rate, (u32)link_widths);
7461 dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n",
7462 (u32)device_id, (u32)device_rev);
7463 /*
7464 * The peer vAU value just read is the peer receiver value. HFI does
7465 * not support a transmit vAU of 0 (AU == 8). We advertised that
7466 * with Z=1 in the fabric capabilities sent to the peer. The peer
7467 * will see our Z=1, and, if it advertised a vAU of 0, will move its
7468 * receive to vAU of 1 (AU == 16). Do the same here. We do not care
7469 * about the peer Z value - our sent vAU is 3 (hardwired) and is not
7470 * subject to the Z value exception.
7471 */
7472 if (vau == 0)
7473 vau = 1;
7474 set_up_vau(dd, vau);
7475
7476 /*
7477 * Set VL15 credits to 0 in global credit register. Cache remote VL15
7478 * credits value and wait for link-up interrupt ot set it.
7479 */
7480 set_up_vl15(dd, 0);
7481 dd->vl15buf_cached = vl15buf;
7482
7483 /* set up the LCB CRC mode */
7484 crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc;
7485
7486 /* order is important: use the lowest bit in common */
7487 if (crc_mask & CAP_CRC_14B)
7488 crc_val = LCB_CRC_14B;
7489 else if (crc_mask & CAP_CRC_48B)
7490 crc_val = LCB_CRC_48B;
7491 else if (crc_mask & CAP_CRC_12B_16B_PER_LANE)
7492 crc_val = LCB_CRC_12B_16B_PER_LANE;
7493 else
7494 crc_val = LCB_CRC_16B;
7495
7496 dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val);
7497 write_csr(dd, DC_LCB_CFG_CRC_MODE,
7498 (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT);
7499
7500 /* set (14b only) or clear sideband credit */
7501 reg = read_csr(dd, SEND_CM_CTRL);
7502 if (crc_val == LCB_CRC_14B && crc_14b_sideband) {
7503 write_csr(dd, SEND_CM_CTRL,
7504 reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
7505 } else {
7506 write_csr(dd, SEND_CM_CTRL,
7507 reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
7508 }
7509
7510 ppd->link_speed_active = 0; /* invalid value */
7511 if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) {
7512 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */
7513 switch (remote_tx_rate) {
7514 case 0:
7515 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7516 break;
7517 case 1:
7518 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7519 break;
7520 }
7521 } else {
7522 /* actual rate is highest bit of the ANDed rates */
7523 u8 rate = remote_tx_rate & ppd->local_tx_rate;
7524
7525 if (rate & 2)
7526 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7527 else if (rate & 1)
7528 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7529 }
7530 if (ppd->link_speed_active == 0) {
7531 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n",
7532 __func__, (int)remote_tx_rate);
7533 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7534 }
7535
7536 /*
7537 * Cache the values of the supported, enabled, and active
7538 * LTP CRC modes to return in 'portinfo' queries. But the bit
7539 * flags that are returned in the portinfo query differ from
7540 * what's in the link_crc_mask, crc_sizes, and crc_val
7541 * variables. Convert these here.
7542 */
7543 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
7544 /* supported crc modes */
7545 ppd->port_ltp_crc_mode |=
7546 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4;
7547 /* enabled crc modes */
7548 ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val);
7549 /* active crc mode */
7550
7551 /* set up the remote credit return table */
7552 assign_remote_cm_au_table(dd, vcu);
7553
7554 /*
7555 * The LCB is reset on entry to handle_verify_cap(), so this must
7556 * be applied on every link up.
7557 *
7558 * Adjust LCB error kill enable to kill the link if
7559 * these RBUF errors are seen:
7560 * REPLAY_BUF_MBE_SMASK
7561 * FLIT_INPUT_BUF_MBE_SMASK
7562 */
7563 if (is_ax(dd)) { /* fixed in B0 */
7564 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN);
7565 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK
7566 | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK;
7567 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg);
7568 }
7569
7570 /* pull LCB fifos out of reset - all fifo clocks must be stable */
7571 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
7572
7573 /* give 8051 access to the LCB CSRs */
7574 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
7575 set_8051_lcb_access(dd);
7576
7577 /* tell the 8051 to go to LinkUp */
7578 set_link_state(ppd, HLS_GOING_UP);
7579 }
7580
7581 /**
7582 * apply_link_downgrade_policy - Apply the link width downgrade enabled
7583 * policy against the current active link widths.
7584 * @ppd: info of physical Hfi port
7585 * @refresh_widths: True indicates link downgrade event
7586 * @return: True indicates a successful link downgrade. False indicates
7587 * link downgrade event failed and the link will bounce back to
7588 * default link width.
7589 *
7590 * Called when the enabled policy changes or the active link widths
7591 * change.
7592 * Refresh_widths indicates that a link downgrade occurred. The
7593 * link_downgraded variable is set by refresh_widths and
7594 * determines the success/failure of the policy application.
7595 */
apply_link_downgrade_policy(struct hfi1_pportdata * ppd,bool refresh_widths)7596 bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd,
7597 bool refresh_widths)
7598 {
7599 int do_bounce = 0;
7600 int tries;
7601 u16 lwde;
7602 u16 tx, rx;
7603 bool link_downgraded = refresh_widths;
7604
7605 /* use the hls lock to avoid a race with actual link up */
7606 tries = 0;
7607 retry:
7608 mutex_lock(&ppd->hls_lock);
7609 /* only apply if the link is up */
7610 if (ppd->host_link_state & HLS_DOWN) {
7611 /* still going up..wait and retry */
7612 if (ppd->host_link_state & HLS_GOING_UP) {
7613 if (++tries < 1000) {
7614 mutex_unlock(&ppd->hls_lock);
7615 usleep_range(100, 120); /* arbitrary */
7616 goto retry;
7617 }
7618 dd_dev_err(ppd->dd,
7619 "%s: giving up waiting for link state change\n",
7620 __func__);
7621 }
7622 goto done;
7623 }
7624
7625 lwde = ppd->link_width_downgrade_enabled;
7626
7627 if (refresh_widths) {
7628 get_link_widths(ppd->dd, &tx, &rx);
7629 ppd->link_width_downgrade_tx_active = tx;
7630 ppd->link_width_downgrade_rx_active = rx;
7631 }
7632
7633 if (ppd->link_width_downgrade_tx_active == 0 ||
7634 ppd->link_width_downgrade_rx_active == 0) {
7635 /* the 8051 reported a dead link as a downgrade */
7636 dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n");
7637 link_downgraded = false;
7638 } else if (lwde == 0) {
7639 /* downgrade is disabled */
7640
7641 /* bounce if not at starting active width */
7642 if ((ppd->link_width_active !=
7643 ppd->link_width_downgrade_tx_active) ||
7644 (ppd->link_width_active !=
7645 ppd->link_width_downgrade_rx_active)) {
7646 dd_dev_err(ppd->dd,
7647 "Link downgrade is disabled and link has downgraded, downing link\n");
7648 dd_dev_err(ppd->dd,
7649 " original 0x%x, tx active 0x%x, rx active 0x%x\n",
7650 ppd->link_width_active,
7651 ppd->link_width_downgrade_tx_active,
7652 ppd->link_width_downgrade_rx_active);
7653 do_bounce = 1;
7654 link_downgraded = false;
7655 }
7656 } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 ||
7657 (lwde & ppd->link_width_downgrade_rx_active) == 0) {
7658 /* Tx or Rx is outside the enabled policy */
7659 dd_dev_err(ppd->dd,
7660 "Link is outside of downgrade allowed, downing link\n");
7661 dd_dev_err(ppd->dd,
7662 " enabled 0x%x, tx active 0x%x, rx active 0x%x\n",
7663 lwde, ppd->link_width_downgrade_tx_active,
7664 ppd->link_width_downgrade_rx_active);
7665 do_bounce = 1;
7666 link_downgraded = false;
7667 }
7668
7669 done:
7670 mutex_unlock(&ppd->hls_lock);
7671
7672 if (do_bounce) {
7673 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0,
7674 OPA_LINKDOWN_REASON_WIDTH_POLICY);
7675 set_link_state(ppd, HLS_DN_OFFLINE);
7676 start_link(ppd);
7677 }
7678
7679 return link_downgraded;
7680 }
7681
7682 /*
7683 * Handle a link downgrade interrupt from the 8051.
7684 *
7685 * This is a work-queue function outside of the interrupt.
7686 */
handle_link_downgrade(struct work_struct * work)7687 void handle_link_downgrade(struct work_struct *work)
7688 {
7689 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7690 link_downgrade_work);
7691
7692 dd_dev_info(ppd->dd, "8051: Link width downgrade\n");
7693 if (apply_link_downgrade_policy(ppd, true))
7694 update_xmit_counters(ppd, ppd->link_width_downgrade_tx_active);
7695 }
7696
dcc_err_string(char * buf,int buf_len,u64 flags)7697 static char *dcc_err_string(char *buf, int buf_len, u64 flags)
7698 {
7699 return flag_string(buf, buf_len, flags, dcc_err_flags,
7700 ARRAY_SIZE(dcc_err_flags));
7701 }
7702
lcb_err_string(char * buf,int buf_len,u64 flags)7703 static char *lcb_err_string(char *buf, int buf_len, u64 flags)
7704 {
7705 return flag_string(buf, buf_len, flags, lcb_err_flags,
7706 ARRAY_SIZE(lcb_err_flags));
7707 }
7708
dc8051_err_string(char * buf,int buf_len,u64 flags)7709 static char *dc8051_err_string(char *buf, int buf_len, u64 flags)
7710 {
7711 return flag_string(buf, buf_len, flags, dc8051_err_flags,
7712 ARRAY_SIZE(dc8051_err_flags));
7713 }
7714
dc8051_info_err_string(char * buf,int buf_len,u64 flags)7715 static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags)
7716 {
7717 return flag_string(buf, buf_len, flags, dc8051_info_err_flags,
7718 ARRAY_SIZE(dc8051_info_err_flags));
7719 }
7720
dc8051_info_host_msg_string(char * buf,int buf_len,u64 flags)7721 static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags)
7722 {
7723 return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags,
7724 ARRAY_SIZE(dc8051_info_host_msg_flags));
7725 }
7726
handle_8051_interrupt(struct hfi1_devdata * dd,u32 unused,u64 reg)7727 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
7728 {
7729 struct hfi1_pportdata *ppd = dd->pport;
7730 u64 info, err, host_msg;
7731 int queue_link_down = 0;
7732 char buf[96];
7733
7734 /* look at the flags */
7735 if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) {
7736 /* 8051 information set by firmware */
7737 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */
7738 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051);
7739 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT)
7740 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK;
7741 host_msg = (info >>
7742 DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT)
7743 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK;
7744
7745 /*
7746 * Handle error flags.
7747 */
7748 if (err & FAILED_LNI) {
7749 /*
7750 * LNI error indications are cleared by the 8051
7751 * only when starting polling. Only pay attention
7752 * to them when in the states that occur during
7753 * LNI.
7754 */
7755 if (ppd->host_link_state
7756 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
7757 queue_link_down = 1;
7758 dd_dev_info(dd, "Link error: %s\n",
7759 dc8051_info_err_string(buf,
7760 sizeof(buf),
7761 err &
7762 FAILED_LNI));
7763 }
7764 err &= ~(u64)FAILED_LNI;
7765 }
7766 /* unknown frames can happen durning LNI, just count */
7767 if (err & UNKNOWN_FRAME) {
7768 ppd->unknown_frame_count++;
7769 err &= ~(u64)UNKNOWN_FRAME;
7770 }
7771 if (err) {
7772 /* report remaining errors, but do not do anything */
7773 dd_dev_err(dd, "8051 info error: %s\n",
7774 dc8051_info_err_string(buf, sizeof(buf),
7775 err));
7776 }
7777
7778 /*
7779 * Handle host message flags.
7780 */
7781 if (host_msg & HOST_REQ_DONE) {
7782 /*
7783 * Presently, the driver does a busy wait for
7784 * host requests to complete. This is only an
7785 * informational message.
7786 * NOTE: The 8051 clears the host message
7787 * information *on the next 8051 command*.
7788 * Therefore, when linkup is achieved,
7789 * this flag will still be set.
7790 */
7791 host_msg &= ~(u64)HOST_REQ_DONE;
7792 }
7793 if (host_msg & BC_SMA_MSG) {
7794 queue_work(ppd->link_wq, &ppd->sma_message_work);
7795 host_msg &= ~(u64)BC_SMA_MSG;
7796 }
7797 if (host_msg & LINKUP_ACHIEVED) {
7798 dd_dev_info(dd, "8051: Link up\n");
7799 queue_work(ppd->link_wq, &ppd->link_up_work);
7800 host_msg &= ~(u64)LINKUP_ACHIEVED;
7801 }
7802 if (host_msg & EXT_DEVICE_CFG_REQ) {
7803 handle_8051_request(ppd);
7804 host_msg &= ~(u64)EXT_DEVICE_CFG_REQ;
7805 }
7806 if (host_msg & VERIFY_CAP_FRAME) {
7807 queue_work(ppd->link_wq, &ppd->link_vc_work);
7808 host_msg &= ~(u64)VERIFY_CAP_FRAME;
7809 }
7810 if (host_msg & LINK_GOING_DOWN) {
7811 const char *extra = "";
7812 /* no downgrade action needed if going down */
7813 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7814 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7815 extra = " (ignoring downgrade)";
7816 }
7817 dd_dev_info(dd, "8051: Link down%s\n", extra);
7818 queue_link_down = 1;
7819 host_msg &= ~(u64)LINK_GOING_DOWN;
7820 }
7821 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7822 queue_work(ppd->link_wq, &ppd->link_downgrade_work);
7823 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7824 }
7825 if (host_msg) {
7826 /* report remaining messages, but do not do anything */
7827 dd_dev_info(dd, "8051 info host message: %s\n",
7828 dc8051_info_host_msg_string(buf,
7829 sizeof(buf),
7830 host_msg));
7831 }
7832
7833 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK;
7834 }
7835 if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) {
7836 /*
7837 * Lost the 8051 heartbeat. If this happens, we
7838 * receive constant interrupts about it. Disable
7839 * the interrupt after the first.
7840 */
7841 dd_dev_err(dd, "Lost 8051 heartbeat\n");
7842 write_csr(dd, DC_DC8051_ERR_EN,
7843 read_csr(dd, DC_DC8051_ERR_EN) &
7844 ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK);
7845
7846 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK;
7847 }
7848 if (reg) {
7849 /* report the error, but do not do anything */
7850 dd_dev_err(dd, "8051 error: %s\n",
7851 dc8051_err_string(buf, sizeof(buf), reg));
7852 }
7853
7854 if (queue_link_down) {
7855 /*
7856 * if the link is already going down or disabled, do not
7857 * queue another. If there's a link down entry already
7858 * queued, don't queue another one.
7859 */
7860 if ((ppd->host_link_state &
7861 (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) ||
7862 ppd->link_enabled == 0) {
7863 dd_dev_info(dd, "%s: not queuing link down. host_link_state %x, link_enabled %x\n",
7864 __func__, ppd->host_link_state,
7865 ppd->link_enabled);
7866 } else {
7867 if (xchg(&ppd->is_link_down_queued, 1) == 1)
7868 dd_dev_info(dd,
7869 "%s: link down request already queued\n",
7870 __func__);
7871 else
7872 queue_work(ppd->link_wq, &ppd->link_down_work);
7873 }
7874 }
7875 }
7876
7877 static const char * const fm_config_txt[] = {
7878 [0] =
7879 "BadHeadDist: Distance violation between two head flits",
7880 [1] =
7881 "BadTailDist: Distance violation between two tail flits",
7882 [2] =
7883 "BadCtrlDist: Distance violation between two credit control flits",
7884 [3] =
7885 "BadCrdAck: Credits return for unsupported VL",
7886 [4] =
7887 "UnsupportedVLMarker: Received VL Marker",
7888 [5] =
7889 "BadPreempt: Exceeded the preemption nesting level",
7890 [6] =
7891 "BadControlFlit: Received unsupported control flit",
7892 /* no 7 */
7893 [8] =
7894 "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL",
7895 };
7896
7897 static const char * const port_rcv_txt[] = {
7898 [1] =
7899 "BadPktLen: Illegal PktLen",
7900 [2] =
7901 "PktLenTooLong: Packet longer than PktLen",
7902 [3] =
7903 "PktLenTooShort: Packet shorter than PktLen",
7904 [4] =
7905 "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)",
7906 [5] =
7907 "BadDLID: Illegal DLID (0, doesn't match HFI)",
7908 [6] =
7909 "BadL2: Illegal L2 opcode",
7910 [7] =
7911 "BadSC: Unsupported SC",
7912 [9] =
7913 "BadRC: Illegal RC",
7914 [11] =
7915 "PreemptError: Preempting with same VL",
7916 [12] =
7917 "PreemptVL15: Preempting a VL15 packet",
7918 };
7919
7920 #define OPA_LDR_FMCONFIG_OFFSET 16
7921 #define OPA_LDR_PORTRCV_OFFSET 0
handle_dcc_err(struct hfi1_devdata * dd,u32 unused,u64 reg)7922 static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
7923 {
7924 u64 info, hdr0, hdr1;
7925 const char *extra;
7926 char buf[96];
7927 struct hfi1_pportdata *ppd = dd->pport;
7928 u8 lcl_reason = 0;
7929 int do_bounce = 0;
7930
7931 if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) {
7932 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) {
7933 info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE);
7934 dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK;
7935 /* set status bit */
7936 dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK;
7937 }
7938 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK;
7939 }
7940
7941 if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) {
7942 struct hfi1_pportdata *ppd = dd->pport;
7943 /* this counter saturates at (2^32) - 1 */
7944 if (ppd->link_downed < (u32)UINT_MAX)
7945 ppd->link_downed++;
7946 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK;
7947 }
7948
7949 if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) {
7950 u8 reason_valid = 1;
7951
7952 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG);
7953 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) {
7954 dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK;
7955 /* set status bit */
7956 dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK;
7957 }
7958 switch (info) {
7959 case 0:
7960 case 1:
7961 case 2:
7962 case 3:
7963 case 4:
7964 case 5:
7965 case 6:
7966 extra = fm_config_txt[info];
7967 break;
7968 case 8:
7969 extra = fm_config_txt[info];
7970 if (ppd->port_error_action &
7971 OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) {
7972 do_bounce = 1;
7973 /*
7974 * lcl_reason cannot be derived from info
7975 * for this error
7976 */
7977 lcl_reason =
7978 OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER;
7979 }
7980 break;
7981 default:
7982 reason_valid = 0;
7983 snprintf(buf, sizeof(buf), "reserved%lld", info);
7984 extra = buf;
7985 break;
7986 }
7987
7988 if (reason_valid && !do_bounce) {
7989 do_bounce = ppd->port_error_action &
7990 (1 << (OPA_LDR_FMCONFIG_OFFSET + info));
7991 lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST;
7992 }
7993
7994 /* just report this */
7995 dd_dev_info_ratelimited(dd, "DCC Error: fmconfig error: %s\n",
7996 extra);
7997 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK;
7998 }
7999
8000 if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) {
8001 u8 reason_valid = 1;
8002
8003 info = read_csr(dd, DCC_ERR_INFO_PORTRCV);
8004 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0);
8005 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1);
8006 if (!(dd->err_info_rcvport.status_and_code &
8007 OPA_EI_STATUS_SMASK)) {
8008 dd->err_info_rcvport.status_and_code =
8009 info & OPA_EI_CODE_SMASK;
8010 /* set status bit */
8011 dd->err_info_rcvport.status_and_code |=
8012 OPA_EI_STATUS_SMASK;
8013 /*
8014 * save first 2 flits in the packet that caused
8015 * the error
8016 */
8017 dd->err_info_rcvport.packet_flit1 = hdr0;
8018 dd->err_info_rcvport.packet_flit2 = hdr1;
8019 }
8020 switch (info) {
8021 case 1:
8022 case 2:
8023 case 3:
8024 case 4:
8025 case 5:
8026 case 6:
8027 case 7:
8028 case 9:
8029 case 11:
8030 case 12:
8031 extra = port_rcv_txt[info];
8032 break;
8033 default:
8034 reason_valid = 0;
8035 snprintf(buf, sizeof(buf), "reserved%lld", info);
8036 extra = buf;
8037 break;
8038 }
8039
8040 if (reason_valid && !do_bounce) {
8041 do_bounce = ppd->port_error_action &
8042 (1 << (OPA_LDR_PORTRCV_OFFSET + info));
8043 lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0;
8044 }
8045
8046 /* just report this */
8047 dd_dev_info_ratelimited(dd, "DCC Error: PortRcv error: %s\n"
8048 " hdr0 0x%llx, hdr1 0x%llx\n",
8049 extra, hdr0, hdr1);
8050
8051 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK;
8052 }
8053
8054 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) {
8055 /* informative only */
8056 dd_dev_info_ratelimited(dd, "8051 access to LCB blocked\n");
8057 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK;
8058 }
8059 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) {
8060 /* informative only */
8061 dd_dev_info_ratelimited(dd, "host access to LCB blocked\n");
8062 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK;
8063 }
8064
8065 if (unlikely(hfi1_dbg_fault_suppress_err(&dd->verbs_dev)))
8066 reg &= ~DCC_ERR_FLG_LATE_EBP_ERR_SMASK;
8067
8068 /* report any remaining errors */
8069 if (reg)
8070 dd_dev_info_ratelimited(dd, "DCC Error: %s\n",
8071 dcc_err_string(buf, sizeof(buf), reg));
8072
8073 if (lcl_reason == 0)
8074 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN;
8075
8076 if (do_bounce) {
8077 dd_dev_info_ratelimited(dd, "%s: PortErrorAction bounce\n",
8078 __func__);
8079 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason);
8080 queue_work(ppd->link_wq, &ppd->link_bounce_work);
8081 }
8082 }
8083
handle_lcb_err(struct hfi1_devdata * dd,u32 unused,u64 reg)8084 static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
8085 {
8086 char buf[96];
8087
8088 dd_dev_info(dd, "LCB Error: %s\n",
8089 lcb_err_string(buf, sizeof(buf), reg));
8090 }
8091
8092 /*
8093 * CCE block DC interrupt. Source is < 8.
8094 */
is_dc_int(struct hfi1_devdata * dd,unsigned int source)8095 static void is_dc_int(struct hfi1_devdata *dd, unsigned int source)
8096 {
8097 const struct err_reg_info *eri = &dc_errs[source];
8098
8099 if (eri->handler) {
8100 interrupt_clear_down(dd, 0, eri);
8101 } else if (source == 3 /* dc_lbm_int */) {
8102 /*
8103 * This indicates that a parity error has occurred on the
8104 * address/control lines presented to the LBM. The error
8105 * is a single pulse, there is no associated error flag,
8106 * and it is non-maskable. This is because if a parity
8107 * error occurs on the request the request is dropped.
8108 * This should never occur, but it is nice to know if it
8109 * ever does.
8110 */
8111 dd_dev_err(dd, "Parity error in DC LBM block\n");
8112 } else {
8113 dd_dev_err(dd, "Invalid DC interrupt %u\n", source);
8114 }
8115 }
8116
8117 /*
8118 * TX block send credit interrupt. Source is < 160.
8119 */
is_send_credit_int(struct hfi1_devdata * dd,unsigned int source)8120 static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source)
8121 {
8122 sc_group_release_update(dd, source);
8123 }
8124
8125 /*
8126 * TX block SDMA interrupt. Source is < 48.
8127 *
8128 * SDMA interrupts are grouped by type:
8129 *
8130 * 0 - N-1 = SDma
8131 * N - 2N-1 = SDmaProgress
8132 * 2N - 3N-1 = SDmaIdle
8133 */
is_sdma_eng_int(struct hfi1_devdata * dd,unsigned int source)8134 static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source)
8135 {
8136 /* what interrupt */
8137 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
8138 /* which engine */
8139 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
8140
8141 #ifdef CONFIG_SDMA_VERBOSITY
8142 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which,
8143 slashstrip(__FILE__), __LINE__, __func__);
8144 sdma_dumpstate(&dd->per_sdma[which]);
8145 #endif
8146
8147 if (likely(what < 3 && which < dd->num_sdma)) {
8148 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source);
8149 } else {
8150 /* should not happen */
8151 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source);
8152 }
8153 }
8154
8155 /**
8156 * is_rcv_avail_int() - User receive context available IRQ handler
8157 * @dd: valid dd
8158 * @source: logical IRQ source (offset from IS_RCVAVAIL_START)
8159 *
8160 * RX block receive available interrupt. Source is < 160.
8161 *
8162 * This is the general interrupt handler for user (PSM) receive contexts,
8163 * and can only be used for non-threaded IRQs.
8164 */
is_rcv_avail_int(struct hfi1_devdata * dd,unsigned int source)8165 static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
8166 {
8167 struct hfi1_ctxtdata *rcd;
8168 char *err_detail;
8169
8170 if (likely(source < dd->num_rcv_contexts)) {
8171 rcd = hfi1_rcd_get_by_index(dd, source);
8172 if (rcd) {
8173 handle_user_interrupt(rcd);
8174 hfi1_rcd_put(rcd);
8175 return; /* OK */
8176 }
8177 /* received an interrupt, but no rcd */
8178 err_detail = "dataless";
8179 } else {
8180 /* received an interrupt, but are not using that context */
8181 err_detail = "out of range";
8182 }
8183 dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n",
8184 err_detail, source);
8185 }
8186
8187 /**
8188 * is_rcv_urgent_int() - User receive context urgent IRQ handler
8189 * @dd: valid dd
8190 * @source: logical IRQ source (offset from IS_RCVURGENT_START)
8191 *
8192 * RX block receive urgent interrupt. Source is < 160.
8193 *
8194 * NOTE: kernel receive contexts specifically do NOT enable this IRQ.
8195 */
is_rcv_urgent_int(struct hfi1_devdata * dd,unsigned int source)8196 static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source)
8197 {
8198 struct hfi1_ctxtdata *rcd;
8199 char *err_detail;
8200
8201 if (likely(source < dd->num_rcv_contexts)) {
8202 rcd = hfi1_rcd_get_by_index(dd, source);
8203 if (rcd) {
8204 handle_user_interrupt(rcd);
8205 hfi1_rcd_put(rcd);
8206 return; /* OK */
8207 }
8208 /* received an interrupt, but no rcd */
8209 err_detail = "dataless";
8210 } else {
8211 /* received an interrupt, but are not using that context */
8212 err_detail = "out of range";
8213 }
8214 dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n",
8215 err_detail, source);
8216 }
8217
8218 /*
8219 * Reserved range interrupt. Should not be called in normal operation.
8220 */
is_reserved_int(struct hfi1_devdata * dd,unsigned int source)8221 static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source)
8222 {
8223 char name[64];
8224
8225 dd_dev_err(dd, "unexpected %s interrupt\n",
8226 is_reserved_name(name, sizeof(name), source));
8227 }
8228
8229 static const struct is_table is_table[] = {
8230 /*
8231 * start end
8232 * name func interrupt func
8233 */
8234 { IS_GENERAL_ERR_START, IS_GENERAL_ERR_END,
8235 is_misc_err_name, is_misc_err_int },
8236 { IS_SDMAENG_ERR_START, IS_SDMAENG_ERR_END,
8237 is_sdma_eng_err_name, is_sdma_eng_err_int },
8238 { IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END,
8239 is_sendctxt_err_name, is_sendctxt_err_int },
8240 { IS_SDMA_START, IS_SDMA_IDLE_END,
8241 is_sdma_eng_name, is_sdma_eng_int },
8242 { IS_VARIOUS_START, IS_VARIOUS_END,
8243 is_various_name, is_various_int },
8244 { IS_DC_START, IS_DC_END,
8245 is_dc_name, is_dc_int },
8246 { IS_RCVAVAIL_START, IS_RCVAVAIL_END,
8247 is_rcv_avail_name, is_rcv_avail_int },
8248 { IS_RCVURGENT_START, IS_RCVURGENT_END,
8249 is_rcv_urgent_name, is_rcv_urgent_int },
8250 { IS_SENDCREDIT_START, IS_SENDCREDIT_END,
8251 is_send_credit_name, is_send_credit_int},
8252 { IS_RESERVED_START, IS_RESERVED_END,
8253 is_reserved_name, is_reserved_int},
8254 };
8255
8256 /*
8257 * Interrupt source interrupt - called when the given source has an interrupt.
8258 * Source is a bit index into an array of 64-bit integers.
8259 */
is_interrupt(struct hfi1_devdata * dd,unsigned int source)8260 static void is_interrupt(struct hfi1_devdata *dd, unsigned int source)
8261 {
8262 const struct is_table *entry;
8263
8264 /* avoids a double compare by walking the table in-order */
8265 for (entry = &is_table[0]; entry->is_name; entry++) {
8266 if (source <= entry->end) {
8267 trace_hfi1_interrupt(dd, entry, source);
8268 entry->is_int(dd, source - entry->start);
8269 return;
8270 }
8271 }
8272 /* fell off the end */
8273 dd_dev_err(dd, "invalid interrupt source %u\n", source);
8274 }
8275
8276 /**
8277 * general_interrupt - General interrupt handler
8278 * @irq: MSIx IRQ vector
8279 * @data: hfi1 devdata
8280 *
8281 * This is able to correctly handle all non-threaded interrupts. Receive
8282 * context DATA IRQs are threaded and are not supported by this handler.
8283 *
8284 */
general_interrupt(int irq,void * data)8285 irqreturn_t general_interrupt(int irq, void *data)
8286 {
8287 struct hfi1_devdata *dd = data;
8288 u64 regs[CCE_NUM_INT_CSRS];
8289 u32 bit;
8290 int i;
8291 irqreturn_t handled = IRQ_NONE;
8292
8293 this_cpu_inc(*dd->int_counter);
8294
8295 /* phase 1: scan and clear all handled interrupts */
8296 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
8297 if (dd->gi_mask[i] == 0) {
8298 regs[i] = 0; /* used later */
8299 continue;
8300 }
8301 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) &
8302 dd->gi_mask[i];
8303 /* only clear if anything is set */
8304 if (regs[i])
8305 write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]);
8306 }
8307
8308 /* phase 2: call the appropriate handler */
8309 for_each_set_bit(bit, (unsigned long *)®s[0],
8310 CCE_NUM_INT_CSRS * 64) {
8311 is_interrupt(dd, bit);
8312 handled = IRQ_HANDLED;
8313 }
8314
8315 return handled;
8316 }
8317
sdma_interrupt(int irq,void * data)8318 irqreturn_t sdma_interrupt(int irq, void *data)
8319 {
8320 struct sdma_engine *sde = data;
8321 struct hfi1_devdata *dd = sde->dd;
8322 u64 status;
8323
8324 #ifdef CONFIG_SDMA_VERBOSITY
8325 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
8326 slashstrip(__FILE__), __LINE__, __func__);
8327 sdma_dumpstate(sde);
8328 #endif
8329
8330 this_cpu_inc(*dd->int_counter);
8331
8332 /* This read_csr is really bad in the hot path */
8333 status = read_csr(dd,
8334 CCE_INT_STATUS + (8 * (IS_SDMA_START / 64)))
8335 & sde->imask;
8336 if (likely(status)) {
8337 /* clear the interrupt(s) */
8338 write_csr(dd,
8339 CCE_INT_CLEAR + (8 * (IS_SDMA_START / 64)),
8340 status);
8341
8342 /* handle the interrupt(s) */
8343 sdma_engine_interrupt(sde, status);
8344 } else {
8345 dd_dev_info_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n",
8346 sde->this_idx);
8347 }
8348 return IRQ_HANDLED;
8349 }
8350
8351 /*
8352 * Clear the receive interrupt. Use a read of the interrupt clear CSR
8353 * to insure that the write completed. This does NOT guarantee that
8354 * queued DMA writes to memory from the chip are pushed.
8355 */
clear_recv_intr(struct hfi1_ctxtdata * rcd)8356 static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd)
8357 {
8358 struct hfi1_devdata *dd = rcd->dd;
8359 u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg);
8360
8361 write_csr(dd, addr, rcd->imask);
8362 /* force the above write on the chip and get a value back */
8363 (void)read_csr(dd, addr);
8364 }
8365
8366 /* force the receive interrupt */
force_recv_intr(struct hfi1_ctxtdata * rcd)8367 void force_recv_intr(struct hfi1_ctxtdata *rcd)
8368 {
8369 write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask);
8370 }
8371
8372 /*
8373 * Return non-zero if a packet is present.
8374 *
8375 * This routine is called when rechecking for packets after the RcvAvail
8376 * interrupt has been cleared down. First, do a quick check of memory for
8377 * a packet present. If not found, use an expensive CSR read of the context
8378 * tail to determine the actual tail. The CSR read is necessary because there
8379 * is no method to push pending DMAs to memory other than an interrupt and we
8380 * are trying to determine if we need to force an interrupt.
8381 */
check_packet_present(struct hfi1_ctxtdata * rcd)8382 static inline int check_packet_present(struct hfi1_ctxtdata *rcd)
8383 {
8384 u32 tail;
8385
8386 if (hfi1_packet_present(rcd))
8387 return 1;
8388
8389 /* fall back to a CSR read, correct indpendent of DMA_RTAIL */
8390 tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
8391 return hfi1_rcd_head(rcd) != tail;
8392 }
8393
8394 /*
8395 * Common code for receive contexts interrupt handlers.
8396 * Update traces, increment kernel IRQ counter and
8397 * setup ASPM when needed.
8398 */
receive_interrupt_common(struct hfi1_ctxtdata * rcd)8399 static void receive_interrupt_common(struct hfi1_ctxtdata *rcd)
8400 {
8401 struct hfi1_devdata *dd = rcd->dd;
8402
8403 trace_hfi1_receive_interrupt(dd, rcd);
8404 this_cpu_inc(*dd->int_counter);
8405 aspm_ctx_disable(rcd);
8406 }
8407
8408 /*
8409 * __hfi1_rcd_eoi_intr() - Make HW issue receive interrupt
8410 * when there are packets present in the queue. When calling
8411 * with interrupts enabled please use hfi1_rcd_eoi_intr.
8412 *
8413 * @rcd: valid receive context
8414 */
__hfi1_rcd_eoi_intr(struct hfi1_ctxtdata * rcd)8415 static void __hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd)
8416 {
8417 if (!rcd->rcvhdrq)
8418 return;
8419 clear_recv_intr(rcd);
8420 if (check_packet_present(rcd))
8421 force_recv_intr(rcd);
8422 }
8423
8424 /**
8425 * hfi1_rcd_eoi_intr() - End of Interrupt processing action
8426 *
8427 * @rcd: Ptr to hfi1_ctxtdata of receive context
8428 *
8429 * Hold IRQs so we can safely clear the interrupt and
8430 * recheck for a packet that may have arrived after the previous
8431 * check and the interrupt clear. If a packet arrived, force another
8432 * interrupt. This routine can be called at the end of receive packet
8433 * processing in interrupt service routines, interrupt service thread
8434 * and softirqs
8435 */
hfi1_rcd_eoi_intr(struct hfi1_ctxtdata * rcd)8436 static void hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd)
8437 {
8438 unsigned long flags;
8439
8440 local_irq_save(flags);
8441 __hfi1_rcd_eoi_intr(rcd);
8442 local_irq_restore(flags);
8443 }
8444
8445 /**
8446 * hfi1_netdev_rx_napi - napi poll function to move eoi inline
8447 * @napi: pointer to napi object
8448 * @budget: netdev budget
8449 */
hfi1_netdev_rx_napi(struct napi_struct * napi,int budget)8450 int hfi1_netdev_rx_napi(struct napi_struct *napi, int budget)
8451 {
8452 struct hfi1_netdev_rxq *rxq = container_of(napi,
8453 struct hfi1_netdev_rxq, napi);
8454 struct hfi1_ctxtdata *rcd = rxq->rcd;
8455 int work_done = 0;
8456
8457 work_done = rcd->do_interrupt(rcd, budget);
8458
8459 if (work_done < budget) {
8460 napi_complete_done(napi, work_done);
8461 hfi1_rcd_eoi_intr(rcd);
8462 }
8463
8464 return work_done;
8465 }
8466
8467 /* Receive packet napi handler for netdevs VNIC and AIP */
receive_context_interrupt_napi(int irq,void * data)8468 irqreturn_t receive_context_interrupt_napi(int irq, void *data)
8469 {
8470 struct hfi1_ctxtdata *rcd = data;
8471
8472 receive_interrupt_common(rcd);
8473
8474 if (likely(rcd->napi)) {
8475 if (likely(napi_schedule_prep(rcd->napi)))
8476 __napi_schedule_irqoff(rcd->napi);
8477 else
8478 __hfi1_rcd_eoi_intr(rcd);
8479 } else {
8480 WARN_ONCE(1, "Napi IRQ handler without napi set up ctxt=%d\n",
8481 rcd->ctxt);
8482 __hfi1_rcd_eoi_intr(rcd);
8483 }
8484
8485 return IRQ_HANDLED;
8486 }
8487
8488 /*
8489 * Receive packet IRQ handler. This routine expects to be on its own IRQ.
8490 * This routine will try to handle packets immediately (latency), but if
8491 * it finds too many, it will invoke the thread handler (bandwitdh). The
8492 * chip receive interrupt is *not* cleared down until this or the thread (if
8493 * invoked) is finished. The intent is to avoid extra interrupts while we
8494 * are processing packets anyway.
8495 */
receive_context_interrupt(int irq,void * data)8496 irqreturn_t receive_context_interrupt(int irq, void *data)
8497 {
8498 struct hfi1_ctxtdata *rcd = data;
8499 int disposition;
8500
8501 receive_interrupt_common(rcd);
8502
8503 /* receive interrupt remains blocked while processing packets */
8504 disposition = rcd->do_interrupt(rcd, 0);
8505
8506 /*
8507 * Too many packets were seen while processing packets in this
8508 * IRQ handler. Invoke the handler thread. The receive interrupt
8509 * remains blocked.
8510 */
8511 if (disposition == RCV_PKT_LIMIT)
8512 return IRQ_WAKE_THREAD;
8513
8514 __hfi1_rcd_eoi_intr(rcd);
8515 return IRQ_HANDLED;
8516 }
8517
8518 /*
8519 * Receive packet thread handler. This expects to be invoked with the
8520 * receive interrupt still blocked.
8521 */
receive_context_thread(int irq,void * data)8522 irqreturn_t receive_context_thread(int irq, void *data)
8523 {
8524 struct hfi1_ctxtdata *rcd = data;
8525
8526 /* receive interrupt is still blocked from the IRQ handler */
8527 (void)rcd->do_interrupt(rcd, 1);
8528
8529 hfi1_rcd_eoi_intr(rcd);
8530
8531 return IRQ_HANDLED;
8532 }
8533
8534 /* ========================================================================= */
8535
read_physical_state(struct hfi1_devdata * dd)8536 u32 read_physical_state(struct hfi1_devdata *dd)
8537 {
8538 u64 reg;
8539
8540 reg = read_csr(dd, DC_DC8051_STS_CUR_STATE);
8541 return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT)
8542 & DC_DC8051_STS_CUR_STATE_PORT_MASK;
8543 }
8544
read_logical_state(struct hfi1_devdata * dd)8545 u32 read_logical_state(struct hfi1_devdata *dd)
8546 {
8547 u64 reg;
8548
8549 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8550 return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT)
8551 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK;
8552 }
8553
set_logical_state(struct hfi1_devdata * dd,u32 chip_lstate)8554 static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate)
8555 {
8556 u64 reg;
8557
8558 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8559 /* clear current state, set new state */
8560 reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK;
8561 reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT;
8562 write_csr(dd, DCC_CFG_PORT_CONFIG, reg);
8563 }
8564
8565 /*
8566 * Use the 8051 to read a LCB CSR.
8567 */
read_lcb_via_8051(struct hfi1_devdata * dd,u32 addr,u64 * data)8568 static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data)
8569 {
8570 u32 regno;
8571 int ret;
8572
8573 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
8574 if (acquire_lcb_access(dd, 0) == 0) {
8575 *data = read_csr(dd, addr);
8576 release_lcb_access(dd, 0);
8577 return 0;
8578 }
8579 return -EBUSY;
8580 }
8581
8582 /* register is an index of LCB registers: (offset - base) / 8 */
8583 regno = (addr - DC_LCB_CFG_RUN) >> 3;
8584 ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data);
8585 if (ret != HCMD_SUCCESS)
8586 return -EBUSY;
8587 return 0;
8588 }
8589
8590 /*
8591 * Provide a cache for some of the LCB registers in case the LCB is
8592 * unavailable.
8593 * (The LCB is unavailable in certain link states, for example.)
8594 */
8595 struct lcb_datum {
8596 u32 off;
8597 u64 val;
8598 };
8599
8600 static struct lcb_datum lcb_cache[] = {
8601 { DC_LCB_ERR_INFO_RX_REPLAY_CNT, 0},
8602 { DC_LCB_ERR_INFO_SEQ_CRC_CNT, 0 },
8603 { DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT, 0 },
8604 };
8605
update_lcb_cache(struct hfi1_devdata * dd)8606 static void update_lcb_cache(struct hfi1_devdata *dd)
8607 {
8608 int i;
8609 int ret;
8610 u64 val;
8611
8612 for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) {
8613 ret = read_lcb_csr(dd, lcb_cache[i].off, &val);
8614
8615 /* Update if we get good data */
8616 if (likely(ret != -EBUSY))
8617 lcb_cache[i].val = val;
8618 }
8619 }
8620
read_lcb_cache(u32 off,u64 * val)8621 static int read_lcb_cache(u32 off, u64 *val)
8622 {
8623 int i;
8624
8625 for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) {
8626 if (lcb_cache[i].off == off) {
8627 *val = lcb_cache[i].val;
8628 return 0;
8629 }
8630 }
8631
8632 pr_warn("%s bad offset 0x%x\n", __func__, off);
8633 return -1;
8634 }
8635
8636 /*
8637 * Read an LCB CSR. Access may not be in host control, so check.
8638 * Return 0 on success, -EBUSY on failure.
8639 */
read_lcb_csr(struct hfi1_devdata * dd,u32 addr,u64 * data)8640 int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data)
8641 {
8642 struct hfi1_pportdata *ppd = dd->pport;
8643
8644 /* if up, go through the 8051 for the value */
8645 if (ppd->host_link_state & HLS_UP)
8646 return read_lcb_via_8051(dd, addr, data);
8647 /* if going up or down, check the cache, otherwise, no access */
8648 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE)) {
8649 if (read_lcb_cache(addr, data))
8650 return -EBUSY;
8651 return 0;
8652 }
8653
8654 /* otherwise, host has access */
8655 *data = read_csr(dd, addr);
8656 return 0;
8657 }
8658
8659 /*
8660 * Use the 8051 to write a LCB CSR.
8661 */
write_lcb_via_8051(struct hfi1_devdata * dd,u32 addr,u64 data)8662 static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data)
8663 {
8664 u32 regno;
8665 int ret;
8666
8667 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR ||
8668 (dd->dc8051_ver < dc8051_ver(0, 20, 0))) {
8669 if (acquire_lcb_access(dd, 0) == 0) {
8670 write_csr(dd, addr, data);
8671 release_lcb_access(dd, 0);
8672 return 0;
8673 }
8674 return -EBUSY;
8675 }
8676
8677 /* register is an index of LCB registers: (offset - base) / 8 */
8678 regno = (addr - DC_LCB_CFG_RUN) >> 3;
8679 ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data);
8680 if (ret != HCMD_SUCCESS)
8681 return -EBUSY;
8682 return 0;
8683 }
8684
8685 /*
8686 * Write an LCB CSR. Access may not be in host control, so check.
8687 * Return 0 on success, -EBUSY on failure.
8688 */
write_lcb_csr(struct hfi1_devdata * dd,u32 addr,u64 data)8689 int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data)
8690 {
8691 struct hfi1_pportdata *ppd = dd->pport;
8692
8693 /* if up, go through the 8051 for the value */
8694 if (ppd->host_link_state & HLS_UP)
8695 return write_lcb_via_8051(dd, addr, data);
8696 /* if going up or down, no access */
8697 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
8698 return -EBUSY;
8699 /* otherwise, host has access */
8700 write_csr(dd, addr, data);
8701 return 0;
8702 }
8703
8704 /*
8705 * Returns:
8706 * < 0 = Linux error, not able to get access
8707 * > 0 = 8051 command RETURN_CODE
8708 */
do_8051_command(struct hfi1_devdata * dd,u32 type,u64 in_data,u64 * out_data)8709 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
8710 u64 *out_data)
8711 {
8712 u64 reg, completed;
8713 int return_code;
8714 unsigned long timeout;
8715
8716 hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data);
8717
8718 mutex_lock(&dd->dc8051_lock);
8719
8720 /* We can't send any commands to the 8051 if it's in reset */
8721 if (dd->dc_shutdown) {
8722 return_code = -ENODEV;
8723 goto fail;
8724 }
8725
8726 /*
8727 * If an 8051 host command timed out previously, then the 8051 is
8728 * stuck.
8729 *
8730 * On first timeout, attempt to reset and restart the entire DC
8731 * block (including 8051). (Is this too big of a hammer?)
8732 *
8733 * If the 8051 times out a second time, the reset did not bring it
8734 * back to healthy life. In that case, fail any subsequent commands.
8735 */
8736 if (dd->dc8051_timed_out) {
8737 if (dd->dc8051_timed_out > 1) {
8738 dd_dev_err(dd,
8739 "Previous 8051 host command timed out, skipping command %u\n",
8740 type);
8741 return_code = -ENXIO;
8742 goto fail;
8743 }
8744 _dc_shutdown(dd);
8745 _dc_start(dd);
8746 }
8747
8748 /*
8749 * If there is no timeout, then the 8051 command interface is
8750 * waiting for a command.
8751 */
8752
8753 /*
8754 * When writing a LCB CSR, out_data contains the full value to
8755 * to be written, while in_data contains the relative LCB
8756 * address in 7:0. Do the work here, rather than the caller,
8757 * of distrubting the write data to where it needs to go:
8758 *
8759 * Write data
8760 * 39:00 -> in_data[47:8]
8761 * 47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE
8762 * 63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA
8763 */
8764 if (type == HCMD_WRITE_LCB_CSR) {
8765 in_data |= ((*out_data) & 0xffffffffffull) << 8;
8766 /* must preserve COMPLETED - it is tied to hardware */
8767 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_0);
8768 reg &= DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK;
8769 reg |= ((((*out_data) >> 40) & 0xff) <<
8770 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT)
8771 | ((((*out_data) >> 48) & 0xffff) <<
8772 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
8773 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg);
8774 }
8775
8776 /*
8777 * Do two writes: the first to stabilize the type and req_data, the
8778 * second to activate.
8779 */
8780 reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK)
8781 << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT
8782 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK)
8783 << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT;
8784 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8785 reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK;
8786 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8787
8788 /* wait for completion, alternate: interrupt */
8789 timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT);
8790 while (1) {
8791 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1);
8792 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK;
8793 if (completed)
8794 break;
8795 if (time_after(jiffies, timeout)) {
8796 dd->dc8051_timed_out++;
8797 dd_dev_err(dd, "8051 host command %u timeout\n", type);
8798 if (out_data)
8799 *out_data = 0;
8800 return_code = -ETIMEDOUT;
8801 goto fail;
8802 }
8803 udelay(2);
8804 }
8805
8806 if (out_data) {
8807 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT)
8808 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK;
8809 if (type == HCMD_READ_LCB_CSR) {
8810 /* top 16 bits are in a different register */
8811 *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1)
8812 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK)
8813 << (48
8814 - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT);
8815 }
8816 }
8817 return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT)
8818 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK;
8819 dd->dc8051_timed_out = 0;
8820 /*
8821 * Clear command for next user.
8822 */
8823 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0);
8824
8825 fail:
8826 mutex_unlock(&dd->dc8051_lock);
8827 return return_code;
8828 }
8829
set_physical_link_state(struct hfi1_devdata * dd,u64 state)8830 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state)
8831 {
8832 return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL);
8833 }
8834
load_8051_config(struct hfi1_devdata * dd,u8 field_id,u8 lane_id,u32 config_data)8835 int load_8051_config(struct hfi1_devdata *dd, u8 field_id,
8836 u8 lane_id, u32 config_data)
8837 {
8838 u64 data;
8839 int ret;
8840
8841 data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT
8842 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT
8843 | (u64)config_data << LOAD_DATA_DATA_SHIFT;
8844 ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL);
8845 if (ret != HCMD_SUCCESS) {
8846 dd_dev_err(dd,
8847 "load 8051 config: field id %d, lane %d, err %d\n",
8848 (int)field_id, (int)lane_id, ret);
8849 }
8850 return ret;
8851 }
8852
8853 /*
8854 * Read the 8051 firmware "registers". Use the RAM directly. Always
8855 * set the result, even on error.
8856 * Return 0 on success, -errno on failure
8857 */
read_8051_config(struct hfi1_devdata * dd,u8 field_id,u8 lane_id,u32 * result)8858 int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id,
8859 u32 *result)
8860 {
8861 u64 big_data;
8862 u32 addr;
8863 int ret;
8864
8865 /* address start depends on the lane_id */
8866 if (lane_id < 4)
8867 addr = (4 * NUM_GENERAL_FIELDS)
8868 + (lane_id * 4 * NUM_LANE_FIELDS);
8869 else
8870 addr = 0;
8871 addr += field_id * 4;
8872
8873 /* read is in 8-byte chunks, hardware will truncate the address down */
8874 ret = read_8051_data(dd, addr, 8, &big_data);
8875
8876 if (ret == 0) {
8877 /* extract the 4 bytes we want */
8878 if (addr & 0x4)
8879 *result = (u32)(big_data >> 32);
8880 else
8881 *result = (u32)big_data;
8882 } else {
8883 *result = 0;
8884 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n",
8885 __func__, lane_id, field_id);
8886 }
8887
8888 return ret;
8889 }
8890
write_vc_local_phy(struct hfi1_devdata * dd,u8 power_management,u8 continuous)8891 static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management,
8892 u8 continuous)
8893 {
8894 u32 frame;
8895
8896 frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT
8897 | power_management << POWER_MANAGEMENT_SHIFT;
8898 return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY,
8899 GENERAL_CONFIG, frame);
8900 }
8901
write_vc_local_fabric(struct hfi1_devdata * dd,u8 vau,u8 z,u8 vcu,u16 vl15buf,u8 crc_sizes)8902 static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu,
8903 u16 vl15buf, u8 crc_sizes)
8904 {
8905 u32 frame;
8906
8907 frame = (u32)vau << VAU_SHIFT
8908 | (u32)z << Z_SHIFT
8909 | (u32)vcu << VCU_SHIFT
8910 | (u32)vl15buf << VL15BUF_SHIFT
8911 | (u32)crc_sizes << CRC_SIZES_SHIFT;
8912 return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC,
8913 GENERAL_CONFIG, frame);
8914 }
8915
read_vc_local_link_mode(struct hfi1_devdata * dd,u8 * misc_bits,u8 * flag_bits,u16 * link_widths)8916 static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits,
8917 u8 *flag_bits, u16 *link_widths)
8918 {
8919 u32 frame;
8920
8921 read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG,
8922 &frame);
8923 *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK;
8924 *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK;
8925 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8926 }
8927
write_vc_local_link_mode(struct hfi1_devdata * dd,u8 misc_bits,u8 flag_bits,u16 link_widths)8928 static int write_vc_local_link_mode(struct hfi1_devdata *dd,
8929 u8 misc_bits,
8930 u8 flag_bits,
8931 u16 link_widths)
8932 {
8933 u32 frame;
8934
8935 frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT
8936 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT
8937 | (u32)link_widths << LINK_WIDTH_SHIFT;
8938 return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG,
8939 frame);
8940 }
8941
write_local_device_id(struct hfi1_devdata * dd,u16 device_id,u8 device_rev)8942 static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id,
8943 u8 device_rev)
8944 {
8945 u32 frame;
8946
8947 frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT)
8948 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT);
8949 return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame);
8950 }
8951
read_remote_device_id(struct hfi1_devdata * dd,u16 * device_id,u8 * device_rev)8952 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
8953 u8 *device_rev)
8954 {
8955 u32 frame;
8956
8957 read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame);
8958 *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK;
8959 *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT)
8960 & REMOTE_DEVICE_REV_MASK;
8961 }
8962
write_host_interface_version(struct hfi1_devdata * dd,u8 version)8963 int write_host_interface_version(struct hfi1_devdata *dd, u8 version)
8964 {
8965 u32 frame;
8966 u32 mask;
8967
8968 mask = (HOST_INTERFACE_VERSION_MASK << HOST_INTERFACE_VERSION_SHIFT);
8969 read_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, &frame);
8970 /* Clear, then set field */
8971 frame &= ~mask;
8972 frame |= ((u32)version << HOST_INTERFACE_VERSION_SHIFT);
8973 return load_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG,
8974 frame);
8975 }
8976
read_misc_status(struct hfi1_devdata * dd,u8 * ver_major,u8 * ver_minor,u8 * ver_patch)8977 void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor,
8978 u8 *ver_patch)
8979 {
8980 u32 frame;
8981
8982 read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame);
8983 *ver_major = (frame >> STS_FM_VERSION_MAJOR_SHIFT) &
8984 STS_FM_VERSION_MAJOR_MASK;
8985 *ver_minor = (frame >> STS_FM_VERSION_MINOR_SHIFT) &
8986 STS_FM_VERSION_MINOR_MASK;
8987
8988 read_8051_config(dd, VERSION_PATCH, GENERAL_CONFIG, &frame);
8989 *ver_patch = (frame >> STS_FM_VERSION_PATCH_SHIFT) &
8990 STS_FM_VERSION_PATCH_MASK;
8991 }
8992
read_vc_remote_phy(struct hfi1_devdata * dd,u8 * power_management,u8 * continuous)8993 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
8994 u8 *continuous)
8995 {
8996 u32 frame;
8997
8998 read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame);
8999 *power_management = (frame >> POWER_MANAGEMENT_SHIFT)
9000 & POWER_MANAGEMENT_MASK;
9001 *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT)
9002 & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK;
9003 }
9004
read_vc_remote_fabric(struct hfi1_devdata * dd,u8 * vau,u8 * z,u8 * vcu,u16 * vl15buf,u8 * crc_sizes)9005 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
9006 u8 *vcu, u16 *vl15buf, u8 *crc_sizes)
9007 {
9008 u32 frame;
9009
9010 read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame);
9011 *vau = (frame >> VAU_SHIFT) & VAU_MASK;
9012 *z = (frame >> Z_SHIFT) & Z_MASK;
9013 *vcu = (frame >> VCU_SHIFT) & VCU_MASK;
9014 *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK;
9015 *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK;
9016 }
9017
read_vc_remote_link_width(struct hfi1_devdata * dd,u8 * remote_tx_rate,u16 * link_widths)9018 static void read_vc_remote_link_width(struct hfi1_devdata *dd,
9019 u8 *remote_tx_rate,
9020 u16 *link_widths)
9021 {
9022 u32 frame;
9023
9024 read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG,
9025 &frame);
9026 *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT)
9027 & REMOTE_TX_RATE_MASK;
9028 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
9029 }
9030
read_local_lni(struct hfi1_devdata * dd,u8 * enable_lane_rx)9031 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx)
9032 {
9033 u32 frame;
9034
9035 read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame);
9036 *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK;
9037 }
9038
read_last_local_state(struct hfi1_devdata * dd,u32 * lls)9039 static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls)
9040 {
9041 read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls);
9042 }
9043
read_last_remote_state(struct hfi1_devdata * dd,u32 * lrs)9044 static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs)
9045 {
9046 read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs);
9047 }
9048
hfi1_read_link_quality(struct hfi1_devdata * dd,u8 * link_quality)9049 void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality)
9050 {
9051 u32 frame;
9052 int ret;
9053
9054 *link_quality = 0;
9055 if (dd->pport->host_link_state & HLS_UP) {
9056 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG,
9057 &frame);
9058 if (ret == 0)
9059 *link_quality = (frame >> LINK_QUALITY_SHIFT)
9060 & LINK_QUALITY_MASK;
9061 }
9062 }
9063
read_planned_down_reason_code(struct hfi1_devdata * dd,u8 * pdrrc)9064 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc)
9065 {
9066 u32 frame;
9067
9068 read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame);
9069 *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK;
9070 }
9071
read_link_down_reason(struct hfi1_devdata * dd,u8 * ldr)9072 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr)
9073 {
9074 u32 frame;
9075
9076 read_8051_config(dd, LINK_DOWN_REASON, GENERAL_CONFIG, &frame);
9077 *ldr = (frame & 0xff);
9078 }
9079
read_tx_settings(struct hfi1_devdata * dd,u8 * enable_lane_tx,u8 * tx_polarity_inversion,u8 * rx_polarity_inversion,u8 * max_rate)9080 static int read_tx_settings(struct hfi1_devdata *dd,
9081 u8 *enable_lane_tx,
9082 u8 *tx_polarity_inversion,
9083 u8 *rx_polarity_inversion,
9084 u8 *max_rate)
9085 {
9086 u32 frame;
9087 int ret;
9088
9089 ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame);
9090 *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT)
9091 & ENABLE_LANE_TX_MASK;
9092 *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT)
9093 & TX_POLARITY_INVERSION_MASK;
9094 *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT)
9095 & RX_POLARITY_INVERSION_MASK;
9096 *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK;
9097 return ret;
9098 }
9099
write_tx_settings(struct hfi1_devdata * dd,u8 enable_lane_tx,u8 tx_polarity_inversion,u8 rx_polarity_inversion,u8 max_rate)9100 static int write_tx_settings(struct hfi1_devdata *dd,
9101 u8 enable_lane_tx,
9102 u8 tx_polarity_inversion,
9103 u8 rx_polarity_inversion,
9104 u8 max_rate)
9105 {
9106 u32 frame;
9107
9108 /* no need to mask, all variable sizes match field widths */
9109 frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT
9110 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT
9111 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT
9112 | max_rate << MAX_RATE_SHIFT;
9113 return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame);
9114 }
9115
9116 /*
9117 * Read an idle LCB message.
9118 *
9119 * Returns 0 on success, -EINVAL on error
9120 */
read_idle_message(struct hfi1_devdata * dd,u64 type,u64 * data_out)9121 static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out)
9122 {
9123 int ret;
9124
9125 ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG, type, data_out);
9126 if (ret != HCMD_SUCCESS) {
9127 dd_dev_err(dd, "read idle message: type %d, err %d\n",
9128 (u32)type, ret);
9129 return -EINVAL;
9130 }
9131 dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out);
9132 /* return only the payload as we already know the type */
9133 *data_out >>= IDLE_PAYLOAD_SHIFT;
9134 return 0;
9135 }
9136
9137 /*
9138 * Read an idle SMA message. To be done in response to a notification from
9139 * the 8051.
9140 *
9141 * Returns 0 on success, -EINVAL on error
9142 */
read_idle_sma(struct hfi1_devdata * dd,u64 * data)9143 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data)
9144 {
9145 return read_idle_message(dd, (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT,
9146 data);
9147 }
9148
9149 /*
9150 * Send an idle LCB message.
9151 *
9152 * Returns 0 on success, -EINVAL on error
9153 */
send_idle_message(struct hfi1_devdata * dd,u64 data)9154 static int send_idle_message(struct hfi1_devdata *dd, u64 data)
9155 {
9156 int ret;
9157
9158 dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data);
9159 ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL);
9160 if (ret != HCMD_SUCCESS) {
9161 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n",
9162 data, ret);
9163 return -EINVAL;
9164 }
9165 return 0;
9166 }
9167
9168 /*
9169 * Send an idle SMA message.
9170 *
9171 * Returns 0 on success, -EINVAL on error
9172 */
send_idle_sma(struct hfi1_devdata * dd,u64 message)9173 int send_idle_sma(struct hfi1_devdata *dd, u64 message)
9174 {
9175 u64 data;
9176
9177 data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT) |
9178 ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT);
9179 return send_idle_message(dd, data);
9180 }
9181
9182 /*
9183 * Initialize the LCB then do a quick link up. This may or may not be
9184 * in loopback.
9185 *
9186 * return 0 on success, -errno on error
9187 */
do_quick_linkup(struct hfi1_devdata * dd)9188 static int do_quick_linkup(struct hfi1_devdata *dd)
9189 {
9190 int ret;
9191
9192 lcb_shutdown(dd, 0);
9193
9194 if (loopback) {
9195 /* LCB_CFG_LOOPBACK.VAL = 2 */
9196 /* LCB_CFG_LANE_WIDTH.VAL = 0 */
9197 write_csr(dd, DC_LCB_CFG_LOOPBACK,
9198 IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT);
9199 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
9200 }
9201
9202 /* start the LCBs */
9203 /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */
9204 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
9205
9206 /* simulator only loopback steps */
9207 if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
9208 /* LCB_CFG_RUN.EN = 1 */
9209 write_csr(dd, DC_LCB_CFG_RUN,
9210 1ull << DC_LCB_CFG_RUN_EN_SHIFT);
9211
9212 ret = wait_link_transfer_active(dd, 10);
9213 if (ret)
9214 return ret;
9215
9216 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP,
9217 1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT);
9218 }
9219
9220 if (!loopback) {
9221 /*
9222 * When doing quick linkup and not in loopback, both
9223 * sides must be done with LCB set-up before either
9224 * starts the quick linkup. Put a delay here so that
9225 * both sides can be started and have a chance to be
9226 * done with LCB set up before resuming.
9227 */
9228 dd_dev_err(dd,
9229 "Pausing for peer to be finished with LCB set up\n");
9230 msleep(5000);
9231 dd_dev_err(dd, "Continuing with quick linkup\n");
9232 }
9233
9234 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
9235 set_8051_lcb_access(dd);
9236
9237 /*
9238 * State "quick" LinkUp request sets the physical link state to
9239 * LinkUp without a verify capability sequence.
9240 * This state is in simulator v37 and later.
9241 */
9242 ret = set_physical_link_state(dd, PLS_QUICK_LINKUP);
9243 if (ret != HCMD_SUCCESS) {
9244 dd_dev_err(dd,
9245 "%s: set physical link state to quick LinkUp failed with return %d\n",
9246 __func__, ret);
9247
9248 set_host_lcb_access(dd);
9249 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
9250
9251 if (ret >= 0)
9252 ret = -EINVAL;
9253 return ret;
9254 }
9255
9256 return 0; /* success */
9257 }
9258
9259 /*
9260 * Do all special steps to set up loopback.
9261 */
init_loopback(struct hfi1_devdata * dd)9262 static int init_loopback(struct hfi1_devdata *dd)
9263 {
9264 dd_dev_info(dd, "Entering loopback mode\n");
9265
9266 /* all loopbacks should disable self GUID check */
9267 write_csr(dd, DC_DC8051_CFG_MODE,
9268 (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK));
9269
9270 /*
9271 * The simulator has only one loopback option - LCB. Switch
9272 * to that option, which includes quick link up.
9273 *
9274 * Accept all valid loopback values.
9275 */
9276 if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR) &&
9277 (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB ||
9278 loopback == LOOPBACK_CABLE)) {
9279 loopback = LOOPBACK_LCB;
9280 quick_linkup = 1;
9281 return 0;
9282 }
9283
9284 /*
9285 * SerDes loopback init sequence is handled in set_local_link_attributes
9286 */
9287 if (loopback == LOOPBACK_SERDES)
9288 return 0;
9289
9290 /* LCB loopback - handled at poll time */
9291 if (loopback == LOOPBACK_LCB) {
9292 quick_linkup = 1; /* LCB is always quick linkup */
9293
9294 /* not supported in emulation due to emulation RTL changes */
9295 if (dd->icode == ICODE_FPGA_EMULATION) {
9296 dd_dev_err(dd,
9297 "LCB loopback not supported in emulation\n");
9298 return -EINVAL;
9299 }
9300 return 0;
9301 }
9302
9303 /* external cable loopback requires no extra steps */
9304 if (loopback == LOOPBACK_CABLE)
9305 return 0;
9306
9307 dd_dev_err(dd, "Invalid loopback mode %d\n", loopback);
9308 return -EINVAL;
9309 }
9310
9311 /*
9312 * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits
9313 * used in the Verify Capability link width attribute.
9314 */
opa_to_vc_link_widths(u16 opa_widths)9315 static u16 opa_to_vc_link_widths(u16 opa_widths)
9316 {
9317 int i;
9318 u16 result = 0;
9319
9320 static const struct link_bits {
9321 u16 from;
9322 u16 to;
9323 } opa_link_xlate[] = {
9324 { OPA_LINK_WIDTH_1X, 1 << (1 - 1) },
9325 { OPA_LINK_WIDTH_2X, 1 << (2 - 1) },
9326 { OPA_LINK_WIDTH_3X, 1 << (3 - 1) },
9327 { OPA_LINK_WIDTH_4X, 1 << (4 - 1) },
9328 };
9329
9330 for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) {
9331 if (opa_widths & opa_link_xlate[i].from)
9332 result |= opa_link_xlate[i].to;
9333 }
9334 return result;
9335 }
9336
9337 /*
9338 * Set link attributes before moving to polling.
9339 */
set_local_link_attributes(struct hfi1_pportdata * ppd)9340 static int set_local_link_attributes(struct hfi1_pportdata *ppd)
9341 {
9342 struct hfi1_devdata *dd = ppd->dd;
9343 u8 enable_lane_tx;
9344 u8 tx_polarity_inversion;
9345 u8 rx_polarity_inversion;
9346 int ret;
9347 u32 misc_bits = 0;
9348 /* reset our fabric serdes to clear any lingering problems */
9349 fabric_serdes_reset(dd);
9350
9351 /* set the local tx rate - need to read-modify-write */
9352 ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
9353 &rx_polarity_inversion, &ppd->local_tx_rate);
9354 if (ret)
9355 goto set_local_link_attributes_fail;
9356
9357 if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) {
9358 /* set the tx rate to the fastest enabled */
9359 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9360 ppd->local_tx_rate = 1;
9361 else
9362 ppd->local_tx_rate = 0;
9363 } else {
9364 /* set the tx rate to all enabled */
9365 ppd->local_tx_rate = 0;
9366 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9367 ppd->local_tx_rate |= 2;
9368 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G)
9369 ppd->local_tx_rate |= 1;
9370 }
9371
9372 enable_lane_tx = 0xF; /* enable all four lanes */
9373 ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion,
9374 rx_polarity_inversion, ppd->local_tx_rate);
9375 if (ret != HCMD_SUCCESS)
9376 goto set_local_link_attributes_fail;
9377
9378 ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION);
9379 if (ret != HCMD_SUCCESS) {
9380 dd_dev_err(dd,
9381 "Failed to set host interface version, return 0x%x\n",
9382 ret);
9383 goto set_local_link_attributes_fail;
9384 }
9385
9386 /*
9387 * DC supports continuous updates.
9388 */
9389 ret = write_vc_local_phy(dd,
9390 0 /* no power management */,
9391 1 /* continuous updates */);
9392 if (ret != HCMD_SUCCESS)
9393 goto set_local_link_attributes_fail;
9394
9395 /* z=1 in the next call: AU of 0 is not supported by the hardware */
9396 ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init,
9397 ppd->port_crc_mode_enabled);
9398 if (ret != HCMD_SUCCESS)
9399 goto set_local_link_attributes_fail;
9400
9401 /*
9402 * SerDes loopback init sequence requires
9403 * setting bit 0 of MISC_CONFIG_BITS
9404 */
9405 if (loopback == LOOPBACK_SERDES)
9406 misc_bits |= 1 << LOOPBACK_SERDES_CONFIG_BIT_MASK_SHIFT;
9407
9408 /*
9409 * An external device configuration request is used to reset the LCB
9410 * to retry to obtain operational lanes when the first attempt is
9411 * unsuccesful.
9412 */
9413 if (dd->dc8051_ver >= dc8051_ver(1, 25, 0))
9414 misc_bits |= 1 << EXT_CFG_LCB_RESET_SUPPORTED_SHIFT;
9415
9416 ret = write_vc_local_link_mode(dd, misc_bits, 0,
9417 opa_to_vc_link_widths(
9418 ppd->link_width_enabled));
9419 if (ret != HCMD_SUCCESS)
9420 goto set_local_link_attributes_fail;
9421
9422 /* let peer know who we are */
9423 ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev);
9424 if (ret == HCMD_SUCCESS)
9425 return 0;
9426
9427 set_local_link_attributes_fail:
9428 dd_dev_err(dd,
9429 "Failed to set local link attributes, return 0x%x\n",
9430 ret);
9431 return ret;
9432 }
9433
9434 /*
9435 * Call this to start the link.
9436 * Do not do anything if the link is disabled.
9437 * Returns 0 if link is disabled, moved to polling, or the driver is not ready.
9438 */
start_link(struct hfi1_pportdata * ppd)9439 int start_link(struct hfi1_pportdata *ppd)
9440 {
9441 /*
9442 * Tune the SerDes to a ballpark setting for optimal signal and bit
9443 * error rate. Needs to be done before starting the link.
9444 */
9445 tune_serdes(ppd);
9446
9447 if (!ppd->driver_link_ready) {
9448 dd_dev_info(ppd->dd,
9449 "%s: stopping link start because driver is not ready\n",
9450 __func__);
9451 return 0;
9452 }
9453
9454 /*
9455 * FULL_MGMT_P_KEY is cleared from the pkey table, so that the
9456 * pkey table can be configured properly if the HFI unit is connected
9457 * to switch port with MgmtAllowed=NO
9458 */
9459 clear_full_mgmt_pkey(ppd);
9460
9461 return set_link_state(ppd, HLS_DN_POLL);
9462 }
9463
wait_for_qsfp_init(struct hfi1_pportdata * ppd)9464 static void wait_for_qsfp_init(struct hfi1_pportdata *ppd)
9465 {
9466 struct hfi1_devdata *dd = ppd->dd;
9467 u64 mask;
9468 unsigned long timeout;
9469
9470 /*
9471 * Some QSFP cables have a quirk that asserts the IntN line as a side
9472 * effect of power up on plug-in. We ignore this false positive
9473 * interrupt until the module has finished powering up by waiting for
9474 * a minimum timeout of the module inrush initialization time of
9475 * 500 ms (SFF 8679 Table 5-6) to ensure the voltage rails in the
9476 * module have stabilized.
9477 */
9478 msleep(500);
9479
9480 /*
9481 * Check for QSFP interrupt for t_init (SFF 8679 Table 8-1)
9482 */
9483 timeout = jiffies + msecs_to_jiffies(2000);
9484 while (1) {
9485 mask = read_csr(dd, dd->hfi1_id ?
9486 ASIC_QSFP2_IN : ASIC_QSFP1_IN);
9487 if (!(mask & QSFP_HFI0_INT_N))
9488 break;
9489 if (time_after(jiffies, timeout)) {
9490 dd_dev_info(dd, "%s: No IntN detected, reset complete\n",
9491 __func__);
9492 break;
9493 }
9494 udelay(2);
9495 }
9496 }
9497
set_qsfp_int_n(struct hfi1_pportdata * ppd,u8 enable)9498 static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable)
9499 {
9500 struct hfi1_devdata *dd = ppd->dd;
9501 u64 mask;
9502
9503 mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK);
9504 if (enable) {
9505 /*
9506 * Clear the status register to avoid an immediate interrupt
9507 * when we re-enable the IntN pin
9508 */
9509 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9510 QSFP_HFI0_INT_N);
9511 mask |= (u64)QSFP_HFI0_INT_N;
9512 } else {
9513 mask &= ~(u64)QSFP_HFI0_INT_N;
9514 }
9515 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask);
9516 }
9517
reset_qsfp(struct hfi1_pportdata * ppd)9518 int reset_qsfp(struct hfi1_pportdata *ppd)
9519 {
9520 struct hfi1_devdata *dd = ppd->dd;
9521 u64 mask, qsfp_mask;
9522
9523 /* Disable INT_N from triggering QSFP interrupts */
9524 set_qsfp_int_n(ppd, 0);
9525
9526 /* Reset the QSFP */
9527 mask = (u64)QSFP_HFI0_RESET_N;
9528
9529 qsfp_mask = read_csr(dd,
9530 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT);
9531 qsfp_mask &= ~mask;
9532 write_csr(dd,
9533 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
9534
9535 udelay(10);
9536
9537 qsfp_mask |= mask;
9538 write_csr(dd,
9539 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
9540
9541 wait_for_qsfp_init(ppd);
9542
9543 /*
9544 * Allow INT_N to trigger the QSFP interrupt to watch
9545 * for alarms and warnings
9546 */
9547 set_qsfp_int_n(ppd, 1);
9548
9549 /*
9550 * After the reset, AOC transmitters are enabled by default. They need
9551 * to be turned off to complete the QSFP setup before they can be
9552 * enabled again.
9553 */
9554 return set_qsfp_tx(ppd, 0);
9555 }
9556
handle_qsfp_error_conditions(struct hfi1_pportdata * ppd,u8 * qsfp_interrupt_status)9557 static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd,
9558 u8 *qsfp_interrupt_status)
9559 {
9560 struct hfi1_devdata *dd = ppd->dd;
9561
9562 if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) ||
9563 (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING))
9564 dd_dev_err(dd, "%s: QSFP cable temperature too high\n",
9565 __func__);
9566
9567 if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) ||
9568 (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING))
9569 dd_dev_err(dd, "%s: QSFP cable temperature too low\n",
9570 __func__);
9571
9572 /*
9573 * The remaining alarms/warnings don't matter if the link is down.
9574 */
9575 if (ppd->host_link_state & HLS_DOWN)
9576 return 0;
9577
9578 if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) ||
9579 (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING))
9580 dd_dev_err(dd, "%s: QSFP supply voltage too high\n",
9581 __func__);
9582
9583 if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) ||
9584 (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING))
9585 dd_dev_err(dd, "%s: QSFP supply voltage too low\n",
9586 __func__);
9587
9588 /* Byte 2 is vendor specific */
9589
9590 if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) ||
9591 (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING))
9592 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too high\n",
9593 __func__);
9594
9595 if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) ||
9596 (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING))
9597 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too low\n",
9598 __func__);
9599
9600 if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) ||
9601 (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING))
9602 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too high\n",
9603 __func__);
9604
9605 if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) ||
9606 (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING))
9607 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too low\n",
9608 __func__);
9609
9610 if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) ||
9611 (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING))
9612 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too high\n",
9613 __func__);
9614
9615 if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) ||
9616 (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING))
9617 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too low\n",
9618 __func__);
9619
9620 if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) ||
9621 (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING))
9622 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too high\n",
9623 __func__);
9624
9625 if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) ||
9626 (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING))
9627 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too low\n",
9628 __func__);
9629
9630 if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) ||
9631 (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING))
9632 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too high\n",
9633 __func__);
9634
9635 if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) ||
9636 (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING))
9637 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too low\n",
9638 __func__);
9639
9640 if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) ||
9641 (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING))
9642 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too high\n",
9643 __func__);
9644
9645 if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) ||
9646 (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING))
9647 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too low\n",
9648 __func__);
9649
9650 /* Bytes 9-10 and 11-12 are reserved */
9651 /* Bytes 13-15 are vendor specific */
9652
9653 return 0;
9654 }
9655
9656 /* This routine will only be scheduled if the QSFP module present is asserted */
qsfp_event(struct work_struct * work)9657 void qsfp_event(struct work_struct *work)
9658 {
9659 struct qsfp_data *qd;
9660 struct hfi1_pportdata *ppd;
9661 struct hfi1_devdata *dd;
9662
9663 qd = container_of(work, struct qsfp_data, qsfp_work);
9664 ppd = qd->ppd;
9665 dd = ppd->dd;
9666
9667 /* Sanity check */
9668 if (!qsfp_mod_present(ppd))
9669 return;
9670
9671 if (ppd->host_link_state == HLS_DN_DISABLE) {
9672 dd_dev_info(ppd->dd,
9673 "%s: stopping link start because link is disabled\n",
9674 __func__);
9675 return;
9676 }
9677
9678 /*
9679 * Turn DC back on after cable has been re-inserted. Up until
9680 * now, the DC has been in reset to save power.
9681 */
9682 dc_start(dd);
9683
9684 if (qd->cache_refresh_required) {
9685 set_qsfp_int_n(ppd, 0);
9686
9687 wait_for_qsfp_init(ppd);
9688
9689 /*
9690 * Allow INT_N to trigger the QSFP interrupt to watch
9691 * for alarms and warnings
9692 */
9693 set_qsfp_int_n(ppd, 1);
9694
9695 start_link(ppd);
9696 }
9697
9698 if (qd->check_interrupt_flags) {
9699 u8 qsfp_interrupt_status[16] = {0,};
9700
9701 if (one_qsfp_read(ppd, dd->hfi1_id, 6,
9702 &qsfp_interrupt_status[0], 16) != 16) {
9703 dd_dev_info(dd,
9704 "%s: Failed to read status of QSFP module\n",
9705 __func__);
9706 } else {
9707 unsigned long flags;
9708
9709 handle_qsfp_error_conditions(
9710 ppd, qsfp_interrupt_status);
9711 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
9712 ppd->qsfp_info.check_interrupt_flags = 0;
9713 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
9714 flags);
9715 }
9716 }
9717 }
9718
init_qsfp_int(struct hfi1_devdata * dd)9719 void init_qsfp_int(struct hfi1_devdata *dd)
9720 {
9721 struct hfi1_pportdata *ppd = dd->pport;
9722 u64 qsfp_mask;
9723
9724 qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
9725 /* Clear current status to avoid spurious interrupts */
9726 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9727 qsfp_mask);
9728 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK,
9729 qsfp_mask);
9730
9731 set_qsfp_int_n(ppd, 0);
9732
9733 /* Handle active low nature of INT_N and MODPRST_N pins */
9734 if (qsfp_mod_present(ppd))
9735 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N;
9736 write_csr(dd,
9737 dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT,
9738 qsfp_mask);
9739
9740 /* Enable the appropriate QSFP IRQ source */
9741 if (!dd->hfi1_id)
9742 set_intr_bits(dd, QSFP1_INT, QSFP1_INT, true);
9743 else
9744 set_intr_bits(dd, QSFP2_INT, QSFP2_INT, true);
9745 }
9746
9747 /*
9748 * Do a one-time initialize of the LCB block.
9749 */
init_lcb(struct hfi1_devdata * dd)9750 static void init_lcb(struct hfi1_devdata *dd)
9751 {
9752 /* simulator does not correctly handle LCB cclk loopback, skip */
9753 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
9754 return;
9755
9756 /* the DC has been reset earlier in the driver load */
9757
9758 /* set LCB for cclk loopback on the port */
9759 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01);
9760 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00);
9761 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00);
9762 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
9763 write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08);
9764 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02);
9765 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00);
9766 }
9767
9768 /*
9769 * Perform a test read on the QSFP. Return 0 on success, -ERRNO
9770 * on error.
9771 */
test_qsfp_read(struct hfi1_pportdata * ppd)9772 static int test_qsfp_read(struct hfi1_pportdata *ppd)
9773 {
9774 int ret;
9775 u8 status;
9776
9777 /*
9778 * Report success if not a QSFP or, if it is a QSFP, but the cable is
9779 * not present
9780 */
9781 if (ppd->port_type != PORT_TYPE_QSFP || !qsfp_mod_present(ppd))
9782 return 0;
9783
9784 /* read byte 2, the status byte */
9785 ret = one_qsfp_read(ppd, ppd->dd->hfi1_id, 2, &status, 1);
9786 if (ret < 0)
9787 return ret;
9788 if (ret != 1)
9789 return -EIO;
9790
9791 return 0; /* success */
9792 }
9793
9794 /*
9795 * Values for QSFP retry.
9796 *
9797 * Give up after 10s (20 x 500ms). The overall timeout was empirically
9798 * arrived at from experience on a large cluster.
9799 */
9800 #define MAX_QSFP_RETRIES 20
9801 #define QSFP_RETRY_WAIT 500 /* msec */
9802
9803 /*
9804 * Try a QSFP read. If it fails, schedule a retry for later.
9805 * Called on first link activation after driver load.
9806 */
try_start_link(struct hfi1_pportdata * ppd)9807 static void try_start_link(struct hfi1_pportdata *ppd)
9808 {
9809 if (test_qsfp_read(ppd)) {
9810 /* read failed */
9811 if (ppd->qsfp_retry_count >= MAX_QSFP_RETRIES) {
9812 dd_dev_err(ppd->dd, "QSFP not responding, giving up\n");
9813 return;
9814 }
9815 dd_dev_info(ppd->dd,
9816 "QSFP not responding, waiting and retrying %d\n",
9817 (int)ppd->qsfp_retry_count);
9818 ppd->qsfp_retry_count++;
9819 queue_delayed_work(ppd->link_wq, &ppd->start_link_work,
9820 msecs_to_jiffies(QSFP_RETRY_WAIT));
9821 return;
9822 }
9823 ppd->qsfp_retry_count = 0;
9824
9825 start_link(ppd);
9826 }
9827
9828 /*
9829 * Workqueue function to start the link after a delay.
9830 */
handle_start_link(struct work_struct * work)9831 void handle_start_link(struct work_struct *work)
9832 {
9833 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
9834 start_link_work.work);
9835 try_start_link(ppd);
9836 }
9837
bringup_serdes(struct hfi1_pportdata * ppd)9838 int bringup_serdes(struct hfi1_pportdata *ppd)
9839 {
9840 struct hfi1_devdata *dd = ppd->dd;
9841 u64 guid;
9842 int ret;
9843
9844 if (HFI1_CAP_IS_KSET(EXTENDED_PSN))
9845 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK);
9846
9847 guid = ppd->guids[HFI1_PORT_GUID_INDEX];
9848 if (!guid) {
9849 if (dd->base_guid)
9850 guid = dd->base_guid + ppd->port - 1;
9851 ppd->guids[HFI1_PORT_GUID_INDEX] = guid;
9852 }
9853
9854 /* Set linkinit_reason on power up per OPA spec */
9855 ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP;
9856
9857 /* one-time init of the LCB */
9858 init_lcb(dd);
9859
9860 if (loopback) {
9861 ret = init_loopback(dd);
9862 if (ret < 0)
9863 return ret;
9864 }
9865
9866 get_port_type(ppd);
9867 if (ppd->port_type == PORT_TYPE_QSFP) {
9868 set_qsfp_int_n(ppd, 0);
9869 wait_for_qsfp_init(ppd);
9870 set_qsfp_int_n(ppd, 1);
9871 }
9872
9873 try_start_link(ppd);
9874 return 0;
9875 }
9876
hfi1_quiet_serdes(struct hfi1_pportdata * ppd)9877 void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)
9878 {
9879 struct hfi1_devdata *dd = ppd->dd;
9880
9881 /*
9882 * Shut down the link and keep it down. First turn off that the
9883 * driver wants to allow the link to be up (driver_link_ready).
9884 * Then make sure the link is not automatically restarted
9885 * (link_enabled). Cancel any pending restart. And finally
9886 * go offline.
9887 */
9888 ppd->driver_link_ready = 0;
9889 ppd->link_enabled = 0;
9890
9891 ppd->qsfp_retry_count = MAX_QSFP_RETRIES; /* prevent more retries */
9892 flush_delayed_work(&ppd->start_link_work);
9893 cancel_delayed_work_sync(&ppd->start_link_work);
9894
9895 ppd->offline_disabled_reason =
9896 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_REBOOT);
9897 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_REBOOT, 0,
9898 OPA_LINKDOWN_REASON_REBOOT);
9899 set_link_state(ppd, HLS_DN_OFFLINE);
9900
9901 /* disable the port */
9902 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
9903 cancel_work_sync(&ppd->freeze_work);
9904 }
9905
init_cpu_counters(struct hfi1_devdata * dd)9906 static inline int init_cpu_counters(struct hfi1_devdata *dd)
9907 {
9908 struct hfi1_pportdata *ppd;
9909 int i;
9910
9911 ppd = (struct hfi1_pportdata *)(dd + 1);
9912 for (i = 0; i < dd->num_pports; i++, ppd++) {
9913 ppd->ibport_data.rvp.rc_acks = NULL;
9914 ppd->ibport_data.rvp.rc_qacks = NULL;
9915 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64);
9916 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64);
9917 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64);
9918 if (!ppd->ibport_data.rvp.rc_acks ||
9919 !ppd->ibport_data.rvp.rc_delayed_comp ||
9920 !ppd->ibport_data.rvp.rc_qacks)
9921 return -ENOMEM;
9922 }
9923
9924 return 0;
9925 }
9926
9927 /*
9928 * index is the index into the receive array
9929 */
hfi1_put_tid(struct hfi1_devdata * dd,u32 index,u32 type,unsigned long pa,u16 order)9930 void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
9931 u32 type, unsigned long pa, u16 order)
9932 {
9933 u64 reg;
9934
9935 if (!(dd->flags & HFI1_PRESENT))
9936 goto done;
9937
9938 if (type == PT_INVALID || type == PT_INVALID_FLUSH) {
9939 pa = 0;
9940 order = 0;
9941 } else if (type > PT_INVALID) {
9942 dd_dev_err(dd,
9943 "unexpected receive array type %u for index %u, not handled\n",
9944 type, index);
9945 goto done;
9946 }
9947 trace_hfi1_put_tid(dd, index, type, pa, order);
9948
9949 #define RT_ADDR_SHIFT 12 /* 4KB kernel address boundary */
9950 reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK
9951 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT
9952 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK)
9953 << RCV_ARRAY_RT_ADDR_SHIFT;
9954 trace_hfi1_write_rcvarray(dd->rcvarray_wc + (index * 8), reg);
9955 writeq(reg, dd->rcvarray_wc + (index * 8));
9956
9957 if (type == PT_EAGER || type == PT_INVALID_FLUSH || (index & 3) == 3)
9958 /*
9959 * Eager entries are written and flushed
9960 *
9961 * Expected entries are flushed every 4 writes
9962 */
9963 flush_wc();
9964 done:
9965 return;
9966 }
9967
hfi1_clear_tids(struct hfi1_ctxtdata * rcd)9968 void hfi1_clear_tids(struct hfi1_ctxtdata *rcd)
9969 {
9970 struct hfi1_devdata *dd = rcd->dd;
9971 u32 i;
9972
9973 /* this could be optimized */
9974 for (i = rcd->eager_base; i < rcd->eager_base +
9975 rcd->egrbufs.alloced; i++)
9976 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9977
9978 for (i = rcd->expected_base;
9979 i < rcd->expected_base + rcd->expected_count; i++)
9980 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9981 }
9982
9983 static const char * const ib_cfg_name_strings[] = {
9984 "HFI1_IB_CFG_LIDLMC",
9985 "HFI1_IB_CFG_LWID_DG_ENB",
9986 "HFI1_IB_CFG_LWID_ENB",
9987 "HFI1_IB_CFG_LWID",
9988 "HFI1_IB_CFG_SPD_ENB",
9989 "HFI1_IB_CFG_SPD",
9990 "HFI1_IB_CFG_RXPOL_ENB",
9991 "HFI1_IB_CFG_LREV_ENB",
9992 "HFI1_IB_CFG_LINKLATENCY",
9993 "HFI1_IB_CFG_HRTBT",
9994 "HFI1_IB_CFG_OP_VLS",
9995 "HFI1_IB_CFG_VL_HIGH_CAP",
9996 "HFI1_IB_CFG_VL_LOW_CAP",
9997 "HFI1_IB_CFG_OVERRUN_THRESH",
9998 "HFI1_IB_CFG_PHYERR_THRESH",
9999 "HFI1_IB_CFG_LINKDEFAULT",
10000 "HFI1_IB_CFG_PKEYS",
10001 "HFI1_IB_CFG_MTU",
10002 "HFI1_IB_CFG_LSTATE",
10003 "HFI1_IB_CFG_VL_HIGH_LIMIT",
10004 "HFI1_IB_CFG_PMA_TICKS",
10005 "HFI1_IB_CFG_PORT"
10006 };
10007
ib_cfg_name(int which)10008 static const char *ib_cfg_name(int which)
10009 {
10010 if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings))
10011 return "invalid";
10012 return ib_cfg_name_strings[which];
10013 }
10014
hfi1_get_ib_cfg(struct hfi1_pportdata * ppd,int which)10015 int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which)
10016 {
10017 struct hfi1_devdata *dd = ppd->dd;
10018 int val = 0;
10019
10020 switch (which) {
10021 case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */
10022 val = ppd->link_width_enabled;
10023 break;
10024 case HFI1_IB_CFG_LWID: /* currently active Link-width */
10025 val = ppd->link_width_active;
10026 break;
10027 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
10028 val = ppd->link_speed_enabled;
10029 break;
10030 case HFI1_IB_CFG_SPD: /* current Link speed */
10031 val = ppd->link_speed_active;
10032 break;
10033
10034 case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */
10035 case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */
10036 case HFI1_IB_CFG_LINKLATENCY:
10037 goto unimplemented;
10038
10039 case HFI1_IB_CFG_OP_VLS:
10040 val = ppd->actual_vls_operational;
10041 break;
10042 case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */
10043 val = VL_ARB_HIGH_PRIO_TABLE_SIZE;
10044 break;
10045 case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */
10046 val = VL_ARB_LOW_PRIO_TABLE_SIZE;
10047 break;
10048 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
10049 val = ppd->overrun_threshold;
10050 break;
10051 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
10052 val = ppd->phy_error_threshold;
10053 break;
10054 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
10055 val = HLS_DEFAULT;
10056 break;
10057
10058 case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */
10059 case HFI1_IB_CFG_PMA_TICKS:
10060 default:
10061 unimplemented:
10062 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
10063 dd_dev_info(
10064 dd,
10065 "%s: which %s: not implemented\n",
10066 __func__,
10067 ib_cfg_name(which));
10068 break;
10069 }
10070
10071 return val;
10072 }
10073
10074 /*
10075 * The largest MAD packet size.
10076 */
10077 #define MAX_MAD_PACKET 2048
10078
10079 /*
10080 * Return the maximum header bytes that can go on the _wire_
10081 * for this device. This count includes the ICRC which is
10082 * not part of the packet held in memory but it is appended
10083 * by the HW.
10084 * This is dependent on the device's receive header entry size.
10085 * HFI allows this to be set per-receive context, but the
10086 * driver presently enforces a global value.
10087 */
lrh_max_header_bytes(struct hfi1_devdata * dd)10088 u32 lrh_max_header_bytes(struct hfi1_devdata *dd)
10089 {
10090 /*
10091 * The maximum non-payload (MTU) bytes in LRH.PktLen are
10092 * the Receive Header Entry Size minus the PBC (or RHF) size
10093 * plus one DW for the ICRC appended by HW.
10094 *
10095 * dd->rcd[0].rcvhdrqentsize is in DW.
10096 * We use rcd[0] as all context will have the same value. Also,
10097 * the first kernel context would have been allocated by now so
10098 * we are guaranteed a valid value.
10099 */
10100 return (get_hdrqentsize(dd->rcd[0]) - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2;
10101 }
10102
10103 /*
10104 * Set Send Length
10105 * @ppd: per port data
10106 *
10107 * Set the MTU by limiting how many DWs may be sent. The SendLenCheck*
10108 * registers compare against LRH.PktLen, so use the max bytes included
10109 * in the LRH.
10110 *
10111 * This routine changes all VL values except VL15, which it maintains at
10112 * the same value.
10113 */
set_send_length(struct hfi1_pportdata * ppd)10114 static void set_send_length(struct hfi1_pportdata *ppd)
10115 {
10116 struct hfi1_devdata *dd = ppd->dd;
10117 u32 max_hb = lrh_max_header_bytes(dd), dcmtu;
10118 u32 maxvlmtu = dd->vld[15].mtu;
10119 u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2)
10120 & SEND_LEN_CHECK1_LEN_VL15_MASK) <<
10121 SEND_LEN_CHECK1_LEN_VL15_SHIFT;
10122 int i, j;
10123 u32 thres;
10124
10125 for (i = 0; i < ppd->vls_supported; i++) {
10126 if (dd->vld[i].mtu > maxvlmtu)
10127 maxvlmtu = dd->vld[i].mtu;
10128 if (i <= 3)
10129 len1 |= (((dd->vld[i].mtu + max_hb) >> 2)
10130 & SEND_LEN_CHECK0_LEN_VL0_MASK) <<
10131 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT);
10132 else
10133 len2 |= (((dd->vld[i].mtu + max_hb) >> 2)
10134 & SEND_LEN_CHECK1_LEN_VL4_MASK) <<
10135 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT);
10136 }
10137 write_csr(dd, SEND_LEN_CHECK0, len1);
10138 write_csr(dd, SEND_LEN_CHECK1, len2);
10139 /* adjust kernel credit return thresholds based on new MTUs */
10140 /* all kernel receive contexts have the same hdrqentsize */
10141 for (i = 0; i < ppd->vls_supported; i++) {
10142 thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50),
10143 sc_mtu_to_threshold(dd->vld[i].sc,
10144 dd->vld[i].mtu,
10145 get_hdrqentsize(dd->rcd[0])));
10146 for (j = 0; j < INIT_SC_PER_VL; j++)
10147 sc_set_cr_threshold(
10148 pio_select_send_context_vl(dd, j, i),
10149 thres);
10150 }
10151 thres = min(sc_percent_to_threshold(dd->vld[15].sc, 50),
10152 sc_mtu_to_threshold(dd->vld[15].sc,
10153 dd->vld[15].mtu,
10154 dd->rcd[0]->rcvhdrqentsize));
10155 sc_set_cr_threshold(dd->vld[15].sc, thres);
10156
10157 /* Adjust maximum MTU for the port in DC */
10158 dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 :
10159 (ilog2(maxvlmtu >> 8) + 1);
10160 len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG);
10161 len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK;
10162 len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) <<
10163 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT;
10164 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1);
10165 }
10166
set_lidlmc(struct hfi1_pportdata * ppd)10167 static void set_lidlmc(struct hfi1_pportdata *ppd)
10168 {
10169 int i;
10170 u64 sreg = 0;
10171 struct hfi1_devdata *dd = ppd->dd;
10172 u32 mask = ~((1U << ppd->lmc) - 1);
10173 u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
10174 u32 lid;
10175
10176 /*
10177 * Program 0 in CSR if port lid is extended. This prevents
10178 * 9B packets being sent out for large lids.
10179 */
10180 lid = (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ? 0 : ppd->lid;
10181 c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
10182 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
10183 c1 |= ((lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
10184 << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) |
10185 ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
10186 << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
10187 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1);
10188
10189 /*
10190 * Iterate over all the send contexts and set their SLID check
10191 */
10192 sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
10193 SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
10194 (((lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
10195 SEND_CTXT_CHECK_SLID_VALUE_SHIFT);
10196
10197 for (i = 0; i < chip_send_contexts(dd); i++) {
10198 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x",
10199 i, (u32)sreg);
10200 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg);
10201 }
10202
10203 /* Now we have to do the same thing for the sdma engines */
10204 sdma_update_lmc(dd, mask, lid);
10205 }
10206
state_completed_string(u32 completed)10207 static const char *state_completed_string(u32 completed)
10208 {
10209 static const char * const state_completed[] = {
10210 "EstablishComm",
10211 "OptimizeEQ",
10212 "VerifyCap"
10213 };
10214
10215 if (completed < ARRAY_SIZE(state_completed))
10216 return state_completed[completed];
10217
10218 return "unknown";
10219 }
10220
10221 static const char all_lanes_dead_timeout_expired[] =
10222 "All lanes were inactive – was the interconnect media removed?";
10223 static const char tx_out_of_policy[] =
10224 "Passing lanes on local port do not meet the local link width policy";
10225 static const char no_state_complete[] =
10226 "State timeout occurred before link partner completed the state";
10227 static const char * const state_complete_reasons[] = {
10228 [0x00] = "Reason unknown",
10229 [0x01] = "Link was halted by driver, refer to LinkDownReason",
10230 [0x02] = "Link partner reported failure",
10231 [0x10] = "Unable to achieve frame sync on any lane",
10232 [0x11] =
10233 "Unable to find a common bit rate with the link partner",
10234 [0x12] =
10235 "Unable to achieve frame sync on sufficient lanes to meet the local link width policy",
10236 [0x13] =
10237 "Unable to identify preset equalization on sufficient lanes to meet the local link width policy",
10238 [0x14] = no_state_complete,
10239 [0x15] =
10240 "State timeout occurred before link partner identified equalization presets",
10241 [0x16] =
10242 "Link partner completed the EstablishComm state, but the passing lanes do not meet the local link width policy",
10243 [0x17] = tx_out_of_policy,
10244 [0x20] = all_lanes_dead_timeout_expired,
10245 [0x21] =
10246 "Unable to achieve acceptable BER on sufficient lanes to meet the local link width policy",
10247 [0x22] = no_state_complete,
10248 [0x23] =
10249 "Link partner completed the OptimizeEq state, but the passing lanes do not meet the local link width policy",
10250 [0x24] = tx_out_of_policy,
10251 [0x30] = all_lanes_dead_timeout_expired,
10252 [0x31] =
10253 "State timeout occurred waiting for host to process received frames",
10254 [0x32] = no_state_complete,
10255 [0x33] =
10256 "Link partner completed the VerifyCap state, but the passing lanes do not meet the local link width policy",
10257 [0x34] = tx_out_of_policy,
10258 [0x35] = "Negotiated link width is mutually exclusive",
10259 [0x36] =
10260 "Timed out before receiving verifycap frames in VerifyCap.Exchange",
10261 [0x37] = "Unable to resolve secure data exchange",
10262 };
10263
state_complete_reason_code_string(struct hfi1_pportdata * ppd,u32 code)10264 static const char *state_complete_reason_code_string(struct hfi1_pportdata *ppd,
10265 u32 code)
10266 {
10267 const char *str = NULL;
10268
10269 if (code < ARRAY_SIZE(state_complete_reasons))
10270 str = state_complete_reasons[code];
10271
10272 if (str)
10273 return str;
10274 return "Reserved";
10275 }
10276
10277 /* describe the given last state complete frame */
decode_state_complete(struct hfi1_pportdata * ppd,u32 frame,const char * prefix)10278 static void decode_state_complete(struct hfi1_pportdata *ppd, u32 frame,
10279 const char *prefix)
10280 {
10281 struct hfi1_devdata *dd = ppd->dd;
10282 u32 success;
10283 u32 state;
10284 u32 reason;
10285 u32 lanes;
10286
10287 /*
10288 * Decode frame:
10289 * [ 0: 0] - success
10290 * [ 3: 1] - state
10291 * [ 7: 4] - next state timeout
10292 * [15: 8] - reason code
10293 * [31:16] - lanes
10294 */
10295 success = frame & 0x1;
10296 state = (frame >> 1) & 0x7;
10297 reason = (frame >> 8) & 0xff;
10298 lanes = (frame >> 16) & 0xffff;
10299
10300 dd_dev_err(dd, "Last %s LNI state complete frame 0x%08x:\n",
10301 prefix, frame);
10302 dd_dev_err(dd, " last reported state state: %s (0x%x)\n",
10303 state_completed_string(state), state);
10304 dd_dev_err(dd, " state successfully completed: %s\n",
10305 success ? "yes" : "no");
10306 dd_dev_err(dd, " fail reason 0x%x: %s\n",
10307 reason, state_complete_reason_code_string(ppd, reason));
10308 dd_dev_err(dd, " passing lane mask: 0x%x", lanes);
10309 }
10310
10311 /*
10312 * Read the last state complete frames and explain them. This routine
10313 * expects to be called if the link went down during link negotiation
10314 * and initialization (LNI). That is, anywhere between polling and link up.
10315 */
check_lni_states(struct hfi1_pportdata * ppd)10316 static void check_lni_states(struct hfi1_pportdata *ppd)
10317 {
10318 u32 last_local_state;
10319 u32 last_remote_state;
10320
10321 read_last_local_state(ppd->dd, &last_local_state);
10322 read_last_remote_state(ppd->dd, &last_remote_state);
10323
10324 /*
10325 * Don't report anything if there is nothing to report. A value of
10326 * 0 means the link was taken down while polling and there was no
10327 * training in-process.
10328 */
10329 if (last_local_state == 0 && last_remote_state == 0)
10330 return;
10331
10332 decode_state_complete(ppd, last_local_state, "transmitted");
10333 decode_state_complete(ppd, last_remote_state, "received");
10334 }
10335
10336 /* wait for wait_ms for LINK_TRANSFER_ACTIVE to go to 1 */
wait_link_transfer_active(struct hfi1_devdata * dd,int wait_ms)10337 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms)
10338 {
10339 u64 reg;
10340 unsigned long timeout;
10341
10342 /* watch LCB_STS_LINK_TRANSFER_ACTIVE */
10343 timeout = jiffies + msecs_to_jiffies(wait_ms);
10344 while (1) {
10345 reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE);
10346 if (reg)
10347 break;
10348 if (time_after(jiffies, timeout)) {
10349 dd_dev_err(dd,
10350 "timeout waiting for LINK_TRANSFER_ACTIVE\n");
10351 return -ETIMEDOUT;
10352 }
10353 udelay(2);
10354 }
10355 return 0;
10356 }
10357
10358 /* called when the logical link state is not down as it should be */
force_logical_link_state_down(struct hfi1_pportdata * ppd)10359 static void force_logical_link_state_down(struct hfi1_pportdata *ppd)
10360 {
10361 struct hfi1_devdata *dd = ppd->dd;
10362
10363 /*
10364 * Bring link up in LCB loopback
10365 */
10366 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1);
10367 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
10368 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
10369
10370 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
10371 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0);
10372 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
10373 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x2);
10374
10375 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
10376 (void)read_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET);
10377 udelay(3);
10378 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 1);
10379 write_csr(dd, DC_LCB_CFG_RUN, 1ull << DC_LCB_CFG_RUN_EN_SHIFT);
10380
10381 wait_link_transfer_active(dd, 100);
10382
10383 /*
10384 * Bring the link down again.
10385 */
10386 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1);
10387 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 0);
10388 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 0);
10389
10390 dd_dev_info(ppd->dd, "logical state forced to LINK_DOWN\n");
10391 }
10392
10393 /*
10394 * Helper for set_link_state(). Do not call except from that routine.
10395 * Expects ppd->hls_mutex to be held.
10396 *
10397 * @rem_reason value to be sent to the neighbor
10398 *
10399 * LinkDownReasons only set if transition succeeds.
10400 */
goto_offline(struct hfi1_pportdata * ppd,u8 rem_reason)10401 static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason)
10402 {
10403 struct hfi1_devdata *dd = ppd->dd;
10404 u32 previous_state;
10405 int offline_state_ret;
10406 int ret;
10407
10408 update_lcb_cache(dd);
10409
10410 previous_state = ppd->host_link_state;
10411 ppd->host_link_state = HLS_GOING_OFFLINE;
10412
10413 /* start offline transition */
10414 ret = set_physical_link_state(dd, (rem_reason << 8) | PLS_OFFLINE);
10415
10416 if (ret != HCMD_SUCCESS) {
10417 dd_dev_err(dd,
10418 "Failed to transition to Offline link state, return %d\n",
10419 ret);
10420 return -EINVAL;
10421 }
10422 if (ppd->offline_disabled_reason ==
10423 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))
10424 ppd->offline_disabled_reason =
10425 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
10426
10427 offline_state_ret = wait_phys_link_offline_substates(ppd, 10000);
10428 if (offline_state_ret < 0)
10429 return offline_state_ret;
10430
10431 /* Disabling AOC transmitters */
10432 if (ppd->port_type == PORT_TYPE_QSFP &&
10433 ppd->qsfp_info.limiting_active &&
10434 qsfp_mod_present(ppd)) {
10435 int ret;
10436
10437 ret = acquire_chip_resource(dd, qsfp_resource(dd), QSFP_WAIT);
10438 if (ret == 0) {
10439 set_qsfp_tx(ppd, 0);
10440 release_chip_resource(dd, qsfp_resource(dd));
10441 } else {
10442 /* not fatal, but should warn */
10443 dd_dev_err(dd,
10444 "Unable to acquire lock to turn off QSFP TX\n");
10445 }
10446 }
10447
10448 /*
10449 * Wait for the offline.Quiet transition if it hasn't happened yet. It
10450 * can take a while for the link to go down.
10451 */
10452 if (offline_state_ret != PLS_OFFLINE_QUIET) {
10453 ret = wait_physical_linkstate(ppd, PLS_OFFLINE, 30000);
10454 if (ret < 0)
10455 return ret;
10456 }
10457
10458 /*
10459 * Now in charge of LCB - must be after the physical state is
10460 * offline.quiet and before host_link_state is changed.
10461 */
10462 set_host_lcb_access(dd);
10463 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
10464
10465 /* make sure the logical state is also down */
10466 ret = wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000);
10467 if (ret)
10468 force_logical_link_state_down(ppd);
10469
10470 ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */
10471 update_statusp(ppd, IB_PORT_DOWN);
10472
10473 /*
10474 * The LNI has a mandatory wait time after the physical state
10475 * moves to Offline.Quiet. The wait time may be different
10476 * depending on how the link went down. The 8051 firmware
10477 * will observe the needed wait time and only move to ready
10478 * when that is completed. The largest of the quiet timeouts
10479 * is 6s, so wait that long and then at least 0.5s more for
10480 * other transitions, and another 0.5s for a buffer.
10481 */
10482 ret = wait_fm_ready(dd, 7000);
10483 if (ret) {
10484 dd_dev_err(dd,
10485 "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n");
10486 /* state is really offline, so make it so */
10487 ppd->host_link_state = HLS_DN_OFFLINE;
10488 return ret;
10489 }
10490
10491 /*
10492 * The state is now offline and the 8051 is ready to accept host
10493 * requests.
10494 * - change our state
10495 * - notify others if we were previously in a linkup state
10496 */
10497 ppd->host_link_state = HLS_DN_OFFLINE;
10498 if (previous_state & HLS_UP) {
10499 /* went down while link was up */
10500 handle_linkup_change(dd, 0);
10501 } else if (previous_state
10502 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
10503 /* went down while attempting link up */
10504 check_lni_states(ppd);
10505
10506 /* The QSFP doesn't need to be reset on LNI failure */
10507 ppd->qsfp_info.reset_needed = 0;
10508 }
10509
10510 /* the active link width (downgrade) is 0 on link down */
10511 ppd->link_width_active = 0;
10512 ppd->link_width_downgrade_tx_active = 0;
10513 ppd->link_width_downgrade_rx_active = 0;
10514 ppd->current_egress_rate = 0;
10515 return 0;
10516 }
10517
10518 /* return the link state name */
link_state_name(u32 state)10519 static const char *link_state_name(u32 state)
10520 {
10521 const char *name;
10522 int n = ilog2(state);
10523 static const char * const names[] = {
10524 [__HLS_UP_INIT_BP] = "INIT",
10525 [__HLS_UP_ARMED_BP] = "ARMED",
10526 [__HLS_UP_ACTIVE_BP] = "ACTIVE",
10527 [__HLS_DN_DOWNDEF_BP] = "DOWNDEF",
10528 [__HLS_DN_POLL_BP] = "POLL",
10529 [__HLS_DN_DISABLE_BP] = "DISABLE",
10530 [__HLS_DN_OFFLINE_BP] = "OFFLINE",
10531 [__HLS_VERIFY_CAP_BP] = "VERIFY_CAP",
10532 [__HLS_GOING_UP_BP] = "GOING_UP",
10533 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE",
10534 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN"
10535 };
10536
10537 name = n < ARRAY_SIZE(names) ? names[n] : NULL;
10538 return name ? name : "unknown";
10539 }
10540
10541 /* return the link state reason name */
link_state_reason_name(struct hfi1_pportdata * ppd,u32 state)10542 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state)
10543 {
10544 if (state == HLS_UP_INIT) {
10545 switch (ppd->linkinit_reason) {
10546 case OPA_LINKINIT_REASON_LINKUP:
10547 return "(LINKUP)";
10548 case OPA_LINKINIT_REASON_FLAPPING:
10549 return "(FLAPPING)";
10550 case OPA_LINKINIT_OUTSIDE_POLICY:
10551 return "(OUTSIDE_POLICY)";
10552 case OPA_LINKINIT_QUARANTINED:
10553 return "(QUARANTINED)";
10554 case OPA_LINKINIT_INSUFIC_CAPABILITY:
10555 return "(INSUFIC_CAPABILITY)";
10556 default:
10557 break;
10558 }
10559 }
10560 return "";
10561 }
10562
10563 /*
10564 * driver_pstate - convert the driver's notion of a port's
10565 * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*).
10566 * Return -1 (converted to a u32) to indicate error.
10567 */
driver_pstate(struct hfi1_pportdata * ppd)10568 u32 driver_pstate(struct hfi1_pportdata *ppd)
10569 {
10570 switch (ppd->host_link_state) {
10571 case HLS_UP_INIT:
10572 case HLS_UP_ARMED:
10573 case HLS_UP_ACTIVE:
10574 return IB_PORTPHYSSTATE_LINKUP;
10575 case HLS_DN_POLL:
10576 return IB_PORTPHYSSTATE_POLLING;
10577 case HLS_DN_DISABLE:
10578 return IB_PORTPHYSSTATE_DISABLED;
10579 case HLS_DN_OFFLINE:
10580 return OPA_PORTPHYSSTATE_OFFLINE;
10581 case HLS_VERIFY_CAP:
10582 return IB_PORTPHYSSTATE_TRAINING;
10583 case HLS_GOING_UP:
10584 return IB_PORTPHYSSTATE_TRAINING;
10585 case HLS_GOING_OFFLINE:
10586 return OPA_PORTPHYSSTATE_OFFLINE;
10587 case HLS_LINK_COOLDOWN:
10588 return OPA_PORTPHYSSTATE_OFFLINE;
10589 case HLS_DN_DOWNDEF:
10590 default:
10591 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10592 ppd->host_link_state);
10593 return -1;
10594 }
10595 }
10596
10597 /*
10598 * driver_lstate - convert the driver's notion of a port's
10599 * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1
10600 * (converted to a u32) to indicate error.
10601 */
driver_lstate(struct hfi1_pportdata * ppd)10602 u32 driver_lstate(struct hfi1_pportdata *ppd)
10603 {
10604 if (ppd->host_link_state && (ppd->host_link_state & HLS_DOWN))
10605 return IB_PORT_DOWN;
10606
10607 switch (ppd->host_link_state & HLS_UP) {
10608 case HLS_UP_INIT:
10609 return IB_PORT_INIT;
10610 case HLS_UP_ARMED:
10611 return IB_PORT_ARMED;
10612 case HLS_UP_ACTIVE:
10613 return IB_PORT_ACTIVE;
10614 default:
10615 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10616 ppd->host_link_state);
10617 return -1;
10618 }
10619 }
10620
set_link_down_reason(struct hfi1_pportdata * ppd,u8 lcl_reason,u8 neigh_reason,u8 rem_reason)10621 void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
10622 u8 neigh_reason, u8 rem_reason)
10623 {
10624 if (ppd->local_link_down_reason.latest == 0 &&
10625 ppd->neigh_link_down_reason.latest == 0) {
10626 ppd->local_link_down_reason.latest = lcl_reason;
10627 ppd->neigh_link_down_reason.latest = neigh_reason;
10628 ppd->remote_link_down_reason = rem_reason;
10629 }
10630 }
10631
10632 /**
10633 * data_vls_operational() - Verify if data VL BCT credits and MTU
10634 * are both set.
10635 * @ppd: pointer to hfi1_pportdata structure
10636 *
10637 * Return: true - Ok, false -otherwise.
10638 */
data_vls_operational(struct hfi1_pportdata * ppd)10639 static inline bool data_vls_operational(struct hfi1_pportdata *ppd)
10640 {
10641 int i;
10642 u64 reg;
10643
10644 if (!ppd->actual_vls_operational)
10645 return false;
10646
10647 for (i = 0; i < ppd->vls_supported; i++) {
10648 reg = read_csr(ppd->dd, SEND_CM_CREDIT_VL + (8 * i));
10649 if ((reg && !ppd->dd->vld[i].mtu) ||
10650 (!reg && ppd->dd->vld[i].mtu))
10651 return false;
10652 }
10653
10654 return true;
10655 }
10656
10657 /*
10658 * Change the physical and/or logical link state.
10659 *
10660 * Do not call this routine while inside an interrupt. It contains
10661 * calls to routines that can take multiple seconds to finish.
10662 *
10663 * Returns 0 on success, -errno on failure.
10664 */
set_link_state(struct hfi1_pportdata * ppd,u32 state)10665 int set_link_state(struct hfi1_pportdata *ppd, u32 state)
10666 {
10667 struct hfi1_devdata *dd = ppd->dd;
10668 struct ib_event event = {.device = NULL};
10669 int ret1, ret = 0;
10670 int orig_new_state, poll_bounce;
10671
10672 mutex_lock(&ppd->hls_lock);
10673
10674 orig_new_state = state;
10675 if (state == HLS_DN_DOWNDEF)
10676 state = HLS_DEFAULT;
10677
10678 /* interpret poll -> poll as a link bounce */
10679 poll_bounce = ppd->host_link_state == HLS_DN_POLL &&
10680 state == HLS_DN_POLL;
10681
10682 dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__,
10683 link_state_name(ppd->host_link_state),
10684 link_state_name(orig_new_state),
10685 poll_bounce ? "(bounce) " : "",
10686 link_state_reason_name(ppd, state));
10687
10688 /*
10689 * If we're going to a (HLS_*) link state that implies the logical
10690 * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then
10691 * reset is_sm_config_started to 0.
10692 */
10693 if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE)))
10694 ppd->is_sm_config_started = 0;
10695
10696 /*
10697 * Do nothing if the states match. Let a poll to poll link bounce
10698 * go through.
10699 */
10700 if (ppd->host_link_state == state && !poll_bounce)
10701 goto done;
10702
10703 switch (state) {
10704 case HLS_UP_INIT:
10705 if (ppd->host_link_state == HLS_DN_POLL &&
10706 (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) {
10707 /*
10708 * Quick link up jumps from polling to here.
10709 *
10710 * Whether in normal or loopback mode, the
10711 * simulator jumps from polling to link up.
10712 * Accept that here.
10713 */
10714 /* OK */
10715 } else if (ppd->host_link_state != HLS_GOING_UP) {
10716 goto unexpected;
10717 }
10718
10719 /*
10720 * Wait for Link_Up physical state.
10721 * Physical and Logical states should already be
10722 * be transitioned to LinkUp and LinkInit respectively.
10723 */
10724 ret = wait_physical_linkstate(ppd, PLS_LINKUP, 1000);
10725 if (ret) {
10726 dd_dev_err(dd,
10727 "%s: physical state did not change to LINK-UP\n",
10728 __func__);
10729 break;
10730 }
10731
10732 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000);
10733 if (ret) {
10734 dd_dev_err(dd,
10735 "%s: logical state did not change to INIT\n",
10736 __func__);
10737 break;
10738 }
10739
10740 /* clear old transient LINKINIT_REASON code */
10741 if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR)
10742 ppd->linkinit_reason =
10743 OPA_LINKINIT_REASON_LINKUP;
10744
10745 /* enable the port */
10746 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
10747
10748 handle_linkup_change(dd, 1);
10749 pio_kernel_linkup(dd);
10750
10751 /*
10752 * After link up, a new link width will have been set.
10753 * Update the xmit counters with regards to the new
10754 * link width.
10755 */
10756 update_xmit_counters(ppd, ppd->link_width_active);
10757
10758 ppd->host_link_state = HLS_UP_INIT;
10759 update_statusp(ppd, IB_PORT_INIT);
10760 break;
10761 case HLS_UP_ARMED:
10762 if (ppd->host_link_state != HLS_UP_INIT)
10763 goto unexpected;
10764
10765 if (!data_vls_operational(ppd)) {
10766 dd_dev_err(dd,
10767 "%s: Invalid data VL credits or mtu\n",
10768 __func__);
10769 ret = -EINVAL;
10770 break;
10771 }
10772
10773 set_logical_state(dd, LSTATE_ARMED);
10774 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000);
10775 if (ret) {
10776 dd_dev_err(dd,
10777 "%s: logical state did not change to ARMED\n",
10778 __func__);
10779 break;
10780 }
10781 ppd->host_link_state = HLS_UP_ARMED;
10782 update_statusp(ppd, IB_PORT_ARMED);
10783 /*
10784 * The simulator does not currently implement SMA messages,
10785 * so neighbor_normal is not set. Set it here when we first
10786 * move to Armed.
10787 */
10788 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
10789 ppd->neighbor_normal = 1;
10790 break;
10791 case HLS_UP_ACTIVE:
10792 if (ppd->host_link_state != HLS_UP_ARMED)
10793 goto unexpected;
10794
10795 set_logical_state(dd, LSTATE_ACTIVE);
10796 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000);
10797 if (ret) {
10798 dd_dev_err(dd,
10799 "%s: logical state did not change to ACTIVE\n",
10800 __func__);
10801 } else {
10802 /* tell all engines to go running */
10803 sdma_all_running(dd);
10804 ppd->host_link_state = HLS_UP_ACTIVE;
10805 update_statusp(ppd, IB_PORT_ACTIVE);
10806
10807 /* Signal the IB layer that the port has went active */
10808 event.device = &dd->verbs_dev.rdi.ibdev;
10809 event.element.port_num = ppd->port;
10810 event.event = IB_EVENT_PORT_ACTIVE;
10811 }
10812 break;
10813 case HLS_DN_POLL:
10814 if ((ppd->host_link_state == HLS_DN_DISABLE ||
10815 ppd->host_link_state == HLS_DN_OFFLINE) &&
10816 dd->dc_shutdown)
10817 dc_start(dd);
10818 /* Hand LED control to the DC */
10819 write_csr(dd, DCC_CFG_LED_CNTRL, 0);
10820
10821 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10822 u8 tmp = ppd->link_enabled;
10823
10824 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10825 if (ret) {
10826 ppd->link_enabled = tmp;
10827 break;
10828 }
10829 ppd->remote_link_down_reason = 0;
10830
10831 if (ppd->driver_link_ready)
10832 ppd->link_enabled = 1;
10833 }
10834
10835 set_all_slowpath(ppd->dd);
10836 ret = set_local_link_attributes(ppd);
10837 if (ret)
10838 break;
10839
10840 ppd->port_error_action = 0;
10841
10842 if (quick_linkup) {
10843 /* quick linkup does not go into polling */
10844 ret = do_quick_linkup(dd);
10845 } else {
10846 ret1 = set_physical_link_state(dd, PLS_POLLING);
10847 if (!ret1)
10848 ret1 = wait_phys_link_out_of_offline(ppd,
10849 3000);
10850 if (ret1 != HCMD_SUCCESS) {
10851 dd_dev_err(dd,
10852 "Failed to transition to Polling link state, return 0x%x\n",
10853 ret1);
10854 ret = -EINVAL;
10855 }
10856 }
10857
10858 /*
10859 * Change the host link state after requesting DC8051 to
10860 * change its physical state so that we can ignore any
10861 * interrupt with stale LNI(XX) error, which will not be
10862 * cleared until DC8051 transitions to Polling state.
10863 */
10864 ppd->host_link_state = HLS_DN_POLL;
10865 ppd->offline_disabled_reason =
10866 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE);
10867 /*
10868 * If an error occurred above, go back to offline. The
10869 * caller may reschedule another attempt.
10870 */
10871 if (ret)
10872 goto_offline(ppd, 0);
10873 else
10874 log_physical_state(ppd, PLS_POLLING);
10875 break;
10876 case HLS_DN_DISABLE:
10877 /* link is disabled */
10878 ppd->link_enabled = 0;
10879
10880 /* allow any state to transition to disabled */
10881
10882 /* must transition to offline first */
10883 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10884 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10885 if (ret)
10886 break;
10887 ppd->remote_link_down_reason = 0;
10888 }
10889
10890 if (!dd->dc_shutdown) {
10891 ret1 = set_physical_link_state(dd, PLS_DISABLED);
10892 if (ret1 != HCMD_SUCCESS) {
10893 dd_dev_err(dd,
10894 "Failed to transition to Disabled link state, return 0x%x\n",
10895 ret1);
10896 ret = -EINVAL;
10897 break;
10898 }
10899 ret = wait_physical_linkstate(ppd, PLS_DISABLED, 10000);
10900 if (ret) {
10901 dd_dev_err(dd,
10902 "%s: physical state did not change to DISABLED\n",
10903 __func__);
10904 break;
10905 }
10906 dc_shutdown(dd);
10907 }
10908 ppd->host_link_state = HLS_DN_DISABLE;
10909 break;
10910 case HLS_DN_OFFLINE:
10911 if (ppd->host_link_state == HLS_DN_DISABLE)
10912 dc_start(dd);
10913
10914 /* allow any state to transition to offline */
10915 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10916 if (!ret)
10917 ppd->remote_link_down_reason = 0;
10918 break;
10919 case HLS_VERIFY_CAP:
10920 if (ppd->host_link_state != HLS_DN_POLL)
10921 goto unexpected;
10922 ppd->host_link_state = HLS_VERIFY_CAP;
10923 log_physical_state(ppd, PLS_CONFIGPHY_VERIFYCAP);
10924 break;
10925 case HLS_GOING_UP:
10926 if (ppd->host_link_state != HLS_VERIFY_CAP)
10927 goto unexpected;
10928
10929 ret1 = set_physical_link_state(dd, PLS_LINKUP);
10930 if (ret1 != HCMD_SUCCESS) {
10931 dd_dev_err(dd,
10932 "Failed to transition to link up state, return 0x%x\n",
10933 ret1);
10934 ret = -EINVAL;
10935 break;
10936 }
10937 ppd->host_link_state = HLS_GOING_UP;
10938 break;
10939
10940 case HLS_GOING_OFFLINE: /* transient within goto_offline() */
10941 case HLS_LINK_COOLDOWN: /* transient within goto_offline() */
10942 default:
10943 dd_dev_info(dd, "%s: state 0x%x: not supported\n",
10944 __func__, state);
10945 ret = -EINVAL;
10946 break;
10947 }
10948
10949 goto done;
10950
10951 unexpected:
10952 dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n",
10953 __func__, link_state_name(ppd->host_link_state),
10954 link_state_name(state));
10955 ret = -EINVAL;
10956
10957 done:
10958 mutex_unlock(&ppd->hls_lock);
10959
10960 if (event.device)
10961 ib_dispatch_event(&event);
10962
10963 return ret;
10964 }
10965
hfi1_set_ib_cfg(struct hfi1_pportdata * ppd,int which,u32 val)10966 int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val)
10967 {
10968 u64 reg;
10969 int ret = 0;
10970
10971 switch (which) {
10972 case HFI1_IB_CFG_LIDLMC:
10973 set_lidlmc(ppd);
10974 break;
10975 case HFI1_IB_CFG_VL_HIGH_LIMIT:
10976 /*
10977 * The VL Arbitrator high limit is sent in units of 4k
10978 * bytes, while HFI stores it in units of 64 bytes.
10979 */
10980 val *= 4096 / 64;
10981 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK)
10982 << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT;
10983 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg);
10984 break;
10985 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
10986 /* HFI only supports POLL as the default link down state */
10987 if (val != HLS_DN_POLL)
10988 ret = -EINVAL;
10989 break;
10990 case HFI1_IB_CFG_OP_VLS:
10991 if (ppd->vls_operational != val) {
10992 ppd->vls_operational = val;
10993 if (!ppd->port)
10994 ret = -EINVAL;
10995 }
10996 break;
10997 /*
10998 * For link width, link width downgrade, and speed enable, always AND
10999 * the setting with what is actually supported. This has two benefits.
11000 * First, enabled can't have unsupported values, no matter what the
11001 * SM or FM might want. Second, the ALL_SUPPORTED wildcards that mean
11002 * "fill in with your supported value" have all the bits in the
11003 * field set, so simply ANDing with supported has the desired result.
11004 */
11005 case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */
11006 ppd->link_width_enabled = val & ppd->link_width_supported;
11007 break;
11008 case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */
11009 ppd->link_width_downgrade_enabled =
11010 val & ppd->link_width_downgrade_supported;
11011 break;
11012 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
11013 ppd->link_speed_enabled = val & ppd->link_speed_supported;
11014 break;
11015 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
11016 /*
11017 * HFI does not follow IB specs, save this value
11018 * so we can report it, if asked.
11019 */
11020 ppd->overrun_threshold = val;
11021 break;
11022 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
11023 /*
11024 * HFI does not follow IB specs, save this value
11025 * so we can report it, if asked.
11026 */
11027 ppd->phy_error_threshold = val;
11028 break;
11029
11030 case HFI1_IB_CFG_MTU:
11031 set_send_length(ppd);
11032 break;
11033
11034 case HFI1_IB_CFG_PKEYS:
11035 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
11036 set_partition_keys(ppd);
11037 break;
11038
11039 default:
11040 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
11041 dd_dev_info(ppd->dd,
11042 "%s: which %s, val 0x%x: not implemented\n",
11043 __func__, ib_cfg_name(which), val);
11044 break;
11045 }
11046 return ret;
11047 }
11048
11049 /* begin functions related to vl arbitration table caching */
init_vl_arb_caches(struct hfi1_pportdata * ppd)11050 static void init_vl_arb_caches(struct hfi1_pportdata *ppd)
11051 {
11052 int i;
11053
11054 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
11055 VL_ARB_LOW_PRIO_TABLE_SIZE);
11056 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
11057 VL_ARB_HIGH_PRIO_TABLE_SIZE);
11058
11059 /*
11060 * Note that we always return values directly from the
11061 * 'vl_arb_cache' (and do no CSR reads) in response to a
11062 * 'Get(VLArbTable)'. This is obviously correct after a
11063 * 'Set(VLArbTable)', since the cache will then be up to
11064 * date. But it's also correct prior to any 'Set(VLArbTable)'
11065 * since then both the cache, and the relevant h/w registers
11066 * will be zeroed.
11067 */
11068
11069 for (i = 0; i < MAX_PRIO_TABLE; i++)
11070 spin_lock_init(&ppd->vl_arb_cache[i].lock);
11071 }
11072
11073 /*
11074 * vl_arb_lock_cache
11075 *
11076 * All other vl_arb_* functions should be called only after locking
11077 * the cache.
11078 */
11079 static inline struct vl_arb_cache *
vl_arb_lock_cache(struct hfi1_pportdata * ppd,int idx)11080 vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx)
11081 {
11082 if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE)
11083 return NULL;
11084 spin_lock(&ppd->vl_arb_cache[idx].lock);
11085 return &ppd->vl_arb_cache[idx];
11086 }
11087
vl_arb_unlock_cache(struct hfi1_pportdata * ppd,int idx)11088 static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx)
11089 {
11090 spin_unlock(&ppd->vl_arb_cache[idx].lock);
11091 }
11092
vl_arb_get_cache(struct vl_arb_cache * cache,struct ib_vl_weight_elem * vl)11093 static void vl_arb_get_cache(struct vl_arb_cache *cache,
11094 struct ib_vl_weight_elem *vl)
11095 {
11096 memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl));
11097 }
11098
vl_arb_set_cache(struct vl_arb_cache * cache,struct ib_vl_weight_elem * vl)11099 static void vl_arb_set_cache(struct vl_arb_cache *cache,
11100 struct ib_vl_weight_elem *vl)
11101 {
11102 memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
11103 }
11104
vl_arb_match_cache(struct vl_arb_cache * cache,struct ib_vl_weight_elem * vl)11105 static int vl_arb_match_cache(struct vl_arb_cache *cache,
11106 struct ib_vl_weight_elem *vl)
11107 {
11108 return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
11109 }
11110
11111 /* end functions related to vl arbitration table caching */
11112
set_vl_weights(struct hfi1_pportdata * ppd,u32 target,u32 size,struct ib_vl_weight_elem * vl)11113 static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target,
11114 u32 size, struct ib_vl_weight_elem *vl)
11115 {
11116 struct hfi1_devdata *dd = ppd->dd;
11117 u64 reg;
11118 unsigned int i, is_up = 0;
11119 int drain, ret = 0;
11120
11121 mutex_lock(&ppd->hls_lock);
11122
11123 if (ppd->host_link_state & HLS_UP)
11124 is_up = 1;
11125
11126 drain = !is_ax(dd) && is_up;
11127
11128 if (drain)
11129 /*
11130 * Before adjusting VL arbitration weights, empty per-VL
11131 * FIFOs, otherwise a packet whose VL weight is being
11132 * set to 0 could get stuck in a FIFO with no chance to
11133 * egress.
11134 */
11135 ret = stop_drain_data_vls(dd);
11136
11137 if (ret) {
11138 dd_dev_err(
11139 dd,
11140 "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n",
11141 __func__);
11142 goto err;
11143 }
11144
11145 for (i = 0; i < size; i++, vl++) {
11146 /*
11147 * NOTE: The low priority shift and mask are used here, but
11148 * they are the same for both the low and high registers.
11149 */
11150 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK)
11151 << SEND_LOW_PRIORITY_LIST_VL_SHIFT)
11152 | (((u64)vl->weight
11153 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK)
11154 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT);
11155 write_csr(dd, target + (i * 8), reg);
11156 }
11157 pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE);
11158
11159 if (drain)
11160 open_fill_data_vls(dd); /* reopen all VLs */
11161
11162 err:
11163 mutex_unlock(&ppd->hls_lock);
11164
11165 return ret;
11166 }
11167
11168 /*
11169 * Read one credit merge VL register.
11170 */
read_one_cm_vl(struct hfi1_devdata * dd,u32 csr,struct vl_limit * vll)11171 static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr,
11172 struct vl_limit *vll)
11173 {
11174 u64 reg = read_csr(dd, csr);
11175
11176 vll->dedicated = cpu_to_be16(
11177 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT)
11178 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK);
11179 vll->shared = cpu_to_be16(
11180 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT)
11181 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK);
11182 }
11183
11184 /*
11185 * Read the current credit merge limits.
11186 */
get_buffer_control(struct hfi1_devdata * dd,struct buffer_control * bc,u16 * overall_limit)11187 static int get_buffer_control(struct hfi1_devdata *dd,
11188 struct buffer_control *bc, u16 *overall_limit)
11189 {
11190 u64 reg;
11191 int i;
11192
11193 /* not all entries are filled in */
11194 memset(bc, 0, sizeof(*bc));
11195
11196 /* OPA and HFI have a 1-1 mapping */
11197 for (i = 0; i < TXE_NUM_DATA_VL; i++)
11198 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8 * i), &bc->vl[i]);
11199
11200 /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */
11201 read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]);
11202
11203 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
11204 bc->overall_shared_limit = cpu_to_be16(
11205 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT)
11206 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK);
11207 if (overall_limit)
11208 *overall_limit = (reg
11209 >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT)
11210 & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK;
11211 return sizeof(struct buffer_control);
11212 }
11213
get_sc2vlnt(struct hfi1_devdata * dd,struct sc2vlnt * dp)11214 static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
11215 {
11216 u64 reg;
11217 int i;
11218
11219 /* each register contains 16 SC->VLnt mappings, 4 bits each */
11220 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0);
11221 for (i = 0; i < sizeof(u64); i++) {
11222 u8 byte = *(((u8 *)®) + i);
11223
11224 dp->vlnt[2 * i] = byte & 0xf;
11225 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4;
11226 }
11227
11228 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16);
11229 for (i = 0; i < sizeof(u64); i++) {
11230 u8 byte = *(((u8 *)®) + i);
11231
11232 dp->vlnt[16 + (2 * i)] = byte & 0xf;
11233 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4;
11234 }
11235 return sizeof(struct sc2vlnt);
11236 }
11237
get_vlarb_preempt(struct hfi1_devdata * dd,u32 nelems,struct ib_vl_weight_elem * vl)11238 static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems,
11239 struct ib_vl_weight_elem *vl)
11240 {
11241 unsigned int i;
11242
11243 for (i = 0; i < nelems; i++, vl++) {
11244 vl->vl = 0xf;
11245 vl->weight = 0;
11246 }
11247 }
11248
set_sc2vlnt(struct hfi1_devdata * dd,struct sc2vlnt * dp)11249 static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
11250 {
11251 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0,
11252 DC_SC_VL_VAL(15_0,
11253 0, dp->vlnt[0] & 0xf,
11254 1, dp->vlnt[1] & 0xf,
11255 2, dp->vlnt[2] & 0xf,
11256 3, dp->vlnt[3] & 0xf,
11257 4, dp->vlnt[4] & 0xf,
11258 5, dp->vlnt[5] & 0xf,
11259 6, dp->vlnt[6] & 0xf,
11260 7, dp->vlnt[7] & 0xf,
11261 8, dp->vlnt[8] & 0xf,
11262 9, dp->vlnt[9] & 0xf,
11263 10, dp->vlnt[10] & 0xf,
11264 11, dp->vlnt[11] & 0xf,
11265 12, dp->vlnt[12] & 0xf,
11266 13, dp->vlnt[13] & 0xf,
11267 14, dp->vlnt[14] & 0xf,
11268 15, dp->vlnt[15] & 0xf));
11269 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16,
11270 DC_SC_VL_VAL(31_16,
11271 16, dp->vlnt[16] & 0xf,
11272 17, dp->vlnt[17] & 0xf,
11273 18, dp->vlnt[18] & 0xf,
11274 19, dp->vlnt[19] & 0xf,
11275 20, dp->vlnt[20] & 0xf,
11276 21, dp->vlnt[21] & 0xf,
11277 22, dp->vlnt[22] & 0xf,
11278 23, dp->vlnt[23] & 0xf,
11279 24, dp->vlnt[24] & 0xf,
11280 25, dp->vlnt[25] & 0xf,
11281 26, dp->vlnt[26] & 0xf,
11282 27, dp->vlnt[27] & 0xf,
11283 28, dp->vlnt[28] & 0xf,
11284 29, dp->vlnt[29] & 0xf,
11285 30, dp->vlnt[30] & 0xf,
11286 31, dp->vlnt[31] & 0xf));
11287 }
11288
nonzero_msg(struct hfi1_devdata * dd,int idx,const char * what,u16 limit)11289 static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what,
11290 u16 limit)
11291 {
11292 if (limit != 0)
11293 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n",
11294 what, (int)limit, idx);
11295 }
11296
11297 /* change only the shared limit portion of SendCmGLobalCredit */
set_global_shared(struct hfi1_devdata * dd,u16 limit)11298 static void set_global_shared(struct hfi1_devdata *dd, u16 limit)
11299 {
11300 u64 reg;
11301
11302 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
11303 reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK;
11304 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT;
11305 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
11306 }
11307
11308 /* change only the total credit limit portion of SendCmGLobalCredit */
set_global_limit(struct hfi1_devdata * dd,u16 limit)11309 static void set_global_limit(struct hfi1_devdata *dd, u16 limit)
11310 {
11311 u64 reg;
11312
11313 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
11314 reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK;
11315 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
11316 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
11317 }
11318
11319 /* set the given per-VL shared limit */
set_vl_shared(struct hfi1_devdata * dd,int vl,u16 limit)11320 static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit)
11321 {
11322 u64 reg;
11323 u32 addr;
11324
11325 if (vl < TXE_NUM_DATA_VL)
11326 addr = SEND_CM_CREDIT_VL + (8 * vl);
11327 else
11328 addr = SEND_CM_CREDIT_VL15;
11329
11330 reg = read_csr(dd, addr);
11331 reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK;
11332 reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT;
11333 write_csr(dd, addr, reg);
11334 }
11335
11336 /* set the given per-VL dedicated limit */
set_vl_dedicated(struct hfi1_devdata * dd,int vl,u16 limit)11337 static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit)
11338 {
11339 u64 reg;
11340 u32 addr;
11341
11342 if (vl < TXE_NUM_DATA_VL)
11343 addr = SEND_CM_CREDIT_VL + (8 * vl);
11344 else
11345 addr = SEND_CM_CREDIT_VL15;
11346
11347 reg = read_csr(dd, addr);
11348 reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK;
11349 reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT;
11350 write_csr(dd, addr, reg);
11351 }
11352
11353 /* spin until the given per-VL status mask bits clear */
wait_for_vl_status_clear(struct hfi1_devdata * dd,u64 mask,const char * which)11354 static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask,
11355 const char *which)
11356 {
11357 unsigned long timeout;
11358 u64 reg;
11359
11360 timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT);
11361 while (1) {
11362 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask;
11363
11364 if (reg == 0)
11365 return; /* success */
11366 if (time_after(jiffies, timeout))
11367 break; /* timed out */
11368 udelay(1);
11369 }
11370
11371 dd_dev_err(dd,
11372 "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n",
11373 which, VL_STATUS_CLEAR_TIMEOUT, mask, reg);
11374 /*
11375 * If this occurs, it is likely there was a credit loss on the link.
11376 * The only recovery from that is a link bounce.
11377 */
11378 dd_dev_err(dd,
11379 "Continuing anyway. A credit loss may occur. Suggest a link bounce\n");
11380 }
11381
11382 /*
11383 * The number of credits on the VLs may be changed while everything
11384 * is "live", but the following algorithm must be followed due to
11385 * how the hardware is actually implemented. In particular,
11386 * Return_Credit_Status[] is the only correct status check.
11387 *
11388 * if (reducing Global_Shared_Credit_Limit or any shared limit changing)
11389 * set Global_Shared_Credit_Limit = 0
11390 * use_all_vl = 1
11391 * mask0 = all VLs that are changing either dedicated or shared limits
11392 * set Shared_Limit[mask0] = 0
11393 * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0
11394 * if (changing any dedicated limit)
11395 * mask1 = all VLs that are lowering dedicated limits
11396 * lower Dedicated_Limit[mask1]
11397 * spin until Return_Credit_Status[mask1] == 0
11398 * raise Dedicated_Limits
11399 * raise Shared_Limits
11400 * raise Global_Shared_Credit_Limit
11401 *
11402 * lower = if the new limit is lower, set the limit to the new value
11403 * raise = if the new limit is higher than the current value (may be changed
11404 * earlier in the algorithm), set the new limit to the new value
11405 */
set_buffer_control(struct hfi1_pportdata * ppd,struct buffer_control * new_bc)11406 int set_buffer_control(struct hfi1_pportdata *ppd,
11407 struct buffer_control *new_bc)
11408 {
11409 struct hfi1_devdata *dd = ppd->dd;
11410 u64 changing_mask, ld_mask, stat_mask;
11411 int change_count;
11412 int i, use_all_mask;
11413 int this_shared_changing;
11414 int vl_count = 0, ret;
11415 /*
11416 * A0: add the variable any_shared_limit_changing below and in the
11417 * algorithm above. If removing A0 support, it can be removed.
11418 */
11419 int any_shared_limit_changing;
11420 struct buffer_control cur_bc;
11421 u8 changing[OPA_MAX_VLS];
11422 u8 lowering_dedicated[OPA_MAX_VLS];
11423 u16 cur_total;
11424 u32 new_total = 0;
11425 const u64 all_mask =
11426 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK
11427 | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK
11428 | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK
11429 | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK
11430 | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK
11431 | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK
11432 | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK
11433 | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK
11434 | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK;
11435
11436 #define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15)
11437 #define NUM_USABLE_VLS 16 /* look at VL15 and less */
11438
11439 /* find the new total credits, do sanity check on unused VLs */
11440 for (i = 0; i < OPA_MAX_VLS; i++) {
11441 if (valid_vl(i)) {
11442 new_total += be16_to_cpu(new_bc->vl[i].dedicated);
11443 continue;
11444 }
11445 nonzero_msg(dd, i, "dedicated",
11446 be16_to_cpu(new_bc->vl[i].dedicated));
11447 nonzero_msg(dd, i, "shared",
11448 be16_to_cpu(new_bc->vl[i].shared));
11449 new_bc->vl[i].dedicated = 0;
11450 new_bc->vl[i].shared = 0;
11451 }
11452 new_total += be16_to_cpu(new_bc->overall_shared_limit);
11453
11454 /* fetch the current values */
11455 get_buffer_control(dd, &cur_bc, &cur_total);
11456
11457 /*
11458 * Create the masks we will use.
11459 */
11460 memset(changing, 0, sizeof(changing));
11461 memset(lowering_dedicated, 0, sizeof(lowering_dedicated));
11462 /*
11463 * NOTE: Assumes that the individual VL bits are adjacent and in
11464 * increasing order
11465 */
11466 stat_mask =
11467 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK;
11468 changing_mask = 0;
11469 ld_mask = 0;
11470 change_count = 0;
11471 any_shared_limit_changing = 0;
11472 for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) {
11473 if (!valid_vl(i))
11474 continue;
11475 this_shared_changing = new_bc->vl[i].shared
11476 != cur_bc.vl[i].shared;
11477 if (this_shared_changing)
11478 any_shared_limit_changing = 1;
11479 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated ||
11480 this_shared_changing) {
11481 changing[i] = 1;
11482 changing_mask |= stat_mask;
11483 change_count++;
11484 }
11485 if (be16_to_cpu(new_bc->vl[i].dedicated) <
11486 be16_to_cpu(cur_bc.vl[i].dedicated)) {
11487 lowering_dedicated[i] = 1;
11488 ld_mask |= stat_mask;
11489 }
11490 }
11491
11492 /* bracket the credit change with a total adjustment */
11493 if (new_total > cur_total)
11494 set_global_limit(dd, new_total);
11495
11496 /*
11497 * Start the credit change algorithm.
11498 */
11499 use_all_mask = 0;
11500 if ((be16_to_cpu(new_bc->overall_shared_limit) <
11501 be16_to_cpu(cur_bc.overall_shared_limit)) ||
11502 (is_ax(dd) && any_shared_limit_changing)) {
11503 set_global_shared(dd, 0);
11504 cur_bc.overall_shared_limit = 0;
11505 use_all_mask = 1;
11506 }
11507
11508 for (i = 0; i < NUM_USABLE_VLS; i++) {
11509 if (!valid_vl(i))
11510 continue;
11511
11512 if (changing[i]) {
11513 set_vl_shared(dd, i, 0);
11514 cur_bc.vl[i].shared = 0;
11515 }
11516 }
11517
11518 wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask,
11519 "shared");
11520
11521 if (change_count > 0) {
11522 for (i = 0; i < NUM_USABLE_VLS; i++) {
11523 if (!valid_vl(i))
11524 continue;
11525
11526 if (lowering_dedicated[i]) {
11527 set_vl_dedicated(dd, i,
11528 be16_to_cpu(new_bc->
11529 vl[i].dedicated));
11530 cur_bc.vl[i].dedicated =
11531 new_bc->vl[i].dedicated;
11532 }
11533 }
11534
11535 wait_for_vl_status_clear(dd, ld_mask, "dedicated");
11536
11537 /* now raise all dedicated that are going up */
11538 for (i = 0; i < NUM_USABLE_VLS; i++) {
11539 if (!valid_vl(i))
11540 continue;
11541
11542 if (be16_to_cpu(new_bc->vl[i].dedicated) >
11543 be16_to_cpu(cur_bc.vl[i].dedicated))
11544 set_vl_dedicated(dd, i,
11545 be16_to_cpu(new_bc->
11546 vl[i].dedicated));
11547 }
11548 }
11549
11550 /* next raise all shared that are going up */
11551 for (i = 0; i < NUM_USABLE_VLS; i++) {
11552 if (!valid_vl(i))
11553 continue;
11554
11555 if (be16_to_cpu(new_bc->vl[i].shared) >
11556 be16_to_cpu(cur_bc.vl[i].shared))
11557 set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared));
11558 }
11559
11560 /* finally raise the global shared */
11561 if (be16_to_cpu(new_bc->overall_shared_limit) >
11562 be16_to_cpu(cur_bc.overall_shared_limit))
11563 set_global_shared(dd,
11564 be16_to_cpu(new_bc->overall_shared_limit));
11565
11566 /* bracket the credit change with a total adjustment */
11567 if (new_total < cur_total)
11568 set_global_limit(dd, new_total);
11569
11570 /*
11571 * Determine the actual number of operational VLS using the number of
11572 * dedicated and shared credits for each VL.
11573 */
11574 if (change_count > 0) {
11575 for (i = 0; i < TXE_NUM_DATA_VL; i++)
11576 if (be16_to_cpu(new_bc->vl[i].dedicated) > 0 ||
11577 be16_to_cpu(new_bc->vl[i].shared) > 0)
11578 vl_count++;
11579 ppd->actual_vls_operational = vl_count;
11580 ret = sdma_map_init(dd, ppd->port - 1, vl_count ?
11581 ppd->actual_vls_operational :
11582 ppd->vls_operational,
11583 NULL);
11584 if (ret == 0)
11585 ret = pio_map_init(dd, ppd->port - 1, vl_count ?
11586 ppd->actual_vls_operational :
11587 ppd->vls_operational, NULL);
11588 if (ret)
11589 return ret;
11590 }
11591 return 0;
11592 }
11593
11594 /*
11595 * Read the given fabric manager table. Return the size of the
11596 * table (in bytes) on success, and a negative error code on
11597 * failure.
11598 */
fm_get_table(struct hfi1_pportdata * ppd,int which,void * t)11599 int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t)
11600
11601 {
11602 int size;
11603 struct vl_arb_cache *vlc;
11604
11605 switch (which) {
11606 case FM_TBL_VL_HIGH_ARB:
11607 size = 256;
11608 /*
11609 * OPA specifies 128 elements (of 2 bytes each), though
11610 * HFI supports only 16 elements in h/w.
11611 */
11612 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11613 vl_arb_get_cache(vlc, t);
11614 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11615 break;
11616 case FM_TBL_VL_LOW_ARB:
11617 size = 256;
11618 /*
11619 * OPA specifies 128 elements (of 2 bytes each), though
11620 * HFI supports only 16 elements in h/w.
11621 */
11622 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11623 vl_arb_get_cache(vlc, t);
11624 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11625 break;
11626 case FM_TBL_BUFFER_CONTROL:
11627 size = get_buffer_control(ppd->dd, t, NULL);
11628 break;
11629 case FM_TBL_SC2VLNT:
11630 size = get_sc2vlnt(ppd->dd, t);
11631 break;
11632 case FM_TBL_VL_PREEMPT_ELEMS:
11633 size = 256;
11634 /* OPA specifies 128 elements, of 2 bytes each */
11635 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t);
11636 break;
11637 case FM_TBL_VL_PREEMPT_MATRIX:
11638 size = 256;
11639 /*
11640 * OPA specifies that this is the same size as the VL
11641 * arbitration tables (i.e., 256 bytes).
11642 */
11643 break;
11644 default:
11645 return -EINVAL;
11646 }
11647 return size;
11648 }
11649
11650 /*
11651 * Write the given fabric manager table.
11652 */
fm_set_table(struct hfi1_pportdata * ppd,int which,void * t)11653 int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t)
11654 {
11655 int ret = 0;
11656 struct vl_arb_cache *vlc;
11657
11658 switch (which) {
11659 case FM_TBL_VL_HIGH_ARB:
11660 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11661 if (vl_arb_match_cache(vlc, t)) {
11662 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11663 break;
11664 }
11665 vl_arb_set_cache(vlc, t);
11666 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11667 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST,
11668 VL_ARB_HIGH_PRIO_TABLE_SIZE, t);
11669 break;
11670 case FM_TBL_VL_LOW_ARB:
11671 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11672 if (vl_arb_match_cache(vlc, t)) {
11673 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11674 break;
11675 }
11676 vl_arb_set_cache(vlc, t);
11677 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11678 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST,
11679 VL_ARB_LOW_PRIO_TABLE_SIZE, t);
11680 break;
11681 case FM_TBL_BUFFER_CONTROL:
11682 ret = set_buffer_control(ppd, t);
11683 break;
11684 case FM_TBL_SC2VLNT:
11685 set_sc2vlnt(ppd->dd, t);
11686 break;
11687 default:
11688 ret = -EINVAL;
11689 }
11690 return ret;
11691 }
11692
11693 /*
11694 * Disable all data VLs.
11695 *
11696 * Return 0 if disabled, non-zero if the VLs cannot be disabled.
11697 */
disable_data_vls(struct hfi1_devdata * dd)11698 static int disable_data_vls(struct hfi1_devdata *dd)
11699 {
11700 if (is_ax(dd))
11701 return 1;
11702
11703 pio_send_control(dd, PSC_DATA_VL_DISABLE);
11704
11705 return 0;
11706 }
11707
11708 /*
11709 * open_fill_data_vls() - the counterpart to stop_drain_data_vls().
11710 * Just re-enables all data VLs (the "fill" part happens
11711 * automatically - the name was chosen for symmetry with
11712 * stop_drain_data_vls()).
11713 *
11714 * Return 0 if successful, non-zero if the VLs cannot be enabled.
11715 */
open_fill_data_vls(struct hfi1_devdata * dd)11716 int open_fill_data_vls(struct hfi1_devdata *dd)
11717 {
11718 if (is_ax(dd))
11719 return 1;
11720
11721 pio_send_control(dd, PSC_DATA_VL_ENABLE);
11722
11723 return 0;
11724 }
11725
11726 /*
11727 * drain_data_vls() - assumes that disable_data_vls() has been called,
11728 * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA
11729 * engines to drop to 0.
11730 */
drain_data_vls(struct hfi1_devdata * dd)11731 static void drain_data_vls(struct hfi1_devdata *dd)
11732 {
11733 sc_wait(dd);
11734 sdma_wait(dd);
11735 pause_for_credit_return(dd);
11736 }
11737
11738 /*
11739 * stop_drain_data_vls() - disable, then drain all per-VL fifos.
11740 *
11741 * Use open_fill_data_vls() to resume using data VLs. This pair is
11742 * meant to be used like this:
11743 *
11744 * stop_drain_data_vls(dd);
11745 * // do things with per-VL resources
11746 * open_fill_data_vls(dd);
11747 */
stop_drain_data_vls(struct hfi1_devdata * dd)11748 int stop_drain_data_vls(struct hfi1_devdata *dd)
11749 {
11750 int ret;
11751
11752 ret = disable_data_vls(dd);
11753 if (ret == 0)
11754 drain_data_vls(dd);
11755
11756 return ret;
11757 }
11758
11759 /*
11760 * Convert a nanosecond time to a cclock count. No matter how slow
11761 * the cclock, a non-zero ns will always have a non-zero result.
11762 */
ns_to_cclock(struct hfi1_devdata * dd,u32 ns)11763 u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns)
11764 {
11765 u32 cclocks;
11766
11767 if (dd->icode == ICODE_FPGA_EMULATION)
11768 cclocks = (ns * 1000) / FPGA_CCLOCK_PS;
11769 else /* simulation pretends to be ASIC */
11770 cclocks = (ns * 1000) / ASIC_CCLOCK_PS;
11771 if (ns && !cclocks) /* if ns nonzero, must be at least 1 */
11772 cclocks = 1;
11773 return cclocks;
11774 }
11775
11776 /*
11777 * Convert a cclock count to nanoseconds. Not matter how slow
11778 * the cclock, a non-zero cclocks will always have a non-zero result.
11779 */
cclock_to_ns(struct hfi1_devdata * dd,u32 cclocks)11780 u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks)
11781 {
11782 u32 ns;
11783
11784 if (dd->icode == ICODE_FPGA_EMULATION)
11785 ns = (cclocks * FPGA_CCLOCK_PS) / 1000;
11786 else /* simulation pretends to be ASIC */
11787 ns = (cclocks * ASIC_CCLOCK_PS) / 1000;
11788 if (cclocks && !ns)
11789 ns = 1;
11790 return ns;
11791 }
11792
11793 /*
11794 * Dynamically adjust the receive interrupt timeout for a context based on
11795 * incoming packet rate.
11796 *
11797 * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero.
11798 */
adjust_rcv_timeout(struct hfi1_ctxtdata * rcd,u32 npkts)11799 static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts)
11800 {
11801 struct hfi1_devdata *dd = rcd->dd;
11802 u32 timeout = rcd->rcvavail_timeout;
11803
11804 /*
11805 * This algorithm doubles or halves the timeout depending on whether
11806 * the number of packets received in this interrupt were less than or
11807 * greater equal the interrupt count.
11808 *
11809 * The calculations below do not allow a steady state to be achieved.
11810 * Only at the endpoints it is possible to have an unchanging
11811 * timeout.
11812 */
11813 if (npkts < rcv_intr_count) {
11814 /*
11815 * Not enough packets arrived before the timeout, adjust
11816 * timeout downward.
11817 */
11818 if (timeout < 2) /* already at minimum? */
11819 return;
11820 timeout >>= 1;
11821 } else {
11822 /*
11823 * More than enough packets arrived before the timeout, adjust
11824 * timeout upward.
11825 */
11826 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */
11827 return;
11828 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr);
11829 }
11830
11831 rcd->rcvavail_timeout = timeout;
11832 /*
11833 * timeout cannot be larger than rcv_intr_timeout_csr which has already
11834 * been verified to be in range
11835 */
11836 write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT,
11837 (u64)timeout <<
11838 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
11839 }
11840
update_usrhead(struct hfi1_ctxtdata * rcd,u32 hd,u32 updegr,u32 egrhd,u32 intr_adjust,u32 npkts)11841 void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd,
11842 u32 intr_adjust, u32 npkts)
11843 {
11844 struct hfi1_devdata *dd = rcd->dd;
11845 u64 reg;
11846 u32 ctxt = rcd->ctxt;
11847
11848 /*
11849 * Need to write timeout register before updating RcvHdrHead to ensure
11850 * that a new value is used when the HW decides to restart counting.
11851 */
11852 if (intr_adjust)
11853 adjust_rcv_timeout(rcd, npkts);
11854 if (updegr) {
11855 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK)
11856 << RCV_EGR_INDEX_HEAD_HEAD_SHIFT;
11857 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg);
11858 }
11859 reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) |
11860 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK)
11861 << RCV_HDR_HEAD_HEAD_SHIFT);
11862 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
11863 }
11864
hdrqempty(struct hfi1_ctxtdata * rcd)11865 u32 hdrqempty(struct hfi1_ctxtdata *rcd)
11866 {
11867 u32 head, tail;
11868
11869 head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD)
11870 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT;
11871
11872 if (hfi1_rcvhdrtail_kvaddr(rcd))
11873 tail = get_rcvhdrtail(rcd);
11874 else
11875 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
11876
11877 return head == tail;
11878 }
11879
11880 /*
11881 * Context Control and Receive Array encoding for buffer size:
11882 * 0x0 invalid
11883 * 0x1 4 KB
11884 * 0x2 8 KB
11885 * 0x3 16 KB
11886 * 0x4 32 KB
11887 * 0x5 64 KB
11888 * 0x6 128 KB
11889 * 0x7 256 KB
11890 * 0x8 512 KB (Receive Array only)
11891 * 0x9 1 MB (Receive Array only)
11892 * 0xa 2 MB (Receive Array only)
11893 *
11894 * 0xB-0xF - reserved (Receive Array only)
11895 *
11896 *
11897 * This routine assumes that the value has already been sanity checked.
11898 */
encoded_size(u32 size)11899 static u32 encoded_size(u32 size)
11900 {
11901 switch (size) {
11902 case 4 * 1024: return 0x1;
11903 case 8 * 1024: return 0x2;
11904 case 16 * 1024: return 0x3;
11905 case 32 * 1024: return 0x4;
11906 case 64 * 1024: return 0x5;
11907 case 128 * 1024: return 0x6;
11908 case 256 * 1024: return 0x7;
11909 case 512 * 1024: return 0x8;
11910 case 1 * 1024 * 1024: return 0x9;
11911 case 2 * 1024 * 1024: return 0xa;
11912 }
11913 return 0x1; /* if invalid, go with the minimum size */
11914 }
11915
11916 /**
11917 * encode_rcv_header_entry_size - return chip specific encoding for size
11918 * @size: size in dwords
11919 *
11920 * Convert a receive header entry size that to the encoding used in the CSR.
11921 *
11922 * Return a zero if the given size is invalid, otherwise the encoding.
11923 */
encode_rcv_header_entry_size(u8 size)11924 u8 encode_rcv_header_entry_size(u8 size)
11925 {
11926 /* there are only 3 valid receive header entry sizes */
11927 if (size == 2)
11928 return 1;
11929 if (size == 16)
11930 return 2;
11931 if (size == 32)
11932 return 4;
11933 return 0; /* invalid */
11934 }
11935
11936 /**
11937 * hfi1_validate_rcvhdrcnt - validate hdrcnt
11938 * @dd: the device data
11939 * @thecnt: the header count
11940 */
hfi1_validate_rcvhdrcnt(struct hfi1_devdata * dd,uint thecnt)11941 int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt)
11942 {
11943 if (thecnt <= HFI1_MIN_HDRQ_EGRBUF_CNT) {
11944 dd_dev_err(dd, "Receive header queue count too small\n");
11945 return -EINVAL;
11946 }
11947
11948 if (thecnt > HFI1_MAX_HDRQ_EGRBUF_CNT) {
11949 dd_dev_err(dd,
11950 "Receive header queue count cannot be greater than %u\n",
11951 HFI1_MAX_HDRQ_EGRBUF_CNT);
11952 return -EINVAL;
11953 }
11954
11955 if (thecnt % HDRQ_INCREMENT) {
11956 dd_dev_err(dd, "Receive header queue count %d must be divisible by %lu\n",
11957 thecnt, HDRQ_INCREMENT);
11958 return -EINVAL;
11959 }
11960
11961 return 0;
11962 }
11963
11964 /**
11965 * set_hdrq_regs - set header queue registers for context
11966 * @dd: the device data
11967 * @ctxt: the context
11968 * @entsize: the dword entry size
11969 * @hdrcnt: the number of header entries
11970 */
set_hdrq_regs(struct hfi1_devdata * dd,u8 ctxt,u8 entsize,u16 hdrcnt)11971 void set_hdrq_regs(struct hfi1_devdata *dd, u8 ctxt, u8 entsize, u16 hdrcnt)
11972 {
11973 u64 reg;
11974
11975 reg = (((u64)hdrcnt >> HDRQ_SIZE_SHIFT) & RCV_HDR_CNT_CNT_MASK) <<
11976 RCV_HDR_CNT_CNT_SHIFT;
11977 write_kctxt_csr(dd, ctxt, RCV_HDR_CNT, reg);
11978 reg = ((u64)encode_rcv_header_entry_size(entsize) &
11979 RCV_HDR_ENT_SIZE_ENT_SIZE_MASK) <<
11980 RCV_HDR_ENT_SIZE_ENT_SIZE_SHIFT;
11981 write_kctxt_csr(dd, ctxt, RCV_HDR_ENT_SIZE, reg);
11982 reg = ((u64)DEFAULT_RCVHDRSIZE & RCV_HDR_SIZE_HDR_SIZE_MASK) <<
11983 RCV_HDR_SIZE_HDR_SIZE_SHIFT;
11984 write_kctxt_csr(dd, ctxt, RCV_HDR_SIZE, reg);
11985
11986 /*
11987 * Program dummy tail address for every receive context
11988 * before enabling any receive context
11989 */
11990 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11991 dd->rcvhdrtail_dummy_dma);
11992 }
11993
hfi1_rcvctrl(struct hfi1_devdata * dd,unsigned int op,struct hfi1_ctxtdata * rcd)11994 void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,
11995 struct hfi1_ctxtdata *rcd)
11996 {
11997 u64 rcvctrl, reg;
11998 int did_enable = 0;
11999 u16 ctxt;
12000
12001 if (!rcd)
12002 return;
12003
12004 ctxt = rcd->ctxt;
12005
12006 hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op);
12007
12008 rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL);
12009 /* if the context already enabled, don't do the extra steps */
12010 if ((op & HFI1_RCVCTRL_CTXT_ENB) &&
12011 !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) {
12012 /* reset the tail and hdr addresses, and sequence count */
12013 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,
12014 rcd->rcvhdrq_dma);
12015 if (hfi1_rcvhdrtail_kvaddr(rcd))
12016 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
12017 rcd->rcvhdrqtailaddr_dma);
12018 hfi1_set_seq_cnt(rcd, 1);
12019
12020 /* reset the cached receive header queue head value */
12021 hfi1_set_rcd_head(rcd, 0);
12022
12023 /*
12024 * Zero the receive header queue so we don't get false
12025 * positives when checking the sequence number. The
12026 * sequence numbers could land exactly on the same spot.
12027 * E.g. a rcd restart before the receive header wrapped.
12028 */
12029 memset(rcd->rcvhdrq, 0, rcvhdrq_size(rcd));
12030
12031 /* starting timeout */
12032 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr;
12033
12034 /* enable the context */
12035 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK;
12036
12037 /* clean the egr buffer size first */
12038 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
12039 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size)
12040 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK)
12041 << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT;
12042
12043 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */
12044 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0);
12045 did_enable = 1;
12046
12047 /* zero RcvEgrIndexHead */
12048 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0);
12049
12050 /* set eager count and base index */
12051 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT)
12052 & RCV_EGR_CTRL_EGR_CNT_MASK)
12053 << RCV_EGR_CTRL_EGR_CNT_SHIFT) |
12054 (((rcd->eager_base >> RCV_SHIFT)
12055 & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK)
12056 << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT);
12057 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg);
12058
12059 /*
12060 * Set TID (expected) count and base index.
12061 * rcd->expected_count is set to individual RcvArray entries,
12062 * not pairs, and the CSR takes a pair-count in groups of
12063 * four, so divide by 8.
12064 */
12065 reg = (((rcd->expected_count >> RCV_SHIFT)
12066 & RCV_TID_CTRL_TID_PAIR_CNT_MASK)
12067 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) |
12068 (((rcd->expected_base >> RCV_SHIFT)
12069 & RCV_TID_CTRL_TID_BASE_INDEX_MASK)
12070 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT);
12071 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg);
12072 if (ctxt == HFI1_CTRL_CTXT)
12073 write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT);
12074 }
12075 if (op & HFI1_RCVCTRL_CTXT_DIS) {
12076 write_csr(dd, RCV_VL15, 0);
12077 /*
12078 * When receive context is being disabled turn on tail
12079 * update with a dummy tail address and then disable
12080 * receive context.
12081 */
12082 if (dd->rcvhdrtail_dummy_dma) {
12083 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
12084 dd->rcvhdrtail_dummy_dma);
12085 /* Enabling RcvCtxtCtrl.TailUpd is intentional. */
12086 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
12087 }
12088
12089 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK;
12090 }
12091 if (op & HFI1_RCVCTRL_INTRAVAIL_ENB) {
12092 set_intr_bits(dd, IS_RCVAVAIL_START + rcd->ctxt,
12093 IS_RCVAVAIL_START + rcd->ctxt, true);
12094 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
12095 }
12096 if (op & HFI1_RCVCTRL_INTRAVAIL_DIS) {
12097 set_intr_bits(dd, IS_RCVAVAIL_START + rcd->ctxt,
12098 IS_RCVAVAIL_START + rcd->ctxt, false);
12099 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
12100 }
12101 if ((op & HFI1_RCVCTRL_TAILUPD_ENB) && hfi1_rcvhdrtail_kvaddr(rcd))
12102 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
12103 if (op & HFI1_RCVCTRL_TAILUPD_DIS) {
12104 /* See comment on RcvCtxtCtrl.TailUpd above */
12105 if (!(op & HFI1_RCVCTRL_CTXT_DIS))
12106 rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK;
12107 }
12108 if (op & HFI1_RCVCTRL_TIDFLOW_ENB)
12109 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
12110 if (op & HFI1_RCVCTRL_TIDFLOW_DIS)
12111 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
12112 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) {
12113 /*
12114 * In one-packet-per-eager mode, the size comes from
12115 * the RcvArray entry.
12116 */
12117 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
12118 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
12119 }
12120 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS)
12121 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
12122 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB)
12123 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
12124 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS)
12125 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
12126 if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB)
12127 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
12128 if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS)
12129 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
12130 if (op & HFI1_RCVCTRL_URGENT_ENB)
12131 set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt,
12132 IS_RCVURGENT_START + rcd->ctxt, true);
12133 if (op & HFI1_RCVCTRL_URGENT_DIS)
12134 set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt,
12135 IS_RCVURGENT_START + rcd->ctxt, false);
12136
12137 hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl);
12138 write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcvctrl);
12139
12140 /* work around sticky RcvCtxtStatus.BlockedRHQFull */
12141 if (did_enable &&
12142 (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) {
12143 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
12144 if (reg != 0) {
12145 dd_dev_info(dd, "ctxt %d status %lld (blocked)\n",
12146 ctxt, reg);
12147 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
12148 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10);
12149 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00);
12150 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
12151 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
12152 dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n",
12153 ctxt, reg, reg == 0 ? "not" : "still");
12154 }
12155 }
12156
12157 if (did_enable) {
12158 /*
12159 * The interrupt timeout and count must be set after
12160 * the context is enabled to take effect.
12161 */
12162 /* set interrupt timeout */
12163 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT,
12164 (u64)rcd->rcvavail_timeout <<
12165 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
12166
12167 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */
12168 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT;
12169 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
12170 }
12171
12172 if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS))
12173 /*
12174 * If the context has been disabled and the Tail Update has
12175 * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address
12176 * so it doesn't contain an address that is invalid.
12177 */
12178 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
12179 dd->rcvhdrtail_dummy_dma);
12180 }
12181
hfi1_read_cntrs(struct hfi1_devdata * dd,char ** namep,u64 ** cntrp)12182 u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp)
12183 {
12184 int ret;
12185 u64 val = 0;
12186
12187 if (namep) {
12188 ret = dd->cntrnameslen;
12189 *namep = dd->cntrnames;
12190 } else {
12191 const struct cntr_entry *entry;
12192 int i, j;
12193
12194 ret = (dd->ndevcntrs) * sizeof(u64);
12195
12196 /* Get the start of the block of counters */
12197 *cntrp = dd->cntrs;
12198
12199 /*
12200 * Now go and fill in each counter in the block.
12201 */
12202 for (i = 0; i < DEV_CNTR_LAST; i++) {
12203 entry = &dev_cntrs[i];
12204 hfi1_cdbg(CNTR, "reading %s", entry->name);
12205 if (entry->flags & CNTR_DISABLED) {
12206 /* Nothing */
12207 hfi1_cdbg(CNTR, "\tDisabled\n");
12208 } else {
12209 if (entry->flags & CNTR_VL) {
12210 hfi1_cdbg(CNTR, "\tPer VL\n");
12211 for (j = 0; j < C_VL_COUNT; j++) {
12212 val = entry->rw_cntr(entry,
12213 dd, j,
12214 CNTR_MODE_R,
12215 0);
12216 hfi1_cdbg(
12217 CNTR,
12218 "\t\tRead 0x%llx for %d\n",
12219 val, j);
12220 dd->cntrs[entry->offset + j] =
12221 val;
12222 }
12223 } else if (entry->flags & CNTR_SDMA) {
12224 hfi1_cdbg(CNTR,
12225 "\t Per SDMA Engine\n");
12226 for (j = 0; j < chip_sdma_engines(dd);
12227 j++) {
12228 val =
12229 entry->rw_cntr(entry, dd, j,
12230 CNTR_MODE_R, 0);
12231 hfi1_cdbg(CNTR,
12232 "\t\tRead 0x%llx for %d\n",
12233 val, j);
12234 dd->cntrs[entry->offset + j] =
12235 val;
12236 }
12237 } else {
12238 val = entry->rw_cntr(entry, dd,
12239 CNTR_INVALID_VL,
12240 CNTR_MODE_R, 0);
12241 dd->cntrs[entry->offset] = val;
12242 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
12243 }
12244 }
12245 }
12246 }
12247 return ret;
12248 }
12249
12250 /*
12251 * Used by sysfs to create files for hfi stats to read
12252 */
hfi1_read_portcntrs(struct hfi1_pportdata * ppd,char ** namep,u64 ** cntrp)12253 u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp)
12254 {
12255 int ret;
12256 u64 val = 0;
12257
12258 if (namep) {
12259 ret = ppd->dd->portcntrnameslen;
12260 *namep = ppd->dd->portcntrnames;
12261 } else {
12262 const struct cntr_entry *entry;
12263 int i, j;
12264
12265 ret = ppd->dd->nportcntrs * sizeof(u64);
12266 *cntrp = ppd->cntrs;
12267
12268 for (i = 0; i < PORT_CNTR_LAST; i++) {
12269 entry = &port_cntrs[i];
12270 hfi1_cdbg(CNTR, "reading %s", entry->name);
12271 if (entry->flags & CNTR_DISABLED) {
12272 /* Nothing */
12273 hfi1_cdbg(CNTR, "\tDisabled\n");
12274 continue;
12275 }
12276
12277 if (entry->flags & CNTR_VL) {
12278 hfi1_cdbg(CNTR, "\tPer VL");
12279 for (j = 0; j < C_VL_COUNT; j++) {
12280 val = entry->rw_cntr(entry, ppd, j,
12281 CNTR_MODE_R,
12282 0);
12283 hfi1_cdbg(
12284 CNTR,
12285 "\t\tRead 0x%llx for %d",
12286 val, j);
12287 ppd->cntrs[entry->offset + j] = val;
12288 }
12289 } else {
12290 val = entry->rw_cntr(entry, ppd,
12291 CNTR_INVALID_VL,
12292 CNTR_MODE_R,
12293 0);
12294 ppd->cntrs[entry->offset] = val;
12295 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
12296 }
12297 }
12298 }
12299 return ret;
12300 }
12301
free_cntrs(struct hfi1_devdata * dd)12302 static void free_cntrs(struct hfi1_devdata *dd)
12303 {
12304 struct hfi1_pportdata *ppd;
12305 int i;
12306
12307 if (dd->synth_stats_timer.function)
12308 del_timer_sync(&dd->synth_stats_timer);
12309 cancel_work_sync(&dd->update_cntr_work);
12310 ppd = (struct hfi1_pportdata *)(dd + 1);
12311 for (i = 0; i < dd->num_pports; i++, ppd++) {
12312 kfree(ppd->cntrs);
12313 kfree(ppd->scntrs);
12314 free_percpu(ppd->ibport_data.rvp.rc_acks);
12315 free_percpu(ppd->ibport_data.rvp.rc_qacks);
12316 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp);
12317 ppd->cntrs = NULL;
12318 ppd->scntrs = NULL;
12319 ppd->ibport_data.rvp.rc_acks = NULL;
12320 ppd->ibport_data.rvp.rc_qacks = NULL;
12321 ppd->ibport_data.rvp.rc_delayed_comp = NULL;
12322 }
12323 kfree(dd->portcntrnames);
12324 dd->portcntrnames = NULL;
12325 kfree(dd->cntrs);
12326 dd->cntrs = NULL;
12327 kfree(dd->scntrs);
12328 dd->scntrs = NULL;
12329 kfree(dd->cntrnames);
12330 dd->cntrnames = NULL;
12331 if (dd->update_cntr_wq) {
12332 destroy_workqueue(dd->update_cntr_wq);
12333 dd->update_cntr_wq = NULL;
12334 }
12335 }
12336
read_dev_port_cntr(struct hfi1_devdata * dd,struct cntr_entry * entry,u64 * psval,void * context,int vl)12337 static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry,
12338 u64 *psval, void *context, int vl)
12339 {
12340 u64 val;
12341 u64 sval = *psval;
12342
12343 if (entry->flags & CNTR_DISABLED) {
12344 dd_dev_err(dd, "Counter %s not enabled", entry->name);
12345 return 0;
12346 }
12347
12348 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
12349
12350 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0);
12351
12352 /* If its a synthetic counter there is more work we need to do */
12353 if (entry->flags & CNTR_SYNTH) {
12354 if (sval == CNTR_MAX) {
12355 /* No need to read already saturated */
12356 return CNTR_MAX;
12357 }
12358
12359 if (entry->flags & CNTR_32BIT) {
12360 /* 32bit counters can wrap multiple times */
12361 u64 upper = sval >> 32;
12362 u64 lower = (sval << 32) >> 32;
12363
12364 if (lower > val) { /* hw wrapped */
12365 if (upper == CNTR_32BIT_MAX)
12366 val = CNTR_MAX;
12367 else
12368 upper++;
12369 }
12370
12371 if (val != CNTR_MAX)
12372 val = (upper << 32) | val;
12373
12374 } else {
12375 /* If we rolled we are saturated */
12376 if ((val < sval) || (val > CNTR_MAX))
12377 val = CNTR_MAX;
12378 }
12379 }
12380
12381 *psval = val;
12382
12383 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
12384
12385 return val;
12386 }
12387
write_dev_port_cntr(struct hfi1_devdata * dd,struct cntr_entry * entry,u64 * psval,void * context,int vl,u64 data)12388 static u64 write_dev_port_cntr(struct hfi1_devdata *dd,
12389 struct cntr_entry *entry,
12390 u64 *psval, void *context, int vl, u64 data)
12391 {
12392 u64 val;
12393
12394 if (entry->flags & CNTR_DISABLED) {
12395 dd_dev_err(dd, "Counter %s not enabled", entry->name);
12396 return 0;
12397 }
12398
12399 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
12400
12401 if (entry->flags & CNTR_SYNTH) {
12402 *psval = data;
12403 if (entry->flags & CNTR_32BIT) {
12404 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
12405 (data << 32) >> 32);
12406 val = data; /* return the full 64bit value */
12407 } else {
12408 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
12409 data);
12410 }
12411 } else {
12412 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data);
12413 }
12414
12415 *psval = val;
12416
12417 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
12418
12419 return val;
12420 }
12421
read_dev_cntr(struct hfi1_devdata * dd,int index,int vl)12422 u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl)
12423 {
12424 struct cntr_entry *entry;
12425 u64 *sval;
12426
12427 entry = &dev_cntrs[index];
12428 sval = dd->scntrs + entry->offset;
12429
12430 if (vl != CNTR_INVALID_VL)
12431 sval += vl;
12432
12433 return read_dev_port_cntr(dd, entry, sval, dd, vl);
12434 }
12435
write_dev_cntr(struct hfi1_devdata * dd,int index,int vl,u64 data)12436 u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data)
12437 {
12438 struct cntr_entry *entry;
12439 u64 *sval;
12440
12441 entry = &dev_cntrs[index];
12442 sval = dd->scntrs + entry->offset;
12443
12444 if (vl != CNTR_INVALID_VL)
12445 sval += vl;
12446
12447 return write_dev_port_cntr(dd, entry, sval, dd, vl, data);
12448 }
12449
read_port_cntr(struct hfi1_pportdata * ppd,int index,int vl)12450 u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl)
12451 {
12452 struct cntr_entry *entry;
12453 u64 *sval;
12454
12455 entry = &port_cntrs[index];
12456 sval = ppd->scntrs + entry->offset;
12457
12458 if (vl != CNTR_INVALID_VL)
12459 sval += vl;
12460
12461 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
12462 (index <= C_RCV_HDR_OVF_LAST)) {
12463 /* We do not want to bother for disabled contexts */
12464 return 0;
12465 }
12466
12467 return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl);
12468 }
12469
write_port_cntr(struct hfi1_pportdata * ppd,int index,int vl,u64 data)12470 u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data)
12471 {
12472 struct cntr_entry *entry;
12473 u64 *sval;
12474
12475 entry = &port_cntrs[index];
12476 sval = ppd->scntrs + entry->offset;
12477
12478 if (vl != CNTR_INVALID_VL)
12479 sval += vl;
12480
12481 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
12482 (index <= C_RCV_HDR_OVF_LAST)) {
12483 /* We do not want to bother for disabled contexts */
12484 return 0;
12485 }
12486
12487 return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data);
12488 }
12489
do_update_synth_timer(struct work_struct * work)12490 static void do_update_synth_timer(struct work_struct *work)
12491 {
12492 u64 cur_tx;
12493 u64 cur_rx;
12494 u64 total_flits;
12495 u8 update = 0;
12496 int i, j, vl;
12497 struct hfi1_pportdata *ppd;
12498 struct cntr_entry *entry;
12499 struct hfi1_devdata *dd = container_of(work, struct hfi1_devdata,
12500 update_cntr_work);
12501
12502 /*
12503 * Rather than keep beating on the CSRs pick a minimal set that we can
12504 * check to watch for potential roll over. We can do this by looking at
12505 * the number of flits sent/recv. If the total flits exceeds 32bits then
12506 * we have to iterate all the counters and update.
12507 */
12508 entry = &dev_cntrs[C_DC_RCV_FLITS];
12509 cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
12510
12511 entry = &dev_cntrs[C_DC_XMIT_FLITS];
12512 cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
12513
12514 hfi1_cdbg(
12515 CNTR,
12516 "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n",
12517 dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx);
12518
12519 if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) {
12520 /*
12521 * May not be strictly necessary to update but it won't hurt and
12522 * simplifies the logic here.
12523 */
12524 update = 1;
12525 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating",
12526 dd->unit);
12527 } else {
12528 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx);
12529 hfi1_cdbg(CNTR,
12530 "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit,
12531 total_flits, (u64)CNTR_32BIT_MAX);
12532 if (total_flits >= CNTR_32BIT_MAX) {
12533 hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating",
12534 dd->unit);
12535 update = 1;
12536 }
12537 }
12538
12539 if (update) {
12540 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit);
12541 for (i = 0; i < DEV_CNTR_LAST; i++) {
12542 entry = &dev_cntrs[i];
12543 if (entry->flags & CNTR_VL) {
12544 for (vl = 0; vl < C_VL_COUNT; vl++)
12545 read_dev_cntr(dd, i, vl);
12546 } else {
12547 read_dev_cntr(dd, i, CNTR_INVALID_VL);
12548 }
12549 }
12550 ppd = (struct hfi1_pportdata *)(dd + 1);
12551 for (i = 0; i < dd->num_pports; i++, ppd++) {
12552 for (j = 0; j < PORT_CNTR_LAST; j++) {
12553 entry = &port_cntrs[j];
12554 if (entry->flags & CNTR_VL) {
12555 for (vl = 0; vl < C_VL_COUNT; vl++)
12556 read_port_cntr(ppd, j, vl);
12557 } else {
12558 read_port_cntr(ppd, j, CNTR_INVALID_VL);
12559 }
12560 }
12561 }
12562
12563 /*
12564 * We want the value in the register. The goal is to keep track
12565 * of the number of "ticks" not the counter value. In other
12566 * words if the register rolls we want to notice it and go ahead
12567 * and force an update.
12568 */
12569 entry = &dev_cntrs[C_DC_XMIT_FLITS];
12570 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
12571 CNTR_MODE_R, 0);
12572
12573 entry = &dev_cntrs[C_DC_RCV_FLITS];
12574 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
12575 CNTR_MODE_R, 0);
12576
12577 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx",
12578 dd->unit, dd->last_tx, dd->last_rx);
12579
12580 } else {
12581 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit);
12582 }
12583 }
12584
update_synth_timer(struct timer_list * t)12585 static void update_synth_timer(struct timer_list *t)
12586 {
12587 struct hfi1_devdata *dd = from_timer(dd, t, synth_stats_timer);
12588
12589 queue_work(dd->update_cntr_wq, &dd->update_cntr_work);
12590 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12591 }
12592
12593 #define C_MAX_NAME 16 /* 15 chars + one for /0 */
init_cntrs(struct hfi1_devdata * dd)12594 static int init_cntrs(struct hfi1_devdata *dd)
12595 {
12596 int i, rcv_ctxts, j;
12597 size_t sz;
12598 char *p;
12599 char name[C_MAX_NAME];
12600 struct hfi1_pportdata *ppd;
12601 const char *bit_type_32 = ",32";
12602 const int bit_type_32_sz = strlen(bit_type_32);
12603 u32 sdma_engines = chip_sdma_engines(dd);
12604
12605 /* set up the stats timer; the add_timer is done at the end */
12606 timer_setup(&dd->synth_stats_timer, update_synth_timer, 0);
12607
12608 /***********************/
12609 /* per device counters */
12610 /***********************/
12611
12612 /* size names and determine how many we have*/
12613 dd->ndevcntrs = 0;
12614 sz = 0;
12615
12616 for (i = 0; i < DEV_CNTR_LAST; i++) {
12617 if (dev_cntrs[i].flags & CNTR_DISABLED) {
12618 hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name);
12619 continue;
12620 }
12621
12622 if (dev_cntrs[i].flags & CNTR_VL) {
12623 dev_cntrs[i].offset = dd->ndevcntrs;
12624 for (j = 0; j < C_VL_COUNT; j++) {
12625 snprintf(name, C_MAX_NAME, "%s%d",
12626 dev_cntrs[i].name, vl_from_idx(j));
12627 sz += strlen(name);
12628 /* Add ",32" for 32-bit counters */
12629 if (dev_cntrs[i].flags & CNTR_32BIT)
12630 sz += bit_type_32_sz;
12631 sz++;
12632 dd->ndevcntrs++;
12633 }
12634 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
12635 dev_cntrs[i].offset = dd->ndevcntrs;
12636 for (j = 0; j < sdma_engines; j++) {
12637 snprintf(name, C_MAX_NAME, "%s%d",
12638 dev_cntrs[i].name, j);
12639 sz += strlen(name);
12640 /* Add ",32" for 32-bit counters */
12641 if (dev_cntrs[i].flags & CNTR_32BIT)
12642 sz += bit_type_32_sz;
12643 sz++;
12644 dd->ndevcntrs++;
12645 }
12646 } else {
12647 /* +1 for newline. */
12648 sz += strlen(dev_cntrs[i].name) + 1;
12649 /* Add ",32" for 32-bit counters */
12650 if (dev_cntrs[i].flags & CNTR_32BIT)
12651 sz += bit_type_32_sz;
12652 dev_cntrs[i].offset = dd->ndevcntrs;
12653 dd->ndevcntrs++;
12654 }
12655 }
12656
12657 /* allocate space for the counter values */
12658 dd->cntrs = kcalloc(dd->ndevcntrs + num_driver_cntrs, sizeof(u64),
12659 GFP_KERNEL);
12660 if (!dd->cntrs)
12661 goto bail;
12662
12663 dd->scntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
12664 if (!dd->scntrs)
12665 goto bail;
12666
12667 /* allocate space for the counter names */
12668 dd->cntrnameslen = sz;
12669 dd->cntrnames = kmalloc(sz, GFP_KERNEL);
12670 if (!dd->cntrnames)
12671 goto bail;
12672
12673 /* fill in the names */
12674 for (p = dd->cntrnames, i = 0; i < DEV_CNTR_LAST; i++) {
12675 if (dev_cntrs[i].flags & CNTR_DISABLED) {
12676 /* Nothing */
12677 } else if (dev_cntrs[i].flags & CNTR_VL) {
12678 for (j = 0; j < C_VL_COUNT; j++) {
12679 snprintf(name, C_MAX_NAME, "%s%d",
12680 dev_cntrs[i].name,
12681 vl_from_idx(j));
12682 memcpy(p, name, strlen(name));
12683 p += strlen(name);
12684
12685 /* Counter is 32 bits */
12686 if (dev_cntrs[i].flags & CNTR_32BIT) {
12687 memcpy(p, bit_type_32, bit_type_32_sz);
12688 p += bit_type_32_sz;
12689 }
12690
12691 *p++ = '\n';
12692 }
12693 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
12694 for (j = 0; j < sdma_engines; j++) {
12695 snprintf(name, C_MAX_NAME, "%s%d",
12696 dev_cntrs[i].name, j);
12697 memcpy(p, name, strlen(name));
12698 p += strlen(name);
12699
12700 /* Counter is 32 bits */
12701 if (dev_cntrs[i].flags & CNTR_32BIT) {
12702 memcpy(p, bit_type_32, bit_type_32_sz);
12703 p += bit_type_32_sz;
12704 }
12705
12706 *p++ = '\n';
12707 }
12708 } else {
12709 memcpy(p, dev_cntrs[i].name, strlen(dev_cntrs[i].name));
12710 p += strlen(dev_cntrs[i].name);
12711
12712 /* Counter is 32 bits */
12713 if (dev_cntrs[i].flags & CNTR_32BIT) {
12714 memcpy(p, bit_type_32, bit_type_32_sz);
12715 p += bit_type_32_sz;
12716 }
12717
12718 *p++ = '\n';
12719 }
12720 }
12721
12722 /*********************/
12723 /* per port counters */
12724 /*********************/
12725
12726 /*
12727 * Go through the counters for the overflows and disable the ones we
12728 * don't need. This varies based on platform so we need to do it
12729 * dynamically here.
12730 */
12731 rcv_ctxts = dd->num_rcv_contexts;
12732 for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts;
12733 i <= C_RCV_HDR_OVF_LAST; i++) {
12734 port_cntrs[i].flags |= CNTR_DISABLED;
12735 }
12736
12737 /* size port counter names and determine how many we have*/
12738 sz = 0;
12739 dd->nportcntrs = 0;
12740 for (i = 0; i < PORT_CNTR_LAST; i++) {
12741 if (port_cntrs[i].flags & CNTR_DISABLED) {
12742 hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name);
12743 continue;
12744 }
12745
12746 if (port_cntrs[i].flags & CNTR_VL) {
12747 port_cntrs[i].offset = dd->nportcntrs;
12748 for (j = 0; j < C_VL_COUNT; j++) {
12749 snprintf(name, C_MAX_NAME, "%s%d",
12750 port_cntrs[i].name, vl_from_idx(j));
12751 sz += strlen(name);
12752 /* Add ",32" for 32-bit counters */
12753 if (port_cntrs[i].flags & CNTR_32BIT)
12754 sz += bit_type_32_sz;
12755 sz++;
12756 dd->nportcntrs++;
12757 }
12758 } else {
12759 /* +1 for newline */
12760 sz += strlen(port_cntrs[i].name) + 1;
12761 /* Add ",32" for 32-bit counters */
12762 if (port_cntrs[i].flags & CNTR_32BIT)
12763 sz += bit_type_32_sz;
12764 port_cntrs[i].offset = dd->nportcntrs;
12765 dd->nportcntrs++;
12766 }
12767 }
12768
12769 /* allocate space for the counter names */
12770 dd->portcntrnameslen = sz;
12771 dd->portcntrnames = kmalloc(sz, GFP_KERNEL);
12772 if (!dd->portcntrnames)
12773 goto bail;
12774
12775 /* fill in port cntr names */
12776 for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) {
12777 if (port_cntrs[i].flags & CNTR_DISABLED)
12778 continue;
12779
12780 if (port_cntrs[i].flags & CNTR_VL) {
12781 for (j = 0; j < C_VL_COUNT; j++) {
12782 snprintf(name, C_MAX_NAME, "%s%d",
12783 port_cntrs[i].name, vl_from_idx(j));
12784 memcpy(p, name, strlen(name));
12785 p += strlen(name);
12786
12787 /* Counter is 32 bits */
12788 if (port_cntrs[i].flags & CNTR_32BIT) {
12789 memcpy(p, bit_type_32, bit_type_32_sz);
12790 p += bit_type_32_sz;
12791 }
12792
12793 *p++ = '\n';
12794 }
12795 } else {
12796 memcpy(p, port_cntrs[i].name,
12797 strlen(port_cntrs[i].name));
12798 p += strlen(port_cntrs[i].name);
12799
12800 /* Counter is 32 bits */
12801 if (port_cntrs[i].flags & CNTR_32BIT) {
12802 memcpy(p, bit_type_32, bit_type_32_sz);
12803 p += bit_type_32_sz;
12804 }
12805
12806 *p++ = '\n';
12807 }
12808 }
12809
12810 /* allocate per port storage for counter values */
12811 ppd = (struct hfi1_pportdata *)(dd + 1);
12812 for (i = 0; i < dd->num_pports; i++, ppd++) {
12813 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12814 if (!ppd->cntrs)
12815 goto bail;
12816
12817 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12818 if (!ppd->scntrs)
12819 goto bail;
12820 }
12821
12822 /* CPU counters need to be allocated and zeroed */
12823 if (init_cpu_counters(dd))
12824 goto bail;
12825
12826 dd->update_cntr_wq = alloc_ordered_workqueue("hfi1_update_cntr_%d",
12827 WQ_MEM_RECLAIM, dd->unit);
12828 if (!dd->update_cntr_wq)
12829 goto bail;
12830
12831 INIT_WORK(&dd->update_cntr_work, do_update_synth_timer);
12832
12833 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12834 return 0;
12835 bail:
12836 free_cntrs(dd);
12837 return -ENOMEM;
12838 }
12839
chip_to_opa_lstate(struct hfi1_devdata * dd,u32 chip_lstate)12840 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)
12841 {
12842 switch (chip_lstate) {
12843 case LSTATE_DOWN:
12844 return IB_PORT_DOWN;
12845 case LSTATE_INIT:
12846 return IB_PORT_INIT;
12847 case LSTATE_ARMED:
12848 return IB_PORT_ARMED;
12849 case LSTATE_ACTIVE:
12850 return IB_PORT_ACTIVE;
12851 default:
12852 dd_dev_err(dd,
12853 "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
12854 chip_lstate);
12855 return IB_PORT_DOWN;
12856 }
12857 }
12858
chip_to_opa_pstate(struct hfi1_devdata * dd,u32 chip_pstate)12859 u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)
12860 {
12861 /* look at the HFI meta-states only */
12862 switch (chip_pstate & 0xf0) {
12863 case PLS_DISABLED:
12864 return IB_PORTPHYSSTATE_DISABLED;
12865 case PLS_OFFLINE:
12866 return OPA_PORTPHYSSTATE_OFFLINE;
12867 case PLS_POLLING:
12868 return IB_PORTPHYSSTATE_POLLING;
12869 case PLS_CONFIGPHY:
12870 return IB_PORTPHYSSTATE_TRAINING;
12871 case PLS_LINKUP:
12872 return IB_PORTPHYSSTATE_LINKUP;
12873 case PLS_PHYTEST:
12874 return IB_PORTPHYSSTATE_PHY_TEST;
12875 default:
12876 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
12877 chip_pstate);
12878 return IB_PORTPHYSSTATE_DISABLED;
12879 }
12880 }
12881
12882 /* return the OPA port logical state name */
opa_lstate_name(u32 lstate)12883 const char *opa_lstate_name(u32 lstate)
12884 {
12885 static const char * const port_logical_names[] = {
12886 "PORT_NOP",
12887 "PORT_DOWN",
12888 "PORT_INIT",
12889 "PORT_ARMED",
12890 "PORT_ACTIVE",
12891 "PORT_ACTIVE_DEFER",
12892 };
12893 if (lstate < ARRAY_SIZE(port_logical_names))
12894 return port_logical_names[lstate];
12895 return "unknown";
12896 }
12897
12898 /* return the OPA port physical state name */
opa_pstate_name(u32 pstate)12899 const char *opa_pstate_name(u32 pstate)
12900 {
12901 static const char * const port_physical_names[] = {
12902 "PHYS_NOP",
12903 "reserved1",
12904 "PHYS_POLL",
12905 "PHYS_DISABLED",
12906 "PHYS_TRAINING",
12907 "PHYS_LINKUP",
12908 "PHYS_LINK_ERR_RECOVER",
12909 "PHYS_PHY_TEST",
12910 "reserved8",
12911 "PHYS_OFFLINE",
12912 "PHYS_GANGED",
12913 "PHYS_TEST",
12914 };
12915 if (pstate < ARRAY_SIZE(port_physical_names))
12916 return port_physical_names[pstate];
12917 return "unknown";
12918 }
12919
12920 /**
12921 * update_statusp - Update userspace status flag
12922 * @ppd: Port data structure
12923 * @state: port state information
12924 *
12925 * Actual port status is determined by the host_link_state value
12926 * in the ppd.
12927 *
12928 * host_link_state MUST be updated before updating the user space
12929 * statusp.
12930 */
update_statusp(struct hfi1_pportdata * ppd,u32 state)12931 static void update_statusp(struct hfi1_pportdata *ppd, u32 state)
12932 {
12933 /*
12934 * Set port status flags in the page mapped into userspace
12935 * memory. Do it here to ensure a reliable state - this is
12936 * the only function called by all state handling code.
12937 * Always set the flags due to the fact that the cache value
12938 * might have been changed explicitly outside of this
12939 * function.
12940 */
12941 if (ppd->statusp) {
12942 switch (state) {
12943 case IB_PORT_DOWN:
12944 case IB_PORT_INIT:
12945 *ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
12946 HFI1_STATUS_IB_READY);
12947 break;
12948 case IB_PORT_ARMED:
12949 *ppd->statusp |= HFI1_STATUS_IB_CONF;
12950 break;
12951 case IB_PORT_ACTIVE:
12952 *ppd->statusp |= HFI1_STATUS_IB_READY;
12953 break;
12954 }
12955 }
12956 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n",
12957 opa_lstate_name(state), state);
12958 }
12959
12960 /**
12961 * wait_logical_linkstate - wait for an IB link state change to occur
12962 * @ppd: port device
12963 * @state: the state to wait for
12964 * @msecs: the number of milliseconds to wait
12965 *
12966 * Wait up to msecs milliseconds for IB link state change to occur.
12967 * For now, take the easy polling route.
12968 * Returns 0 if state reached, otherwise -ETIMEDOUT.
12969 */
wait_logical_linkstate(struct hfi1_pportdata * ppd,u32 state,int msecs)12970 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
12971 int msecs)
12972 {
12973 unsigned long timeout;
12974 u32 new_state;
12975
12976 timeout = jiffies + msecs_to_jiffies(msecs);
12977 while (1) {
12978 new_state = chip_to_opa_lstate(ppd->dd,
12979 read_logical_state(ppd->dd));
12980 if (new_state == state)
12981 break;
12982 if (time_after(jiffies, timeout)) {
12983 dd_dev_err(ppd->dd,
12984 "timeout waiting for link state 0x%x\n",
12985 state);
12986 return -ETIMEDOUT;
12987 }
12988 msleep(20);
12989 }
12990
12991 return 0;
12992 }
12993
log_state_transition(struct hfi1_pportdata * ppd,u32 state)12994 static void log_state_transition(struct hfi1_pportdata *ppd, u32 state)
12995 {
12996 u32 ib_pstate = chip_to_opa_pstate(ppd->dd, state);
12997
12998 dd_dev_info(ppd->dd,
12999 "physical state changed to %s (0x%x), phy 0x%x\n",
13000 opa_pstate_name(ib_pstate), ib_pstate, state);
13001 }
13002
13003 /*
13004 * Read the physical hardware link state and check if it matches host
13005 * drivers anticipated state.
13006 */
log_physical_state(struct hfi1_pportdata * ppd,u32 state)13007 static void log_physical_state(struct hfi1_pportdata *ppd, u32 state)
13008 {
13009 u32 read_state = read_physical_state(ppd->dd);
13010
13011 if (read_state == state) {
13012 log_state_transition(ppd, state);
13013 } else {
13014 dd_dev_err(ppd->dd,
13015 "anticipated phy link state 0x%x, read 0x%x\n",
13016 state, read_state);
13017 }
13018 }
13019
13020 /*
13021 * wait_physical_linkstate - wait for an physical link state change to occur
13022 * @ppd: port device
13023 * @state: the state to wait for
13024 * @msecs: the number of milliseconds to wait
13025 *
13026 * Wait up to msecs milliseconds for physical link state change to occur.
13027 * Returns 0 if state reached, otherwise -ETIMEDOUT.
13028 */
wait_physical_linkstate(struct hfi1_pportdata * ppd,u32 state,int msecs)13029 static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state,
13030 int msecs)
13031 {
13032 u32 read_state;
13033 unsigned long timeout;
13034
13035 timeout = jiffies + msecs_to_jiffies(msecs);
13036 while (1) {
13037 read_state = read_physical_state(ppd->dd);
13038 if (read_state == state)
13039 break;
13040 if (time_after(jiffies, timeout)) {
13041 dd_dev_err(ppd->dd,
13042 "timeout waiting for phy link state 0x%x\n",
13043 state);
13044 return -ETIMEDOUT;
13045 }
13046 usleep_range(1950, 2050); /* sleep 2ms-ish */
13047 }
13048
13049 log_state_transition(ppd, state);
13050 return 0;
13051 }
13052
13053 /*
13054 * wait_phys_link_offline_quiet_substates - wait for any offline substate
13055 * @ppd: port device
13056 * @msecs: the number of milliseconds to wait
13057 *
13058 * Wait up to msecs milliseconds for any offline physical link
13059 * state change to occur.
13060 * Returns 0 if at least one state is reached, otherwise -ETIMEDOUT.
13061 */
wait_phys_link_offline_substates(struct hfi1_pportdata * ppd,int msecs)13062 static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd,
13063 int msecs)
13064 {
13065 u32 read_state;
13066 unsigned long timeout;
13067
13068 timeout = jiffies + msecs_to_jiffies(msecs);
13069 while (1) {
13070 read_state = read_physical_state(ppd->dd);
13071 if ((read_state & 0xF0) == PLS_OFFLINE)
13072 break;
13073 if (time_after(jiffies, timeout)) {
13074 dd_dev_err(ppd->dd,
13075 "timeout waiting for phy link offline.quiet substates. Read state 0x%x, %dms\n",
13076 read_state, msecs);
13077 return -ETIMEDOUT;
13078 }
13079 usleep_range(1950, 2050); /* sleep 2ms-ish */
13080 }
13081
13082 log_state_transition(ppd, read_state);
13083 return read_state;
13084 }
13085
13086 /*
13087 * wait_phys_link_out_of_offline - wait for any out of offline state
13088 * @ppd: port device
13089 * @msecs: the number of milliseconds to wait
13090 *
13091 * Wait up to msecs milliseconds for any out of offline physical link
13092 * state change to occur.
13093 * Returns 0 if at least one state is reached, otherwise -ETIMEDOUT.
13094 */
wait_phys_link_out_of_offline(struct hfi1_pportdata * ppd,int msecs)13095 static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd,
13096 int msecs)
13097 {
13098 u32 read_state;
13099 unsigned long timeout;
13100
13101 timeout = jiffies + msecs_to_jiffies(msecs);
13102 while (1) {
13103 read_state = read_physical_state(ppd->dd);
13104 if ((read_state & 0xF0) != PLS_OFFLINE)
13105 break;
13106 if (time_after(jiffies, timeout)) {
13107 dd_dev_err(ppd->dd,
13108 "timeout waiting for phy link out of offline. Read state 0x%x, %dms\n",
13109 read_state, msecs);
13110 return -ETIMEDOUT;
13111 }
13112 usleep_range(1950, 2050); /* sleep 2ms-ish */
13113 }
13114
13115 log_state_transition(ppd, read_state);
13116 return read_state;
13117 }
13118
13119 #define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
13120 (r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
13121
13122 #define SET_STATIC_RATE_CONTROL_SMASK(r) \
13123 (r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
13124
hfi1_init_ctxt(struct send_context * sc)13125 void hfi1_init_ctxt(struct send_context *sc)
13126 {
13127 if (sc) {
13128 struct hfi1_devdata *dd = sc->dd;
13129 u64 reg;
13130 u8 set = (sc->type == SC_USER ?
13131 HFI1_CAP_IS_USET(STATIC_RATE_CTRL) :
13132 HFI1_CAP_IS_KSET(STATIC_RATE_CTRL));
13133 reg = read_kctxt_csr(dd, sc->hw_context,
13134 SEND_CTXT_CHECK_ENABLE);
13135 if (set)
13136 CLEAR_STATIC_RATE_CONTROL_SMASK(reg);
13137 else
13138 SET_STATIC_RATE_CONTROL_SMASK(reg);
13139 write_kctxt_csr(dd, sc->hw_context,
13140 SEND_CTXT_CHECK_ENABLE, reg);
13141 }
13142 }
13143
hfi1_tempsense_rd(struct hfi1_devdata * dd,struct hfi1_temp * temp)13144 int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp)
13145 {
13146 int ret = 0;
13147 u64 reg;
13148
13149 if (dd->icode != ICODE_RTL_SILICON) {
13150 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
13151 dd_dev_info(dd, "%s: tempsense not supported by HW\n",
13152 __func__);
13153 return -EINVAL;
13154 }
13155 reg = read_csr(dd, ASIC_STS_THERM);
13156 temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) &
13157 ASIC_STS_THERM_CURR_TEMP_MASK);
13158 temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) &
13159 ASIC_STS_THERM_LO_TEMP_MASK);
13160 temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) &
13161 ASIC_STS_THERM_HI_TEMP_MASK);
13162 temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) &
13163 ASIC_STS_THERM_CRIT_TEMP_MASK);
13164 /* triggers is a 3-bit value - 1 bit per trigger. */
13165 temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7);
13166
13167 return ret;
13168 }
13169
13170 /* ========================================================================= */
13171
13172 /**
13173 * read_mod_write() - Calculate the IRQ register index and set/clear the bits
13174 * @dd: valid devdata
13175 * @src: IRQ source to determine register index from
13176 * @bits: the bits to set or clear
13177 * @set: true == set the bits, false == clear the bits
13178 *
13179 */
read_mod_write(struct hfi1_devdata * dd,u16 src,u64 bits,bool set)13180 static void read_mod_write(struct hfi1_devdata *dd, u16 src, u64 bits,
13181 bool set)
13182 {
13183 u64 reg;
13184 u16 idx = src / BITS_PER_REGISTER;
13185
13186 spin_lock(&dd->irq_src_lock);
13187 reg = read_csr(dd, CCE_INT_MASK + (8 * idx));
13188 if (set)
13189 reg |= bits;
13190 else
13191 reg &= ~bits;
13192 write_csr(dd, CCE_INT_MASK + (8 * idx), reg);
13193 spin_unlock(&dd->irq_src_lock);
13194 }
13195
13196 /**
13197 * set_intr_bits() - Enable/disable a range (one or more) IRQ sources
13198 * @dd: valid devdata
13199 * @first: first IRQ source to set/clear
13200 * @last: last IRQ source (inclusive) to set/clear
13201 * @set: true == set the bits, false == clear the bits
13202 *
13203 * If first == last, set the exact source.
13204 */
set_intr_bits(struct hfi1_devdata * dd,u16 first,u16 last,bool set)13205 int set_intr_bits(struct hfi1_devdata *dd, u16 first, u16 last, bool set)
13206 {
13207 u64 bits = 0;
13208 u64 bit;
13209 u16 src;
13210
13211 if (first > NUM_INTERRUPT_SOURCES || last > NUM_INTERRUPT_SOURCES)
13212 return -EINVAL;
13213
13214 if (last < first)
13215 return -ERANGE;
13216
13217 for (src = first; src <= last; src++) {
13218 bit = src % BITS_PER_REGISTER;
13219 /* wrapped to next register? */
13220 if (!bit && bits) {
13221 read_mod_write(dd, src - 1, bits, set);
13222 bits = 0;
13223 }
13224 bits |= BIT_ULL(bit);
13225 }
13226 read_mod_write(dd, last, bits, set);
13227
13228 return 0;
13229 }
13230
13231 /*
13232 * Clear all interrupt sources on the chip.
13233 */
clear_all_interrupts(struct hfi1_devdata * dd)13234 void clear_all_interrupts(struct hfi1_devdata *dd)
13235 {
13236 int i;
13237
13238 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
13239 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~(u64)0);
13240
13241 write_csr(dd, CCE_ERR_CLEAR, ~(u64)0);
13242 write_csr(dd, MISC_ERR_CLEAR, ~(u64)0);
13243 write_csr(dd, RCV_ERR_CLEAR, ~(u64)0);
13244 write_csr(dd, SEND_ERR_CLEAR, ~(u64)0);
13245 write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0);
13246 write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0);
13247 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0);
13248 for (i = 0; i < chip_send_contexts(dd); i++)
13249 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0);
13250 for (i = 0; i < chip_sdma_engines(dd); i++)
13251 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0);
13252
13253 write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0);
13254 write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0);
13255 write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0);
13256 }
13257
13258 /*
13259 * Remap the interrupt source from the general handler to the given MSI-X
13260 * interrupt.
13261 */
remap_intr(struct hfi1_devdata * dd,int isrc,int msix_intr)13262 void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr)
13263 {
13264 u64 reg;
13265 int m, n;
13266
13267 /* clear from the handled mask of the general interrupt */
13268 m = isrc / 64;
13269 n = isrc % 64;
13270 if (likely(m < CCE_NUM_INT_CSRS)) {
13271 dd->gi_mask[m] &= ~((u64)1 << n);
13272 } else {
13273 dd_dev_err(dd, "remap interrupt err\n");
13274 return;
13275 }
13276
13277 /* direct the chip source to the given MSI-X interrupt */
13278 m = isrc / 8;
13279 n = isrc % 8;
13280 reg = read_csr(dd, CCE_INT_MAP + (8 * m));
13281 reg &= ~((u64)0xff << (8 * n));
13282 reg |= ((u64)msix_intr & 0xff) << (8 * n);
13283 write_csr(dd, CCE_INT_MAP + (8 * m), reg);
13284 }
13285
remap_sdma_interrupts(struct hfi1_devdata * dd,int engine,int msix_intr)13286 void remap_sdma_interrupts(struct hfi1_devdata *dd, int engine, int msix_intr)
13287 {
13288 /*
13289 * SDMA engine interrupt sources grouped by type, rather than
13290 * engine. Per-engine interrupts are as follows:
13291 * SDMA
13292 * SDMAProgress
13293 * SDMAIdle
13294 */
13295 remap_intr(dd, IS_SDMA_START + engine, msix_intr);
13296 remap_intr(dd, IS_SDMA_PROGRESS_START + engine, msix_intr);
13297 remap_intr(dd, IS_SDMA_IDLE_START + engine, msix_intr);
13298 }
13299
13300 /*
13301 * Set the general handler to accept all interrupts, remap all
13302 * chip interrupts back to MSI-X 0.
13303 */
reset_interrupts(struct hfi1_devdata * dd)13304 void reset_interrupts(struct hfi1_devdata *dd)
13305 {
13306 int i;
13307
13308 /* all interrupts handled by the general handler */
13309 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
13310 dd->gi_mask[i] = ~(u64)0;
13311
13312 /* all chip interrupts map to MSI-X 0 */
13313 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13314 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
13315 }
13316
13317 /**
13318 * set_up_interrupts() - Initialize the IRQ resources and state
13319 * @dd: valid devdata
13320 *
13321 */
set_up_interrupts(struct hfi1_devdata * dd)13322 static int set_up_interrupts(struct hfi1_devdata *dd)
13323 {
13324 int ret;
13325
13326 /* mask all interrupts */
13327 set_intr_bits(dd, IS_FIRST_SOURCE, IS_LAST_SOURCE, false);
13328
13329 /* clear all pending interrupts */
13330 clear_all_interrupts(dd);
13331
13332 /* reset general handler mask, chip MSI-X mappings */
13333 reset_interrupts(dd);
13334
13335 /* ask for MSI-X interrupts */
13336 ret = msix_initialize(dd);
13337 if (ret)
13338 return ret;
13339
13340 ret = msix_request_irqs(dd);
13341 if (ret)
13342 msix_clean_up_interrupts(dd);
13343
13344 return ret;
13345 }
13346
13347 /*
13348 * Set up context values in dd. Sets:
13349 *
13350 * num_rcv_contexts - number of contexts being used
13351 * n_krcv_queues - number of kernel contexts
13352 * first_dyn_alloc_ctxt - first dynamically allocated context
13353 * in array of contexts
13354 * freectxts - number of free user contexts
13355 * num_send_contexts - number of PIO send contexts being used
13356 * num_netdev_contexts - number of contexts reserved for netdev
13357 */
set_up_context_variables(struct hfi1_devdata * dd)13358 static int set_up_context_variables(struct hfi1_devdata *dd)
13359 {
13360 unsigned long num_kernel_contexts;
13361 u16 num_netdev_contexts;
13362 int ret;
13363 unsigned ngroups;
13364 int rmt_count;
13365 u32 n_usr_ctxts;
13366 u32 send_contexts = chip_send_contexts(dd);
13367 u32 rcv_contexts = chip_rcv_contexts(dd);
13368
13369 /*
13370 * Kernel receive contexts:
13371 * - Context 0 - control context (VL15/multicast/error)
13372 * - Context 1 - first kernel context
13373 * - Context 2 - second kernel context
13374 * ...
13375 */
13376 if (n_krcvqs)
13377 /*
13378 * n_krcvqs is the sum of module parameter kernel receive
13379 * contexts, krcvqs[]. It does not include the control
13380 * context, so add that.
13381 */
13382 num_kernel_contexts = n_krcvqs + 1;
13383 else
13384 num_kernel_contexts = DEFAULT_KRCVQS + 1;
13385 /*
13386 * Every kernel receive context needs an ACK send context.
13387 * one send context is allocated for each VL{0-7} and VL15
13388 */
13389 if (num_kernel_contexts > (send_contexts - num_vls - 1)) {
13390 dd_dev_err(dd,
13391 "Reducing # kernel rcv contexts to: %d, from %lu\n",
13392 send_contexts - num_vls - 1,
13393 num_kernel_contexts);
13394 num_kernel_contexts = send_contexts - num_vls - 1;
13395 }
13396
13397 /*
13398 * User contexts:
13399 * - default to 1 user context per real (non-HT) CPU core if
13400 * num_user_contexts is negative
13401 */
13402 if (num_user_contexts < 0)
13403 n_usr_ctxts = cpumask_weight(&node_affinity.real_cpu_mask);
13404 else
13405 n_usr_ctxts = num_user_contexts;
13406 /*
13407 * Adjust the counts given a global max.
13408 */
13409 if (num_kernel_contexts + n_usr_ctxts > rcv_contexts) {
13410 dd_dev_err(dd,
13411 "Reducing # user receive contexts to: %u, from %u\n",
13412 (u32)(rcv_contexts - num_kernel_contexts),
13413 n_usr_ctxts);
13414 /* recalculate */
13415 n_usr_ctxts = rcv_contexts - num_kernel_contexts;
13416 }
13417
13418 num_netdev_contexts =
13419 hfi1_num_netdev_contexts(dd, rcv_contexts -
13420 (num_kernel_contexts + n_usr_ctxts),
13421 &node_affinity.real_cpu_mask);
13422 /*
13423 * RMT entries are allocated as follows:
13424 * 1. QOS (0 to 128 entries)
13425 * 2. FECN (num_kernel_context - 1 [a] + num_user_contexts +
13426 * num_netdev_contexts [b])
13427 * 3. netdev (NUM_NETDEV_MAP_ENTRIES)
13428 *
13429 * Notes:
13430 * [a] Kernel contexts (except control) are included in FECN if kernel
13431 * TID_RDMA is active.
13432 * [b] Netdev and user contexts are randomly allocated from the same
13433 * context pool, so FECN must cover all contexts in the pool.
13434 */
13435 rmt_count = qos_rmt_entries(num_kernel_contexts - 1, NULL, NULL)
13436 + (HFI1_CAP_IS_KSET(TID_RDMA) ? num_kernel_contexts - 1
13437 : 0)
13438 + n_usr_ctxts
13439 + num_netdev_contexts
13440 + NUM_NETDEV_MAP_ENTRIES;
13441 if (rmt_count > NUM_MAP_ENTRIES) {
13442 int over = rmt_count - NUM_MAP_ENTRIES;
13443 /* try to squish user contexts, minimum of 1 */
13444 if (over >= n_usr_ctxts) {
13445 dd_dev_err(dd, "RMT overflow: reduce the requested number of contexts\n");
13446 return -EINVAL;
13447 }
13448 dd_dev_err(dd, "RMT overflow: reducing # user contexts from %u to %u\n",
13449 n_usr_ctxts, n_usr_ctxts - over);
13450 n_usr_ctxts -= over;
13451 }
13452
13453 /* the first N are kernel contexts, the rest are user/netdev contexts */
13454 dd->num_rcv_contexts =
13455 num_kernel_contexts + n_usr_ctxts + num_netdev_contexts;
13456 dd->n_krcv_queues = num_kernel_contexts;
13457 dd->first_dyn_alloc_ctxt = num_kernel_contexts;
13458 dd->num_netdev_contexts = num_netdev_contexts;
13459 dd->num_user_contexts = n_usr_ctxts;
13460 dd->freectxts = n_usr_ctxts;
13461 dd_dev_info(dd,
13462 "rcv contexts: chip %d, used %d (kernel %d, netdev %u, user %u)\n",
13463 rcv_contexts,
13464 (int)dd->num_rcv_contexts,
13465 (int)dd->n_krcv_queues,
13466 dd->num_netdev_contexts,
13467 dd->num_user_contexts);
13468
13469 /*
13470 * Receive array allocation:
13471 * All RcvArray entries are divided into groups of 8. This
13472 * is required by the hardware and will speed up writes to
13473 * consecutive entries by using write-combining of the entire
13474 * cacheline.
13475 *
13476 * The number of groups are evenly divided among all contexts.
13477 * any left over groups will be given to the first N user
13478 * contexts.
13479 */
13480 dd->rcv_entries.group_size = RCV_INCREMENT;
13481 ngroups = chip_rcv_array_count(dd) / dd->rcv_entries.group_size;
13482 dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts;
13483 dd->rcv_entries.nctxt_extra = ngroups -
13484 (dd->num_rcv_contexts * dd->rcv_entries.ngroups);
13485 dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n",
13486 dd->rcv_entries.ngroups,
13487 dd->rcv_entries.nctxt_extra);
13488 if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size >
13489 MAX_EAGER_ENTRIES * 2) {
13490 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) /
13491 dd->rcv_entries.group_size;
13492 dd_dev_info(dd,
13493 "RcvArray group count too high, change to %u\n",
13494 dd->rcv_entries.ngroups);
13495 dd->rcv_entries.nctxt_extra = 0;
13496 }
13497 /*
13498 * PIO send contexts
13499 */
13500 ret = init_sc_pools_and_sizes(dd);
13501 if (ret >= 0) { /* success */
13502 dd->num_send_contexts = ret;
13503 dd_dev_info(
13504 dd,
13505 "send contexts: chip %d, used %d (kernel %d, ack %d, user %d, vl15 %d)\n",
13506 send_contexts,
13507 dd->num_send_contexts,
13508 dd->sc_sizes[SC_KERNEL].count,
13509 dd->sc_sizes[SC_ACK].count,
13510 dd->sc_sizes[SC_USER].count,
13511 dd->sc_sizes[SC_VL15].count);
13512 ret = 0; /* success */
13513 }
13514
13515 return ret;
13516 }
13517
13518 /*
13519 * Set the device/port partition key table. The MAD code
13520 * will ensure that, at least, the partial management
13521 * partition key is present in the table.
13522 */
set_partition_keys(struct hfi1_pportdata * ppd)13523 static void set_partition_keys(struct hfi1_pportdata *ppd)
13524 {
13525 struct hfi1_devdata *dd = ppd->dd;
13526 u64 reg = 0;
13527 int i;
13528
13529 dd_dev_info(dd, "Setting partition keys\n");
13530 for (i = 0; i < hfi1_get_npkeys(dd); i++) {
13531 reg |= (ppd->pkeys[i] &
13532 RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) <<
13533 ((i % 4) *
13534 RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT);
13535 /* Each register holds 4 PKey values. */
13536 if ((i % 4) == 3) {
13537 write_csr(dd, RCV_PARTITION_KEY +
13538 ((i - 3) * 2), reg);
13539 reg = 0;
13540 }
13541 }
13542
13543 /* Always enable HW pkeys check when pkeys table is set */
13544 add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK);
13545 }
13546
13547 /*
13548 * These CSRs and memories are uninitialized on reset and must be
13549 * written before reading to set the ECC/parity bits.
13550 *
13551 * NOTE: All user context CSRs that are not mmaped write-only
13552 * (e.g. the TID flows) must be initialized even if the driver never
13553 * reads them.
13554 */
write_uninitialized_csrs_and_memories(struct hfi1_devdata * dd)13555 static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd)
13556 {
13557 int i, j;
13558
13559 /* CceIntMap */
13560 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13561 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
13562
13563 /* SendCtxtCreditReturnAddr */
13564 for (i = 0; i < chip_send_contexts(dd); i++)
13565 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13566
13567 /* PIO Send buffers */
13568 /* SDMA Send buffers */
13569 /*
13570 * These are not normally read, and (presently) have no method
13571 * to be read, so are not pre-initialized
13572 */
13573
13574 /* RcvHdrAddr */
13575 /* RcvHdrTailAddr */
13576 /* RcvTidFlowTable */
13577 for (i = 0; i < chip_rcv_contexts(dd); i++) {
13578 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13579 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13580 for (j = 0; j < RXE_NUM_TID_FLOWS; j++)
13581 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j), 0);
13582 }
13583
13584 /* RcvArray */
13585 for (i = 0; i < chip_rcv_array_count(dd); i++)
13586 hfi1_put_tid(dd, i, PT_INVALID_FLUSH, 0, 0);
13587
13588 /* RcvQPMapTable */
13589 for (i = 0; i < 32; i++)
13590 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13591 }
13592
13593 /*
13594 * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus.
13595 */
clear_cce_status(struct hfi1_devdata * dd,u64 status_bits,u64 ctrl_bits)13596 static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits,
13597 u64 ctrl_bits)
13598 {
13599 unsigned long timeout;
13600 u64 reg;
13601
13602 /* is the condition present? */
13603 reg = read_csr(dd, CCE_STATUS);
13604 if ((reg & status_bits) == 0)
13605 return;
13606
13607 /* clear the condition */
13608 write_csr(dd, CCE_CTRL, ctrl_bits);
13609
13610 /* wait for the condition to clear */
13611 timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT);
13612 while (1) {
13613 reg = read_csr(dd, CCE_STATUS);
13614 if ((reg & status_bits) == 0)
13615 return;
13616 if (time_after(jiffies, timeout)) {
13617 dd_dev_err(dd,
13618 "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n",
13619 status_bits, reg & status_bits);
13620 return;
13621 }
13622 udelay(1);
13623 }
13624 }
13625
13626 /* set CCE CSRs to chip reset defaults */
reset_cce_csrs(struct hfi1_devdata * dd)13627 static void reset_cce_csrs(struct hfi1_devdata *dd)
13628 {
13629 int i;
13630
13631 /* CCE_REVISION read-only */
13632 /* CCE_REVISION2 read-only */
13633 /* CCE_CTRL - bits clear automatically */
13634 /* CCE_STATUS read-only, use CceCtrl to clear */
13635 clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK);
13636 clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK);
13637 clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK);
13638 for (i = 0; i < CCE_NUM_SCRATCH; i++)
13639 write_csr(dd, CCE_SCRATCH + (8 * i), 0);
13640 /* CCE_ERR_STATUS read-only */
13641 write_csr(dd, CCE_ERR_MASK, 0);
13642 write_csr(dd, CCE_ERR_CLEAR, ~0ull);
13643 /* CCE_ERR_FORCE leave alone */
13644 for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++)
13645 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0);
13646 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR);
13647 /* CCE_PCIE_CTRL leave alone */
13648 for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) {
13649 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0);
13650 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i),
13651 CCE_MSIX_TABLE_UPPER_RESETCSR);
13652 }
13653 for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) {
13654 /* CCE_MSIX_PBA read-only */
13655 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull);
13656 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull);
13657 }
13658 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13659 write_csr(dd, CCE_INT_MAP, 0);
13660 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
13661 /* CCE_INT_STATUS read-only */
13662 write_csr(dd, CCE_INT_MASK + (8 * i), 0);
13663 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull);
13664 /* CCE_INT_FORCE leave alone */
13665 /* CCE_INT_BLOCKED read-only */
13666 }
13667 for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++)
13668 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0);
13669 }
13670
13671 /* set MISC CSRs to chip reset defaults */
reset_misc_csrs(struct hfi1_devdata * dd)13672 static void reset_misc_csrs(struct hfi1_devdata *dd)
13673 {
13674 int i;
13675
13676 for (i = 0; i < 32; i++) {
13677 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0);
13678 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0);
13679 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0);
13680 }
13681 /*
13682 * MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can
13683 * only be written 128-byte chunks
13684 */
13685 /* init RSA engine to clear lingering errors */
13686 write_csr(dd, MISC_CFG_RSA_CMD, 1);
13687 write_csr(dd, MISC_CFG_RSA_MU, 0);
13688 write_csr(dd, MISC_CFG_FW_CTRL, 0);
13689 /* MISC_STS_8051_DIGEST read-only */
13690 /* MISC_STS_SBM_DIGEST read-only */
13691 /* MISC_STS_PCIE_DIGEST read-only */
13692 /* MISC_STS_FAB_DIGEST read-only */
13693 /* MISC_ERR_STATUS read-only */
13694 write_csr(dd, MISC_ERR_MASK, 0);
13695 write_csr(dd, MISC_ERR_CLEAR, ~0ull);
13696 /* MISC_ERR_FORCE leave alone */
13697 }
13698
13699 /* set TXE CSRs to chip reset defaults */
reset_txe_csrs(struct hfi1_devdata * dd)13700 static void reset_txe_csrs(struct hfi1_devdata *dd)
13701 {
13702 int i;
13703
13704 /*
13705 * TXE Kernel CSRs
13706 */
13707 write_csr(dd, SEND_CTRL, 0);
13708 __cm_reset(dd, 0); /* reset CM internal state */
13709 /* SEND_CONTEXTS read-only */
13710 /* SEND_DMA_ENGINES read-only */
13711 /* SEND_PIO_MEM_SIZE read-only */
13712 /* SEND_DMA_MEM_SIZE read-only */
13713 write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0);
13714 pio_reset_all(dd); /* SEND_PIO_INIT_CTXT */
13715 /* SEND_PIO_ERR_STATUS read-only */
13716 write_csr(dd, SEND_PIO_ERR_MASK, 0);
13717 write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull);
13718 /* SEND_PIO_ERR_FORCE leave alone */
13719 /* SEND_DMA_ERR_STATUS read-only */
13720 write_csr(dd, SEND_DMA_ERR_MASK, 0);
13721 write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull);
13722 /* SEND_DMA_ERR_FORCE leave alone */
13723 /* SEND_EGRESS_ERR_STATUS read-only */
13724 write_csr(dd, SEND_EGRESS_ERR_MASK, 0);
13725 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull);
13726 /* SEND_EGRESS_ERR_FORCE leave alone */
13727 write_csr(dd, SEND_BTH_QP, 0);
13728 write_csr(dd, SEND_STATIC_RATE_CONTROL, 0);
13729 write_csr(dd, SEND_SC2VLT0, 0);
13730 write_csr(dd, SEND_SC2VLT1, 0);
13731 write_csr(dd, SEND_SC2VLT2, 0);
13732 write_csr(dd, SEND_SC2VLT3, 0);
13733 write_csr(dd, SEND_LEN_CHECK0, 0);
13734 write_csr(dd, SEND_LEN_CHECK1, 0);
13735 /* SEND_ERR_STATUS read-only */
13736 write_csr(dd, SEND_ERR_MASK, 0);
13737 write_csr(dd, SEND_ERR_CLEAR, ~0ull);
13738 /* SEND_ERR_FORCE read-only */
13739 for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++)
13740 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8 * i), 0);
13741 for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++)
13742 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8 * i), 0);
13743 for (i = 0; i < chip_send_contexts(dd) / NUM_CONTEXTS_PER_SET; i++)
13744 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8 * i), 0);
13745 for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++)
13746 write_csr(dd, SEND_COUNTER_ARRAY32 + (8 * i), 0);
13747 for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++)
13748 write_csr(dd, SEND_COUNTER_ARRAY64 + (8 * i), 0);
13749 write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR);
13750 write_csr(dd, SEND_CM_GLOBAL_CREDIT, SEND_CM_GLOBAL_CREDIT_RESETCSR);
13751 /* SEND_CM_CREDIT_USED_STATUS read-only */
13752 write_csr(dd, SEND_CM_TIMER_CTRL, 0);
13753 write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0);
13754 write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0);
13755 write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0);
13756 write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0);
13757 for (i = 0; i < TXE_NUM_DATA_VL; i++)
13758 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
13759 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
13760 /* SEND_CM_CREDIT_USED_VL read-only */
13761 /* SEND_CM_CREDIT_USED_VL15 read-only */
13762 /* SEND_EGRESS_CTXT_STATUS read-only */
13763 /* SEND_EGRESS_SEND_DMA_STATUS read-only */
13764 write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull);
13765 /* SEND_EGRESS_ERR_INFO read-only */
13766 /* SEND_EGRESS_ERR_SOURCE read-only */
13767
13768 /*
13769 * TXE Per-Context CSRs
13770 */
13771 for (i = 0; i < chip_send_contexts(dd); i++) {
13772 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13773 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0);
13774 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13775 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0);
13776 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0);
13777 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull);
13778 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0);
13779 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0);
13780 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0);
13781 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0);
13782 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0);
13783 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0);
13784 }
13785
13786 /*
13787 * TXE Per-SDMA CSRs
13788 */
13789 for (i = 0; i < chip_sdma_engines(dd); i++) {
13790 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13791 /* SEND_DMA_STATUS read-only */
13792 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0);
13793 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0);
13794 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0);
13795 /* SEND_DMA_HEAD read-only */
13796 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0);
13797 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0);
13798 /* SEND_DMA_IDLE_CNT read-only */
13799 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0);
13800 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0);
13801 /* SEND_DMA_DESC_FETCHED_CNT read-only */
13802 /* SEND_DMA_ENG_ERR_STATUS read-only */
13803 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0);
13804 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull);
13805 /* SEND_DMA_ENG_ERR_FORCE leave alone */
13806 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0);
13807 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0);
13808 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0);
13809 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0);
13810 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0);
13811 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0);
13812 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0);
13813 }
13814 }
13815
13816 /*
13817 * Expect on entry:
13818 * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0
13819 */
init_rbufs(struct hfi1_devdata * dd)13820 static void init_rbufs(struct hfi1_devdata *dd)
13821 {
13822 u64 reg;
13823 int count;
13824
13825 /*
13826 * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are
13827 * clear.
13828 */
13829 count = 0;
13830 while (1) {
13831 reg = read_csr(dd, RCV_STATUS);
13832 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK
13833 | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0)
13834 break;
13835 /*
13836 * Give up after 1ms - maximum wait time.
13837 *
13838 * RBuf size is 136KiB. Slowest possible is PCIe Gen1 x1 at
13839 * 250MB/s bandwidth. Lower rate to 66% for overhead to get:
13840 * 136 KB / (66% * 250MB/s) = 844us
13841 */
13842 if (count++ > 500) {
13843 dd_dev_err(dd,
13844 "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n",
13845 __func__, reg);
13846 break;
13847 }
13848 udelay(2); /* do not busy-wait the CSR */
13849 }
13850
13851 /* start the init - expect RcvCtrl to be 0 */
13852 write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK);
13853
13854 /*
13855 * Read to force the write of Rcvtrl.RxRbufInit. There is a brief
13856 * period after the write before RcvStatus.RxRbufInitDone is valid.
13857 * The delay in the first run through the loop below is sufficient and
13858 * required before the first read of RcvStatus.RxRbufInintDone.
13859 */
13860 read_csr(dd, RCV_CTRL);
13861
13862 /* wait for the init to finish */
13863 count = 0;
13864 while (1) {
13865 /* delay is required first time through - see above */
13866 udelay(2); /* do not busy-wait the CSR */
13867 reg = read_csr(dd, RCV_STATUS);
13868 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK))
13869 break;
13870
13871 /* give up after 100us - slowest possible at 33MHz is 73us */
13872 if (count++ > 50) {
13873 dd_dev_err(dd,
13874 "%s: RcvStatus.RxRbufInit not set, continuing\n",
13875 __func__);
13876 break;
13877 }
13878 }
13879 }
13880
13881 /* set RXE CSRs to chip reset defaults */
reset_rxe_csrs(struct hfi1_devdata * dd)13882 static void reset_rxe_csrs(struct hfi1_devdata *dd)
13883 {
13884 int i, j;
13885
13886 /*
13887 * RXE Kernel CSRs
13888 */
13889 write_csr(dd, RCV_CTRL, 0);
13890 init_rbufs(dd);
13891 /* RCV_STATUS read-only */
13892 /* RCV_CONTEXTS read-only */
13893 /* RCV_ARRAY_CNT read-only */
13894 /* RCV_BUF_SIZE read-only */
13895 write_csr(dd, RCV_BTH_QP, 0);
13896 write_csr(dd, RCV_MULTICAST, 0);
13897 write_csr(dd, RCV_BYPASS, 0);
13898 write_csr(dd, RCV_VL15, 0);
13899 /* this is a clear-down */
13900 write_csr(dd, RCV_ERR_INFO,
13901 RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK);
13902 /* RCV_ERR_STATUS read-only */
13903 write_csr(dd, RCV_ERR_MASK, 0);
13904 write_csr(dd, RCV_ERR_CLEAR, ~0ull);
13905 /* RCV_ERR_FORCE leave alone */
13906 for (i = 0; i < 32; i++)
13907 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13908 for (i = 0; i < 4; i++)
13909 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0);
13910 for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++)
13911 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0);
13912 for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++)
13913 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0);
13914 for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++)
13915 clear_rsm_rule(dd, i);
13916 for (i = 0; i < 32; i++)
13917 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0);
13918
13919 /*
13920 * RXE Kernel and User Per-Context CSRs
13921 */
13922 for (i = 0; i < chip_rcv_contexts(dd); i++) {
13923 /* kernel */
13924 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0);
13925 /* RCV_CTXT_STATUS read-only */
13926 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0);
13927 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0);
13928 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0);
13929 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13930 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0);
13931 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0);
13932 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0);
13933 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13934 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0);
13935 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0);
13936
13937 /* user */
13938 /* RCV_HDR_TAIL read-only */
13939 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0);
13940 /* RCV_EGR_INDEX_TAIL read-only */
13941 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0);
13942 /* RCV_EGR_OFFSET_TAIL read-only */
13943 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) {
13944 write_uctxt_csr(dd, i,
13945 RCV_TID_FLOW_TABLE + (8 * j), 0);
13946 }
13947 }
13948 }
13949
13950 /*
13951 * Set sc2vl tables.
13952 *
13953 * They power on to zeros, so to avoid send context errors
13954 * they need to be set:
13955 *
13956 * SC 0-7 -> VL 0-7 (respectively)
13957 * SC 15 -> VL 15
13958 * otherwise
13959 * -> VL 0
13960 */
init_sc2vl_tables(struct hfi1_devdata * dd)13961 static void init_sc2vl_tables(struct hfi1_devdata *dd)
13962 {
13963 int i;
13964 /* init per architecture spec, constrained by hardware capability */
13965
13966 /* HFI maps sent packets */
13967 write_csr(dd, SEND_SC2VLT0, SC2VL_VAL(
13968 0,
13969 0, 0, 1, 1,
13970 2, 2, 3, 3,
13971 4, 4, 5, 5,
13972 6, 6, 7, 7));
13973 write_csr(dd, SEND_SC2VLT1, SC2VL_VAL(
13974 1,
13975 8, 0, 9, 0,
13976 10, 0, 11, 0,
13977 12, 0, 13, 0,
13978 14, 0, 15, 15));
13979 write_csr(dd, SEND_SC2VLT2, SC2VL_VAL(
13980 2,
13981 16, 0, 17, 0,
13982 18, 0, 19, 0,
13983 20, 0, 21, 0,
13984 22, 0, 23, 0));
13985 write_csr(dd, SEND_SC2VLT3, SC2VL_VAL(
13986 3,
13987 24, 0, 25, 0,
13988 26, 0, 27, 0,
13989 28, 0, 29, 0,
13990 30, 0, 31, 0));
13991
13992 /* DC maps received packets */
13993 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL(
13994 15_0,
13995 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
13996 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15));
13997 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL(
13998 31_16,
13999 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0,
14000 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0));
14001
14002 /* initialize the cached sc2vl values consistently with h/w */
14003 for (i = 0; i < 32; i++) {
14004 if (i < 8 || i == 15)
14005 *((u8 *)(dd->sc2vl) + i) = (u8)i;
14006 else
14007 *((u8 *)(dd->sc2vl) + i) = 0;
14008 }
14009 }
14010
14011 /*
14012 * Read chip sizes and then reset parts to sane, disabled, values. We cannot
14013 * depend on the chip going through a power-on reset - a driver may be loaded
14014 * and unloaded many times.
14015 *
14016 * Do not write any CSR values to the chip in this routine - there may be
14017 * a reset following the (possible) FLR in this routine.
14018 *
14019 */
init_chip(struct hfi1_devdata * dd)14020 static int init_chip(struct hfi1_devdata *dd)
14021 {
14022 int i;
14023 int ret = 0;
14024
14025 /*
14026 * Put the HFI CSRs in a known state.
14027 * Combine this with a DC reset.
14028 *
14029 * Stop the device from doing anything while we do a
14030 * reset. We know there are no other active users of
14031 * the device since we are now in charge. Turn off
14032 * off all outbound and inbound traffic and make sure
14033 * the device does not generate any interrupts.
14034 */
14035
14036 /* disable send contexts and SDMA engines */
14037 write_csr(dd, SEND_CTRL, 0);
14038 for (i = 0; i < chip_send_contexts(dd); i++)
14039 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
14040 for (i = 0; i < chip_sdma_engines(dd); i++)
14041 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
14042 /* disable port (turn off RXE inbound traffic) and contexts */
14043 write_csr(dd, RCV_CTRL, 0);
14044 for (i = 0; i < chip_rcv_contexts(dd); i++)
14045 write_csr(dd, RCV_CTXT_CTRL, 0);
14046 /* mask all interrupt sources */
14047 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
14048 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull);
14049
14050 /*
14051 * DC Reset: do a full DC reset before the register clear.
14052 * A recommended length of time to hold is one CSR read,
14053 * so reread the CceDcCtrl. Then, hold the DC in reset
14054 * across the clear.
14055 */
14056 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);
14057 (void)read_csr(dd, CCE_DC_CTRL);
14058
14059 if (use_flr) {
14060 /*
14061 * A FLR will reset the SPC core and part of the PCIe.
14062 * The parts that need to be restored have already been
14063 * saved.
14064 */
14065 dd_dev_info(dd, "Resetting CSRs with FLR\n");
14066
14067 /* do the FLR, the DC reset will remain */
14068 pcie_flr(dd->pcidev);
14069
14070 /* restore command and BARs */
14071 ret = restore_pci_variables(dd);
14072 if (ret) {
14073 dd_dev_err(dd, "%s: Could not restore PCI variables\n",
14074 __func__);
14075 return ret;
14076 }
14077
14078 if (is_ax(dd)) {
14079 dd_dev_info(dd, "Resetting CSRs with FLR\n");
14080 pcie_flr(dd->pcidev);
14081 ret = restore_pci_variables(dd);
14082 if (ret) {
14083 dd_dev_err(dd, "%s: Could not restore PCI variables\n",
14084 __func__);
14085 return ret;
14086 }
14087 }
14088 } else {
14089 dd_dev_info(dd, "Resetting CSRs with writes\n");
14090 reset_cce_csrs(dd);
14091 reset_txe_csrs(dd);
14092 reset_rxe_csrs(dd);
14093 reset_misc_csrs(dd);
14094 }
14095 /* clear the DC reset */
14096 write_csr(dd, CCE_DC_CTRL, 0);
14097
14098 /* Set the LED off */
14099 setextled(dd, 0);
14100
14101 /*
14102 * Clear the QSFP reset.
14103 * An FLR enforces a 0 on all out pins. The driver does not touch
14104 * ASIC_QSFPn_OUT otherwise. This leaves RESET_N low and
14105 * anything plugged constantly in reset, if it pays attention
14106 * to RESET_N.
14107 * Prime examples of this are optical cables. Set all pins high.
14108 * I2CCLK and I2CDAT will change per direction, and INT_N and
14109 * MODPRS_N are input only and their value is ignored.
14110 */
14111 write_csr(dd, ASIC_QSFP1_OUT, 0x1f);
14112 write_csr(dd, ASIC_QSFP2_OUT, 0x1f);
14113 init_chip_resources(dd);
14114 return ret;
14115 }
14116
init_early_variables(struct hfi1_devdata * dd)14117 static void init_early_variables(struct hfi1_devdata *dd)
14118 {
14119 int i;
14120
14121 /* assign link credit variables */
14122 dd->vau = CM_VAU;
14123 dd->link_credits = CM_GLOBAL_CREDITS;
14124 if (is_ax(dd))
14125 dd->link_credits--;
14126 dd->vcu = cu_to_vcu(hfi1_cu);
14127 /* enough room for 8 MAD packets plus header - 17K */
14128 dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau);
14129 if (dd->vl15_init > dd->link_credits)
14130 dd->vl15_init = dd->link_credits;
14131
14132 write_uninitialized_csrs_and_memories(dd);
14133
14134 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
14135 for (i = 0; i < dd->num_pports; i++) {
14136 struct hfi1_pportdata *ppd = &dd->pport[i];
14137
14138 set_partition_keys(ppd);
14139 }
14140 init_sc2vl_tables(dd);
14141 }
14142
init_kdeth_qp(struct hfi1_devdata * dd)14143 static void init_kdeth_qp(struct hfi1_devdata *dd)
14144 {
14145 write_csr(dd, SEND_BTH_QP,
14146 (RVT_KDETH_QP_PREFIX & SEND_BTH_QP_KDETH_QP_MASK) <<
14147 SEND_BTH_QP_KDETH_QP_SHIFT);
14148
14149 write_csr(dd, RCV_BTH_QP,
14150 (RVT_KDETH_QP_PREFIX & RCV_BTH_QP_KDETH_QP_MASK) <<
14151 RCV_BTH_QP_KDETH_QP_SHIFT);
14152 }
14153
14154 /**
14155 * hfi1_get_qp_map - get qp map
14156 * @dd: device data
14157 * @idx: index to read
14158 */
hfi1_get_qp_map(struct hfi1_devdata * dd,u8 idx)14159 u8 hfi1_get_qp_map(struct hfi1_devdata *dd, u8 idx)
14160 {
14161 u64 reg = read_csr(dd, RCV_QP_MAP_TABLE + (idx / 8) * 8);
14162
14163 reg >>= (idx % 8) * 8;
14164 return reg;
14165 }
14166
14167 /**
14168 * init_qpmap_table - init qp map
14169 * @dd: device data
14170 * @first_ctxt: first context
14171 * @last_ctxt: first context
14172 *
14173 * This return sets the qpn mapping table that
14174 * is indexed by qpn[8:1].
14175 *
14176 * The routine will round robin the 256 settings
14177 * from first_ctxt to last_ctxt.
14178 *
14179 * The first/last looks ahead to having specialized
14180 * receive contexts for mgmt and bypass. Normal
14181 * verbs traffic will assumed to be on a range
14182 * of receive contexts.
14183 */
init_qpmap_table(struct hfi1_devdata * dd,u32 first_ctxt,u32 last_ctxt)14184 static void init_qpmap_table(struct hfi1_devdata *dd,
14185 u32 first_ctxt,
14186 u32 last_ctxt)
14187 {
14188 u64 reg = 0;
14189 u64 regno = RCV_QP_MAP_TABLE;
14190 int i;
14191 u64 ctxt = first_ctxt;
14192
14193 for (i = 0; i < 256; i++) {
14194 reg |= ctxt << (8 * (i % 8));
14195 ctxt++;
14196 if (ctxt > last_ctxt)
14197 ctxt = first_ctxt;
14198 if (i % 8 == 7) {
14199 write_csr(dd, regno, reg);
14200 reg = 0;
14201 regno += 8;
14202 }
14203 }
14204
14205 add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK
14206 | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK);
14207 }
14208
14209 struct rsm_map_table {
14210 u64 map[NUM_MAP_REGS];
14211 unsigned int used;
14212 };
14213
14214 struct rsm_rule_data {
14215 u8 offset;
14216 u8 pkt_type;
14217 u32 field1_off;
14218 u32 field2_off;
14219 u32 index1_off;
14220 u32 index1_width;
14221 u32 index2_off;
14222 u32 index2_width;
14223 u32 mask1;
14224 u32 value1;
14225 u32 mask2;
14226 u32 value2;
14227 };
14228
14229 /*
14230 * Return an initialized RMT map table for users to fill in. OK if it
14231 * returns NULL, indicating no table.
14232 */
alloc_rsm_map_table(struct hfi1_devdata * dd)14233 static struct rsm_map_table *alloc_rsm_map_table(struct hfi1_devdata *dd)
14234 {
14235 struct rsm_map_table *rmt;
14236 u8 rxcontext = is_ax(dd) ? 0 : 0xff; /* 0 is default if a0 ver. */
14237
14238 rmt = kmalloc(sizeof(*rmt), GFP_KERNEL);
14239 if (rmt) {
14240 memset(rmt->map, rxcontext, sizeof(rmt->map));
14241 rmt->used = 0;
14242 }
14243
14244 return rmt;
14245 }
14246
14247 /*
14248 * Write the final RMT map table to the chip and free the table. OK if
14249 * table is NULL.
14250 */
complete_rsm_map_table(struct hfi1_devdata * dd,struct rsm_map_table * rmt)14251 static void complete_rsm_map_table(struct hfi1_devdata *dd,
14252 struct rsm_map_table *rmt)
14253 {
14254 int i;
14255
14256 if (rmt) {
14257 /* write table to chip */
14258 for (i = 0; i < NUM_MAP_REGS; i++)
14259 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rmt->map[i]);
14260
14261 /* enable RSM */
14262 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
14263 }
14264 }
14265
14266 /* Is a receive side mapping rule */
has_rsm_rule(struct hfi1_devdata * dd,u8 rule_index)14267 static bool has_rsm_rule(struct hfi1_devdata *dd, u8 rule_index)
14268 {
14269 return read_csr(dd, RCV_RSM_CFG + (8 * rule_index)) != 0;
14270 }
14271
14272 /*
14273 * Add a receive side mapping rule.
14274 */
add_rsm_rule(struct hfi1_devdata * dd,u8 rule_index,struct rsm_rule_data * rrd)14275 static void add_rsm_rule(struct hfi1_devdata *dd, u8 rule_index,
14276 struct rsm_rule_data *rrd)
14277 {
14278 write_csr(dd, RCV_RSM_CFG + (8 * rule_index),
14279 (u64)rrd->offset << RCV_RSM_CFG_OFFSET_SHIFT |
14280 1ull << rule_index | /* enable bit */
14281 (u64)rrd->pkt_type << RCV_RSM_CFG_PACKET_TYPE_SHIFT);
14282 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index),
14283 (u64)rrd->field1_off << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT |
14284 (u64)rrd->field2_off << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT |
14285 (u64)rrd->index1_off << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT |
14286 (u64)rrd->index1_width << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT |
14287 (u64)rrd->index2_off << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT |
14288 (u64)rrd->index2_width << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT);
14289 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index),
14290 (u64)rrd->mask1 << RCV_RSM_MATCH_MASK1_SHIFT |
14291 (u64)rrd->value1 << RCV_RSM_MATCH_VALUE1_SHIFT |
14292 (u64)rrd->mask2 << RCV_RSM_MATCH_MASK2_SHIFT |
14293 (u64)rrd->value2 << RCV_RSM_MATCH_VALUE2_SHIFT);
14294 }
14295
14296 /*
14297 * Clear a receive side mapping rule.
14298 */
clear_rsm_rule(struct hfi1_devdata * dd,u8 rule_index)14299 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index)
14300 {
14301 write_csr(dd, RCV_RSM_CFG + (8 * rule_index), 0);
14302 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index), 0);
14303 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index), 0);
14304 }
14305
14306 /* return the number of RSM map table entries that will be used for QOS */
qos_rmt_entries(unsigned int n_krcv_queues,unsigned int * mp,unsigned int * np)14307 static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp,
14308 unsigned int *np)
14309 {
14310 int i;
14311 unsigned int m, n;
14312 uint max_by_vl = 0;
14313
14314 /* is QOS active at all? */
14315 if (n_krcv_queues < MIN_KERNEL_KCTXTS ||
14316 num_vls == 1 ||
14317 krcvqsset <= 1)
14318 goto no_qos;
14319
14320 /* determine bits for qpn */
14321 for (i = 0; i < min_t(unsigned int, num_vls, krcvqsset); i++)
14322 if (krcvqs[i] > max_by_vl)
14323 max_by_vl = krcvqs[i];
14324 if (max_by_vl > 32)
14325 goto no_qos;
14326 m = ilog2(__roundup_pow_of_two(max_by_vl));
14327
14328 /* determine bits for vl */
14329 n = ilog2(__roundup_pow_of_two(num_vls));
14330
14331 /* reject if too much is used */
14332 if ((m + n) > 7)
14333 goto no_qos;
14334
14335 if (mp)
14336 *mp = m;
14337 if (np)
14338 *np = n;
14339
14340 return 1 << (m + n);
14341
14342 no_qos:
14343 if (mp)
14344 *mp = 0;
14345 if (np)
14346 *np = 0;
14347 return 0;
14348 }
14349
14350 /**
14351 * init_qos - init RX qos
14352 * @dd: device data
14353 * @rmt: RSM map table
14354 *
14355 * This routine initializes Rule 0 and the RSM map table to implement
14356 * quality of service (qos).
14357 *
14358 * If all of the limit tests succeed, qos is applied based on the array
14359 * interpretation of krcvqs where entry 0 is VL0.
14360 *
14361 * The number of vl bits (n) and the number of qpn bits (m) are computed to
14362 * feed both the RSM map table and the single rule.
14363 */
init_qos(struct hfi1_devdata * dd,struct rsm_map_table * rmt)14364 static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt)
14365 {
14366 struct rsm_rule_data rrd;
14367 unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m;
14368 unsigned int rmt_entries;
14369 u64 reg;
14370
14371 if (!rmt)
14372 goto bail;
14373 rmt_entries = qos_rmt_entries(dd->n_krcv_queues - 1, &m, &n);
14374 if (rmt_entries == 0)
14375 goto bail;
14376 qpns_per_vl = 1 << m;
14377
14378 /* enough room in the map table? */
14379 rmt_entries = 1 << (m + n);
14380 if (rmt->used + rmt_entries >= NUM_MAP_ENTRIES)
14381 goto bail;
14382
14383 /* add qos entries to the RSM map table */
14384 for (i = 0, ctxt = FIRST_KERNEL_KCTXT; i < num_vls; i++) {
14385 unsigned tctxt;
14386
14387 for (qpn = 0, tctxt = ctxt;
14388 krcvqs[i] && qpn < qpns_per_vl; qpn++) {
14389 unsigned idx, regoff, regidx;
14390
14391 /* generate the index the hardware will produce */
14392 idx = rmt->used + ((qpn << n) ^ i);
14393 regoff = (idx % 8) * 8;
14394 regidx = idx / 8;
14395 /* replace default with context number */
14396 reg = rmt->map[regidx];
14397 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK
14398 << regoff);
14399 reg |= (u64)(tctxt++) << regoff;
14400 rmt->map[regidx] = reg;
14401 if (tctxt == ctxt + krcvqs[i])
14402 tctxt = ctxt;
14403 }
14404 ctxt += krcvqs[i];
14405 }
14406
14407 rrd.offset = rmt->used;
14408 rrd.pkt_type = 2;
14409 rrd.field1_off = LRH_BTH_MATCH_OFFSET;
14410 rrd.field2_off = LRH_SC_MATCH_OFFSET;
14411 rrd.index1_off = LRH_SC_SELECT_OFFSET;
14412 rrd.index1_width = n;
14413 rrd.index2_off = QPN_SELECT_OFFSET;
14414 rrd.index2_width = m + n;
14415 rrd.mask1 = LRH_BTH_MASK;
14416 rrd.value1 = LRH_BTH_VALUE;
14417 rrd.mask2 = LRH_SC_MASK;
14418 rrd.value2 = LRH_SC_VALUE;
14419
14420 /* add rule 0 */
14421 add_rsm_rule(dd, RSM_INS_VERBS, &rrd);
14422
14423 /* mark RSM map entries as used */
14424 rmt->used += rmt_entries;
14425 /* map everything else to the mcast/err/vl15 context */
14426 init_qpmap_table(dd, HFI1_CTRL_CTXT, HFI1_CTRL_CTXT);
14427 dd->qos_shift = n + 1;
14428 return;
14429 bail:
14430 dd->qos_shift = 1;
14431 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1);
14432 }
14433
init_fecn_handling(struct hfi1_devdata * dd,struct rsm_map_table * rmt)14434 static void init_fecn_handling(struct hfi1_devdata *dd,
14435 struct rsm_map_table *rmt)
14436 {
14437 struct rsm_rule_data rrd;
14438 u64 reg;
14439 int i, idx, regoff, regidx, start;
14440 u8 offset;
14441 u32 total_cnt;
14442
14443 if (HFI1_CAP_IS_KSET(TID_RDMA))
14444 /* Exclude context 0 */
14445 start = 1;
14446 else
14447 start = dd->first_dyn_alloc_ctxt;
14448
14449 total_cnt = dd->num_rcv_contexts - start;
14450
14451 /* there needs to be enough room in the map table */
14452 if (rmt->used + total_cnt >= NUM_MAP_ENTRIES) {
14453 dd_dev_err(dd, "FECN handling disabled - too many contexts allocated\n");
14454 return;
14455 }
14456
14457 /*
14458 * RSM will extract the destination context as an index into the
14459 * map table. The destination contexts are a sequential block
14460 * in the range start...num_rcv_contexts-1 (inclusive).
14461 * Map entries are accessed as offset + extracted value. Adjust
14462 * the added offset so this sequence can be placed anywhere in
14463 * the table - as long as the entries themselves do not wrap.
14464 * There are only enough bits in offset for the table size, so
14465 * start with that to allow for a "negative" offset.
14466 */
14467 offset = (u8)(NUM_MAP_ENTRIES + rmt->used - start);
14468
14469 for (i = start, idx = rmt->used; i < dd->num_rcv_contexts;
14470 i++, idx++) {
14471 /* replace with identity mapping */
14472 regoff = (idx % 8) * 8;
14473 regidx = idx / 8;
14474 reg = rmt->map[regidx];
14475 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK << regoff);
14476 reg |= (u64)i << regoff;
14477 rmt->map[regidx] = reg;
14478 }
14479
14480 /*
14481 * For RSM intercept of Expected FECN packets:
14482 * o packet type 0 - expected
14483 * o match on F (bit 95), using select/match 1, and
14484 * o match on SH (bit 133), using select/match 2.
14485 *
14486 * Use index 1 to extract the 8-bit receive context from DestQP
14487 * (start at bit 64). Use that as the RSM map table index.
14488 */
14489 rrd.offset = offset;
14490 rrd.pkt_type = 0;
14491 rrd.field1_off = 95;
14492 rrd.field2_off = 133;
14493 rrd.index1_off = 64;
14494 rrd.index1_width = 8;
14495 rrd.index2_off = 0;
14496 rrd.index2_width = 0;
14497 rrd.mask1 = 1;
14498 rrd.value1 = 1;
14499 rrd.mask2 = 1;
14500 rrd.value2 = 1;
14501
14502 /* add rule 1 */
14503 add_rsm_rule(dd, RSM_INS_FECN, &rrd);
14504
14505 rmt->used += total_cnt;
14506 }
14507
hfi1_is_rmt_full(int start,int spare)14508 static inline bool hfi1_is_rmt_full(int start, int spare)
14509 {
14510 return (start + spare) > NUM_MAP_ENTRIES;
14511 }
14512
hfi1_netdev_update_rmt(struct hfi1_devdata * dd)14513 static bool hfi1_netdev_update_rmt(struct hfi1_devdata *dd)
14514 {
14515 u8 i, j;
14516 u8 ctx_id = 0;
14517 u64 reg;
14518 u32 regoff;
14519 int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);
14520 int ctxt_count = hfi1_netdev_ctxt_count(dd);
14521
14522 /* We already have contexts mapped in RMT */
14523 if (has_rsm_rule(dd, RSM_INS_VNIC) || has_rsm_rule(dd, RSM_INS_AIP)) {
14524 dd_dev_info(dd, "Contexts are already mapped in RMT\n");
14525 return true;
14526 }
14527
14528 if (hfi1_is_rmt_full(rmt_start, NUM_NETDEV_MAP_ENTRIES)) {
14529 dd_dev_err(dd, "Not enough RMT entries used = %d\n",
14530 rmt_start);
14531 return false;
14532 }
14533
14534 dev_dbg(&(dd)->pcidev->dev, "RMT start = %d, end %d\n",
14535 rmt_start,
14536 rmt_start + NUM_NETDEV_MAP_ENTRIES);
14537
14538 /* Update RSM mapping table, 32 regs, 256 entries - 1 ctx per byte */
14539 regoff = RCV_RSM_MAP_TABLE + (rmt_start / 8) * 8;
14540 reg = read_csr(dd, regoff);
14541 for (i = 0; i < NUM_NETDEV_MAP_ENTRIES; i++) {
14542 /* Update map register with netdev context */
14543 j = (rmt_start + i) % 8;
14544 reg &= ~(0xffllu << (j * 8));
14545 reg |= (u64)hfi1_netdev_get_ctxt(dd, ctx_id++)->ctxt << (j * 8);
14546 /* Wrap up netdev ctx index */
14547 ctx_id %= ctxt_count;
14548 /* Write back map register */
14549 if (j == 7 || ((i + 1) == NUM_NETDEV_MAP_ENTRIES)) {
14550 dev_dbg(&(dd)->pcidev->dev,
14551 "RMT[%d] =0x%llx\n",
14552 regoff - RCV_RSM_MAP_TABLE, reg);
14553
14554 write_csr(dd, regoff, reg);
14555 regoff += 8;
14556 if (i < (NUM_NETDEV_MAP_ENTRIES - 1))
14557 reg = read_csr(dd, regoff);
14558 }
14559 }
14560
14561 return true;
14562 }
14563
hfi1_enable_rsm_rule(struct hfi1_devdata * dd,int rule,struct rsm_rule_data * rrd)14564 static void hfi1_enable_rsm_rule(struct hfi1_devdata *dd,
14565 int rule, struct rsm_rule_data *rrd)
14566 {
14567 if (!hfi1_netdev_update_rmt(dd)) {
14568 dd_dev_err(dd, "Failed to update RMT for RSM%d rule\n", rule);
14569 return;
14570 }
14571
14572 add_rsm_rule(dd, rule, rrd);
14573 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
14574 }
14575
hfi1_init_aip_rsm(struct hfi1_devdata * dd)14576 void hfi1_init_aip_rsm(struct hfi1_devdata *dd)
14577 {
14578 /*
14579 * go through with the initialisation only if this rule actually doesn't
14580 * exist yet
14581 */
14582 if (atomic_fetch_inc(&dd->ipoib_rsm_usr_num) == 0) {
14583 int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);
14584 struct rsm_rule_data rrd = {
14585 .offset = rmt_start,
14586 .pkt_type = IB_PACKET_TYPE,
14587 .field1_off = LRH_BTH_MATCH_OFFSET,
14588 .mask1 = LRH_BTH_MASK,
14589 .value1 = LRH_BTH_VALUE,
14590 .field2_off = BTH_DESTQP_MATCH_OFFSET,
14591 .mask2 = BTH_DESTQP_MASK,
14592 .value2 = BTH_DESTQP_VALUE,
14593 .index1_off = DETH_AIP_SQPN_SELECT_OFFSET +
14594 ilog2(NUM_NETDEV_MAP_ENTRIES),
14595 .index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES),
14596 .index2_off = DETH_AIP_SQPN_SELECT_OFFSET,
14597 .index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES)
14598 };
14599
14600 hfi1_enable_rsm_rule(dd, RSM_INS_AIP, &rrd);
14601 }
14602 }
14603
14604 /* Initialize RSM for VNIC */
hfi1_init_vnic_rsm(struct hfi1_devdata * dd)14605 void hfi1_init_vnic_rsm(struct hfi1_devdata *dd)
14606 {
14607 int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);
14608 struct rsm_rule_data rrd = {
14609 /* Add rule for vnic */
14610 .offset = rmt_start,
14611 .pkt_type = 4,
14612 /* Match 16B packets */
14613 .field1_off = L2_TYPE_MATCH_OFFSET,
14614 .mask1 = L2_TYPE_MASK,
14615 .value1 = L2_16B_VALUE,
14616 /* Match ETH L4 packets */
14617 .field2_off = L4_TYPE_MATCH_OFFSET,
14618 .mask2 = L4_16B_TYPE_MASK,
14619 .value2 = L4_16B_ETH_VALUE,
14620 /* Calc context from veswid and entropy */
14621 .index1_off = L4_16B_HDR_VESWID_OFFSET,
14622 .index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES),
14623 .index2_off = L2_16B_ENTROPY_OFFSET,
14624 .index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES)
14625 };
14626
14627 hfi1_enable_rsm_rule(dd, RSM_INS_VNIC, &rrd);
14628 }
14629
hfi1_deinit_vnic_rsm(struct hfi1_devdata * dd)14630 void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd)
14631 {
14632 clear_rsm_rule(dd, RSM_INS_VNIC);
14633 }
14634
hfi1_deinit_aip_rsm(struct hfi1_devdata * dd)14635 void hfi1_deinit_aip_rsm(struct hfi1_devdata *dd)
14636 {
14637 /* only actually clear the rule if it's the last user asking to do so */
14638 if (atomic_fetch_add_unless(&dd->ipoib_rsm_usr_num, -1, 0) == 1)
14639 clear_rsm_rule(dd, RSM_INS_AIP);
14640 }
14641
init_rxe(struct hfi1_devdata * dd)14642 static int init_rxe(struct hfi1_devdata *dd)
14643 {
14644 struct rsm_map_table *rmt;
14645 u64 val;
14646
14647 /* enable all receive errors */
14648 write_csr(dd, RCV_ERR_MASK, ~0ull);
14649
14650 rmt = alloc_rsm_map_table(dd);
14651 if (!rmt)
14652 return -ENOMEM;
14653
14654 /* set up QOS, including the QPN map table */
14655 init_qos(dd, rmt);
14656 init_fecn_handling(dd, rmt);
14657 complete_rsm_map_table(dd, rmt);
14658 /* record number of used rsm map entries for netdev */
14659 hfi1_netdev_set_free_rmt_idx(dd, rmt->used);
14660 kfree(rmt);
14661
14662 /*
14663 * make sure RcvCtrl.RcvWcb <= PCIe Device Control
14664 * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config
14665 * space, PciCfgCap2.MaxPayloadSize in HFI). There is only one
14666 * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and
14667 * Max_PayLoad_Size set to its minimum of 128.
14668 *
14669 * Presently, RcvCtrl.RcvWcb is not modified from its default of 0
14670 * (64 bytes). Max_Payload_Size is possibly modified upward in
14671 * tune_pcie_caps() which is called after this routine.
14672 */
14673
14674 /* Have 16 bytes (4DW) of bypass header available in header queue */
14675 val = read_csr(dd, RCV_BYPASS);
14676 val &= ~RCV_BYPASS_HDR_SIZE_SMASK;
14677 val |= ((4ull & RCV_BYPASS_HDR_SIZE_MASK) <<
14678 RCV_BYPASS_HDR_SIZE_SHIFT);
14679 write_csr(dd, RCV_BYPASS, val);
14680 return 0;
14681 }
14682
init_other(struct hfi1_devdata * dd)14683 static void init_other(struct hfi1_devdata *dd)
14684 {
14685 /* enable all CCE errors */
14686 write_csr(dd, CCE_ERR_MASK, ~0ull);
14687 /* enable *some* Misc errors */
14688 write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK);
14689 /* enable all DC errors, except LCB */
14690 write_csr(dd, DCC_ERR_FLG_EN, ~0ull);
14691 write_csr(dd, DC_DC8051_ERR_EN, ~0ull);
14692 }
14693
14694 /*
14695 * Fill out the given AU table using the given CU. A CU is defined in terms
14696 * AUs. The table is a an encoding: given the index, how many AUs does that
14697 * represent?
14698 *
14699 * NOTE: Assumes that the register layout is the same for the
14700 * local and remote tables.
14701 */
assign_cm_au_table(struct hfi1_devdata * dd,u32 cu,u32 csr0to3,u32 csr4to7)14702 static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu,
14703 u32 csr0to3, u32 csr4to7)
14704 {
14705 write_csr(dd, csr0to3,
14706 0ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT |
14707 1ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT |
14708 2ull * cu <<
14709 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT |
14710 4ull * cu <<
14711 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT);
14712 write_csr(dd, csr4to7,
14713 8ull * cu <<
14714 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT |
14715 16ull * cu <<
14716 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT |
14717 32ull * cu <<
14718 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT |
14719 64ull * cu <<
14720 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT);
14721 }
14722
assign_local_cm_au_table(struct hfi1_devdata * dd,u8 vcu)14723 static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
14724 {
14725 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3,
14726 SEND_CM_LOCAL_AU_TABLE4_TO7);
14727 }
14728
assign_remote_cm_au_table(struct hfi1_devdata * dd,u8 vcu)14729 void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
14730 {
14731 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3,
14732 SEND_CM_REMOTE_AU_TABLE4_TO7);
14733 }
14734
init_txe(struct hfi1_devdata * dd)14735 static void init_txe(struct hfi1_devdata *dd)
14736 {
14737 int i;
14738
14739 /* enable all PIO, SDMA, general, and Egress errors */
14740 write_csr(dd, SEND_PIO_ERR_MASK, ~0ull);
14741 write_csr(dd, SEND_DMA_ERR_MASK, ~0ull);
14742 write_csr(dd, SEND_ERR_MASK, ~0ull);
14743 write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull);
14744
14745 /* enable all per-context and per-SDMA engine errors */
14746 for (i = 0; i < chip_send_contexts(dd); i++)
14747 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull);
14748 for (i = 0; i < chip_sdma_engines(dd); i++)
14749 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull);
14750
14751 /* set the local CU to AU mapping */
14752 assign_local_cm_au_table(dd, dd->vcu);
14753
14754 /*
14755 * Set reasonable default for Credit Return Timer
14756 * Don't set on Simulator - causes it to choke.
14757 */
14758 if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)
14759 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE);
14760 }
14761
hfi1_set_ctxt_jkey(struct hfi1_devdata * dd,struct hfi1_ctxtdata * rcd,u16 jkey)14762 int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd,
14763 u16 jkey)
14764 {
14765 u8 hw_ctxt;
14766 u64 reg;
14767
14768 if (!rcd || !rcd->sc)
14769 return -EINVAL;
14770
14771 hw_ctxt = rcd->sc->hw_context;
14772 reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */
14773 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) <<
14774 SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT);
14775 /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */
14776 if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY))
14777 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK;
14778 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, reg);
14779 /*
14780 * Enable send-side J_KEY integrity check, unless this is A0 h/w
14781 */
14782 if (!is_ax(dd)) {
14783 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14784 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
14785 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14786 }
14787
14788 /* Enable J_KEY check on receive context. */
14789 reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK |
14790 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) <<
14791 RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT);
14792 write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, reg);
14793
14794 return 0;
14795 }
14796
hfi1_clear_ctxt_jkey(struct hfi1_devdata * dd,struct hfi1_ctxtdata * rcd)14797 int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
14798 {
14799 u8 hw_ctxt;
14800 u64 reg;
14801
14802 if (!rcd || !rcd->sc)
14803 return -EINVAL;
14804
14805 hw_ctxt = rcd->sc->hw_context;
14806 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, 0);
14807 /*
14808 * Disable send-side J_KEY integrity check, unless this is A0 h/w.
14809 * This check would not have been enabled for A0 h/w, see
14810 * set_ctxt_jkey().
14811 */
14812 if (!is_ax(dd)) {
14813 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14814 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
14815 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14816 }
14817 /* Turn off the J_KEY on the receive side */
14818 write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, 0);
14819
14820 return 0;
14821 }
14822
hfi1_set_ctxt_pkey(struct hfi1_devdata * dd,struct hfi1_ctxtdata * rcd,u16 pkey)14823 int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd,
14824 u16 pkey)
14825 {
14826 u8 hw_ctxt;
14827 u64 reg;
14828
14829 if (!rcd || !rcd->sc)
14830 return -EINVAL;
14831
14832 hw_ctxt = rcd->sc->hw_context;
14833 reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) <<
14834 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT;
14835 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg);
14836 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14837 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14838 reg &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK;
14839 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14840
14841 return 0;
14842 }
14843
hfi1_clear_ctxt_pkey(struct hfi1_devdata * dd,struct hfi1_ctxtdata * ctxt)14844 int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt)
14845 {
14846 u8 hw_ctxt;
14847 u64 reg;
14848
14849 if (!ctxt || !ctxt->sc)
14850 return -EINVAL;
14851
14852 hw_ctxt = ctxt->sc->hw_context;
14853 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14854 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14855 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14856 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0);
14857
14858 return 0;
14859 }
14860
14861 /*
14862 * Start doing the clean up the chip. Our clean up happens in multiple
14863 * stages and this is just the first.
14864 */
hfi1_start_cleanup(struct hfi1_devdata * dd)14865 void hfi1_start_cleanup(struct hfi1_devdata *dd)
14866 {
14867 aspm_exit(dd);
14868 free_cntrs(dd);
14869 free_rcverr(dd);
14870 finish_chip_resources(dd);
14871 }
14872
14873 #define HFI_BASE_GUID(dev) \
14874 ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT))
14875
14876 /*
14877 * Information can be shared between the two HFIs on the same ASIC
14878 * in the same OS. This function finds the peer device and sets
14879 * up a shared structure.
14880 */
init_asic_data(struct hfi1_devdata * dd)14881 static int init_asic_data(struct hfi1_devdata *dd)
14882 {
14883 unsigned long index;
14884 struct hfi1_devdata *peer;
14885 struct hfi1_asic_data *asic_data;
14886 int ret = 0;
14887
14888 /* pre-allocate the asic structure in case we are the first device */
14889 asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL);
14890 if (!asic_data)
14891 return -ENOMEM;
14892
14893 xa_lock_irq(&hfi1_dev_table);
14894 /* Find our peer device */
14895 xa_for_each(&hfi1_dev_table, index, peer) {
14896 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(peer)) &&
14897 dd->unit != peer->unit)
14898 break;
14899 }
14900
14901 if (peer) {
14902 /* use already allocated structure */
14903 dd->asic_data = peer->asic_data;
14904 kfree(asic_data);
14905 } else {
14906 dd->asic_data = asic_data;
14907 mutex_init(&dd->asic_data->asic_resource_mutex);
14908 }
14909 dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */
14910 xa_unlock_irq(&hfi1_dev_table);
14911
14912 /* first one through - set up i2c devices */
14913 if (!peer)
14914 ret = set_up_i2c(dd, dd->asic_data);
14915
14916 return ret;
14917 }
14918
14919 /*
14920 * Set dd->boardname. Use a generic name if a name is not returned from
14921 * EFI variable space.
14922 *
14923 * Return 0 on success, -ENOMEM if space could not be allocated.
14924 */
obtain_boardname(struct hfi1_devdata * dd)14925 static int obtain_boardname(struct hfi1_devdata *dd)
14926 {
14927 /* generic board description */
14928 const char generic[] =
14929 "Intel Omni-Path Host Fabric Interface Adapter 100 Series";
14930 unsigned long size;
14931 int ret;
14932
14933 ret = read_hfi1_efi_var(dd, "description", &size,
14934 (void **)&dd->boardname);
14935 if (ret) {
14936 dd_dev_info(dd, "Board description not found\n");
14937 /* use generic description */
14938 dd->boardname = kstrdup(generic, GFP_KERNEL);
14939 if (!dd->boardname)
14940 return -ENOMEM;
14941 }
14942 return 0;
14943 }
14944
14945 /*
14946 * Check the interrupt registers to make sure that they are mapped correctly.
14947 * It is intended to help user identify any mismapping by VMM when the driver
14948 * is running in a VM. This function should only be called before interrupt
14949 * is set up properly.
14950 *
14951 * Return 0 on success, -EINVAL on failure.
14952 */
check_int_registers(struct hfi1_devdata * dd)14953 static int check_int_registers(struct hfi1_devdata *dd)
14954 {
14955 u64 reg;
14956 u64 all_bits = ~(u64)0;
14957 u64 mask;
14958
14959 /* Clear CceIntMask[0] to avoid raising any interrupts */
14960 mask = read_csr(dd, CCE_INT_MASK);
14961 write_csr(dd, CCE_INT_MASK, 0ull);
14962 reg = read_csr(dd, CCE_INT_MASK);
14963 if (reg)
14964 goto err_exit;
14965
14966 /* Clear all interrupt status bits */
14967 write_csr(dd, CCE_INT_CLEAR, all_bits);
14968 reg = read_csr(dd, CCE_INT_STATUS);
14969 if (reg)
14970 goto err_exit;
14971
14972 /* Set all interrupt status bits */
14973 write_csr(dd, CCE_INT_FORCE, all_bits);
14974 reg = read_csr(dd, CCE_INT_STATUS);
14975 if (reg != all_bits)
14976 goto err_exit;
14977
14978 /* Restore the interrupt mask */
14979 write_csr(dd, CCE_INT_CLEAR, all_bits);
14980 write_csr(dd, CCE_INT_MASK, mask);
14981
14982 return 0;
14983 err_exit:
14984 write_csr(dd, CCE_INT_MASK, mask);
14985 dd_dev_err(dd, "Interrupt registers not properly mapped by VMM\n");
14986 return -EINVAL;
14987 }
14988
14989 /**
14990 * hfi1_init_dd() - Initialize most of the dd structure.
14991 * @dd: the dd device
14992 *
14993 * This is global, and is called directly at init to set up the
14994 * chip-specific function pointers for later use.
14995 */
hfi1_init_dd(struct hfi1_devdata * dd)14996 int hfi1_init_dd(struct hfi1_devdata *dd)
14997 {
14998 struct pci_dev *pdev = dd->pcidev;
14999 struct hfi1_pportdata *ppd;
15000 u64 reg;
15001 int i, ret;
15002 static const char * const inames[] = { /* implementation names */
15003 "RTL silicon",
15004 "RTL VCS simulation",
15005 "RTL FPGA emulation",
15006 "Functional simulator"
15007 };
15008 struct pci_dev *parent = pdev->bus->self;
15009 u32 sdma_engines = chip_sdma_engines(dd);
15010
15011 ppd = dd->pport;
15012 for (i = 0; i < dd->num_pports; i++, ppd++) {
15013 int vl;
15014 /* init common fields */
15015 hfi1_init_pportdata(pdev, ppd, dd, 0, 1);
15016 /* DC supports 4 link widths */
15017 ppd->link_width_supported =
15018 OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X |
15019 OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X;
15020 ppd->link_width_downgrade_supported =
15021 ppd->link_width_supported;
15022 /* start out enabling only 4X */
15023 ppd->link_width_enabled = OPA_LINK_WIDTH_4X;
15024 ppd->link_width_downgrade_enabled =
15025 ppd->link_width_downgrade_supported;
15026 /* link width active is 0 when link is down */
15027 /* link width downgrade active is 0 when link is down */
15028
15029 if (num_vls < HFI1_MIN_VLS_SUPPORTED ||
15030 num_vls > HFI1_MAX_VLS_SUPPORTED) {
15031 dd_dev_err(dd, "Invalid num_vls %u, using %u VLs\n",
15032 num_vls, HFI1_MAX_VLS_SUPPORTED);
15033 num_vls = HFI1_MAX_VLS_SUPPORTED;
15034 }
15035 ppd->vls_supported = num_vls;
15036 ppd->vls_operational = ppd->vls_supported;
15037 /* Set the default MTU. */
15038 for (vl = 0; vl < num_vls; vl++)
15039 dd->vld[vl].mtu = hfi1_max_mtu;
15040 dd->vld[15].mtu = MAX_MAD_PACKET;
15041 /*
15042 * Set the initial values to reasonable default, will be set
15043 * for real when link is up.
15044 */
15045 ppd->overrun_threshold = 0x4;
15046 ppd->phy_error_threshold = 0xf;
15047 ppd->port_crc_mode_enabled = link_crc_mask;
15048 /* initialize supported LTP CRC mode */
15049 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
15050 /* initialize enabled LTP CRC mode */
15051 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4;
15052 /* start in offline */
15053 ppd->host_link_state = HLS_DN_OFFLINE;
15054 init_vl_arb_caches(ppd);
15055 }
15056
15057 /*
15058 * Do remaining PCIe setup and save PCIe values in dd.
15059 * Any error printing is already done by the init code.
15060 * On return, we have the chip mapped.
15061 */
15062 ret = hfi1_pcie_ddinit(dd, pdev);
15063 if (ret < 0)
15064 goto bail_free;
15065
15066 /* Save PCI space registers to rewrite after device reset */
15067 ret = save_pci_variables(dd);
15068 if (ret < 0)
15069 goto bail_cleanup;
15070
15071 dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT)
15072 & CCE_REVISION_CHIP_REV_MAJOR_MASK;
15073 dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT)
15074 & CCE_REVISION_CHIP_REV_MINOR_MASK;
15075
15076 /*
15077 * Check interrupt registers mapping if the driver has no access to
15078 * the upstream component. In this case, it is likely that the driver
15079 * is running in a VM.
15080 */
15081 if (!parent) {
15082 ret = check_int_registers(dd);
15083 if (ret)
15084 goto bail_cleanup;
15085 }
15086
15087 /*
15088 * obtain the hardware ID - NOT related to unit, which is a
15089 * software enumeration
15090 */
15091 reg = read_csr(dd, CCE_REVISION2);
15092 dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT)
15093 & CCE_REVISION2_HFI_ID_MASK;
15094 /* the variable size will remove unwanted bits */
15095 dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT;
15096 dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT;
15097 dd_dev_info(dd, "Implementation: %s, revision 0x%x\n",
15098 dd->icode < ARRAY_SIZE(inames) ?
15099 inames[dd->icode] : "unknown", (int)dd->irev);
15100
15101 /* speeds the hardware can support */
15102 dd->pport->link_speed_supported = OPA_LINK_SPEED_25G;
15103 /* speeds allowed to run at */
15104 dd->pport->link_speed_enabled = dd->pport->link_speed_supported;
15105 /* give a reasonable active value, will be set on link up */
15106 dd->pport->link_speed_active = OPA_LINK_SPEED_25G;
15107
15108 /* fix up link widths for emulation _p */
15109 ppd = dd->pport;
15110 if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) {
15111 ppd->link_width_supported =
15112 ppd->link_width_enabled =
15113 ppd->link_width_downgrade_supported =
15114 ppd->link_width_downgrade_enabled =
15115 OPA_LINK_WIDTH_1X;
15116 }
15117 /* insure num_vls isn't larger than number of sdma engines */
15118 if (HFI1_CAP_IS_KSET(SDMA) && num_vls > sdma_engines) {
15119 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n",
15120 num_vls, sdma_engines);
15121 num_vls = sdma_engines;
15122 ppd->vls_supported = sdma_engines;
15123 ppd->vls_operational = ppd->vls_supported;
15124 }
15125
15126 /*
15127 * Convert the ns parameter to the 64 * cclocks used in the CSR.
15128 * Limit the max if larger than the field holds. If timeout is
15129 * non-zero, then the calculated field will be at least 1.
15130 *
15131 * Must be after icode is set up - the cclock rate depends
15132 * on knowing the hardware being used.
15133 */
15134 dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64;
15135 if (dd->rcv_intr_timeout_csr >
15136 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK)
15137 dd->rcv_intr_timeout_csr =
15138 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK;
15139 else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout)
15140 dd->rcv_intr_timeout_csr = 1;
15141
15142 /* needs to be done before we look for the peer device */
15143 read_guid(dd);
15144
15145 /* set up shared ASIC data with peer device */
15146 ret = init_asic_data(dd);
15147 if (ret)
15148 goto bail_cleanup;
15149
15150 /* obtain chip sizes, reset chip CSRs */
15151 ret = init_chip(dd);
15152 if (ret)
15153 goto bail_cleanup;
15154
15155 /* read in the PCIe link speed information */
15156 ret = pcie_speeds(dd);
15157 if (ret)
15158 goto bail_cleanup;
15159
15160 /* call before get_platform_config(), after init_chip_resources() */
15161 ret = eprom_init(dd);
15162 if (ret)
15163 goto bail_free_rcverr;
15164
15165 /* Needs to be called before hfi1_firmware_init */
15166 get_platform_config(dd);
15167
15168 /* read in firmware */
15169 ret = hfi1_firmware_init(dd);
15170 if (ret)
15171 goto bail_cleanup;
15172
15173 /*
15174 * In general, the PCIe Gen3 transition must occur after the
15175 * chip has been idled (so it won't initiate any PCIe transactions
15176 * e.g. an interrupt) and before the driver changes any registers
15177 * (the transition will reset the registers).
15178 *
15179 * In particular, place this call after:
15180 * - init_chip() - the chip will not initiate any PCIe transactions
15181 * - pcie_speeds() - reads the current link speed
15182 * - hfi1_firmware_init() - the needed firmware is ready to be
15183 * downloaded
15184 */
15185 ret = do_pcie_gen3_transition(dd);
15186 if (ret)
15187 goto bail_cleanup;
15188
15189 /*
15190 * This should probably occur in hfi1_pcie_init(), but historically
15191 * occurs after the do_pcie_gen3_transition() code.
15192 */
15193 tune_pcie_caps(dd);
15194
15195 /* start setting dd values and adjusting CSRs */
15196 init_early_variables(dd);
15197
15198 parse_platform_config(dd);
15199
15200 ret = obtain_boardname(dd);
15201 if (ret)
15202 goto bail_cleanup;
15203
15204 snprintf(dd->boardversion, BOARD_VERS_MAX,
15205 "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n",
15206 HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN,
15207 (u32)dd->majrev,
15208 (u32)dd->minrev,
15209 (dd->revision >> CCE_REVISION_SW_SHIFT)
15210 & CCE_REVISION_SW_MASK);
15211
15212 /* alloc VNIC/AIP rx data */
15213 ret = hfi1_alloc_rx(dd);
15214 if (ret)
15215 goto bail_cleanup;
15216
15217 ret = set_up_context_variables(dd);
15218 if (ret)
15219 goto bail_cleanup;
15220
15221 /* set initial RXE CSRs */
15222 ret = init_rxe(dd);
15223 if (ret)
15224 goto bail_cleanup;
15225
15226 /* set initial TXE CSRs */
15227 init_txe(dd);
15228 /* set initial non-RXE, non-TXE CSRs */
15229 init_other(dd);
15230 /* set up KDETH QP prefix in both RX and TX CSRs */
15231 init_kdeth_qp(dd);
15232
15233 ret = hfi1_dev_affinity_init(dd);
15234 if (ret)
15235 goto bail_cleanup;
15236
15237 /* send contexts must be set up before receive contexts */
15238 ret = init_send_contexts(dd);
15239 if (ret)
15240 goto bail_cleanup;
15241
15242 ret = hfi1_create_kctxts(dd);
15243 if (ret)
15244 goto bail_cleanup;
15245
15246 /*
15247 * Initialize aspm, to be done after gen3 transition and setting up
15248 * contexts and before enabling interrupts
15249 */
15250 aspm_init(dd);
15251
15252 ret = init_pervl_scs(dd);
15253 if (ret)
15254 goto bail_cleanup;
15255
15256 /* sdma init */
15257 for (i = 0; i < dd->num_pports; ++i) {
15258 ret = sdma_init(dd, i);
15259 if (ret)
15260 goto bail_cleanup;
15261 }
15262
15263 /* use contexts created by hfi1_create_kctxts */
15264 ret = set_up_interrupts(dd);
15265 if (ret)
15266 goto bail_cleanup;
15267
15268 ret = hfi1_comp_vectors_set_up(dd);
15269 if (ret)
15270 goto bail_clear_intr;
15271
15272 /* set up LCB access - must be after set_up_interrupts() */
15273 init_lcb_access(dd);
15274
15275 /*
15276 * Serial number is created from the base guid:
15277 * [27:24] = base guid [38:35]
15278 * [23: 0] = base guid [23: 0]
15279 */
15280 snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n",
15281 (dd->base_guid & 0xFFFFFF) |
15282 ((dd->base_guid >> 11) & 0xF000000));
15283
15284 dd->oui1 = dd->base_guid >> 56 & 0xFF;
15285 dd->oui2 = dd->base_guid >> 48 & 0xFF;
15286 dd->oui3 = dd->base_guid >> 40 & 0xFF;
15287
15288 ret = load_firmware(dd); /* asymmetric with dispose_firmware() */
15289 if (ret)
15290 goto bail_clear_intr;
15291
15292 thermal_init(dd);
15293
15294 ret = init_cntrs(dd);
15295 if (ret)
15296 goto bail_clear_intr;
15297
15298 ret = init_rcverr(dd);
15299 if (ret)
15300 goto bail_free_cntrs;
15301
15302 init_completion(&dd->user_comp);
15303
15304 /* The user refcount starts with one to inidicate an active device */
15305 refcount_set(&dd->user_refcount, 1);
15306
15307 goto bail;
15308
15309 bail_free_rcverr:
15310 free_rcverr(dd);
15311 bail_free_cntrs:
15312 free_cntrs(dd);
15313 bail_clear_intr:
15314 hfi1_comp_vectors_clean_up(dd);
15315 msix_clean_up_interrupts(dd);
15316 bail_cleanup:
15317 hfi1_free_rx(dd);
15318 hfi1_pcie_ddcleanup(dd);
15319 bail_free:
15320 hfi1_free_devdata(dd);
15321 bail:
15322 return ret;
15323 }
15324
delay_cycles(struct hfi1_pportdata * ppd,u32 desired_egress_rate,u32 dw_len)15325 static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate,
15326 u32 dw_len)
15327 {
15328 u32 delta_cycles;
15329 u32 current_egress_rate = ppd->current_egress_rate;
15330 /* rates here are in units of 10^6 bits/sec */
15331
15332 if (desired_egress_rate == -1)
15333 return 0; /* shouldn't happen */
15334
15335 if (desired_egress_rate >= current_egress_rate)
15336 return 0; /* we can't help go faster, only slower */
15337
15338 delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) -
15339 egress_cycles(dw_len * 4, current_egress_rate);
15340
15341 return (u16)delta_cycles;
15342 }
15343
15344 /**
15345 * create_pbc - build a pbc for transmission
15346 * @ppd: info of physical Hfi port
15347 * @flags: special case flags or-ed in built pbc
15348 * @srate_mbs: static rate
15349 * @vl: vl
15350 * @dw_len: dword length (header words + data words + pbc words)
15351 *
15352 * Create a PBC with the given flags, rate, VL, and length.
15353 *
15354 * NOTE: The PBC created will not insert any HCRC - all callers but one are
15355 * for verbs, which does not use this PSM feature. The lone other caller
15356 * is for the diagnostic interface which calls this if the user does not
15357 * supply their own PBC.
15358 */
create_pbc(struct hfi1_pportdata * ppd,u64 flags,int srate_mbs,u32 vl,u32 dw_len)15359 u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,
15360 u32 dw_len)
15361 {
15362 u64 pbc, delay = 0;
15363
15364 if (unlikely(srate_mbs))
15365 delay = delay_cycles(ppd, srate_mbs, dw_len);
15366
15367 pbc = flags
15368 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT)
15369 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT)
15370 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT
15371 | (dw_len & PBC_LENGTH_DWS_MASK)
15372 << PBC_LENGTH_DWS_SHIFT;
15373
15374 return pbc;
15375 }
15376
15377 #define SBUS_THERMAL 0x4f
15378 #define SBUS_THERM_MONITOR_MODE 0x1
15379
15380 #define THERM_FAILURE(dev, ret, reason) \
15381 dd_dev_err((dd), \
15382 "Thermal sensor initialization failed: %s (%d)\n", \
15383 (reason), (ret))
15384
15385 /*
15386 * Initialize the thermal sensor.
15387 *
15388 * After initialization, enable polling of thermal sensor through
15389 * SBus interface. In order for this to work, the SBus Master
15390 * firmware has to be loaded due to the fact that the HW polling
15391 * logic uses SBus interrupts, which are not supported with
15392 * default firmware. Otherwise, no data will be returned through
15393 * the ASIC_STS_THERM CSR.
15394 */
thermal_init(struct hfi1_devdata * dd)15395 static int thermal_init(struct hfi1_devdata *dd)
15396 {
15397 int ret = 0;
15398
15399 if (dd->icode != ICODE_RTL_SILICON ||
15400 check_chip_resource(dd, CR_THERM_INIT, NULL))
15401 return ret;
15402
15403 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
15404 if (ret) {
15405 THERM_FAILURE(dd, ret, "Acquire SBus");
15406 return ret;
15407 }
15408
15409 dd_dev_info(dd, "Initializing thermal sensor\n");
15410 /* Disable polling of thermal readings */
15411 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);
15412 msleep(100);
15413 /* Thermal Sensor Initialization */
15414 /* Step 1: Reset the Thermal SBus Receiver */
15415 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
15416 RESET_SBUS_RECEIVER, 0);
15417 if (ret) {
15418 THERM_FAILURE(dd, ret, "Bus Reset");
15419 goto done;
15420 }
15421 /* Step 2: Set Reset bit in Thermal block */
15422 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
15423 WRITE_SBUS_RECEIVER, 0x1);
15424 if (ret) {
15425 THERM_FAILURE(dd, ret, "Therm Block Reset");
15426 goto done;
15427 }
15428 /* Step 3: Write clock divider value (100MHz -> 2MHz) */
15429 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1,
15430 WRITE_SBUS_RECEIVER, 0x32);
15431 if (ret) {
15432 THERM_FAILURE(dd, ret, "Write Clock Div");
15433 goto done;
15434 }
15435 /* Step 4: Select temperature mode */
15436 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3,
15437 WRITE_SBUS_RECEIVER,
15438 SBUS_THERM_MONITOR_MODE);
15439 if (ret) {
15440 THERM_FAILURE(dd, ret, "Write Mode Sel");
15441 goto done;
15442 }
15443 /* Step 5: De-assert block reset and start conversion */
15444 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
15445 WRITE_SBUS_RECEIVER, 0x2);
15446 if (ret) {
15447 THERM_FAILURE(dd, ret, "Write Reset Deassert");
15448 goto done;
15449 }
15450 /* Step 5.1: Wait for first conversion (21.5ms per spec) */
15451 msleep(22);
15452
15453 /* Enable polling of thermal readings */
15454 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);
15455
15456 /* Set initialized flag */
15457 ret = acquire_chip_resource(dd, CR_THERM_INIT, 0);
15458 if (ret)
15459 THERM_FAILURE(dd, ret, "Unable to set thermal init flag");
15460
15461 done:
15462 release_chip_resource(dd, CR_SBUS);
15463 return ret;
15464 }
15465
handle_temp_err(struct hfi1_devdata * dd)15466 static void handle_temp_err(struct hfi1_devdata *dd)
15467 {
15468 struct hfi1_pportdata *ppd = &dd->pport[0];
15469 /*
15470 * Thermal Critical Interrupt
15471 * Put the device into forced freeze mode, take link down to
15472 * offline, and put DC into reset.
15473 */
15474 dd_dev_emerg(dd,
15475 "Critical temperature reached! Forcing device into freeze mode!\n");
15476 dd->flags |= HFI1_FORCED_FREEZE;
15477 start_freeze_handling(ppd, FREEZE_SELF | FREEZE_ABORT);
15478 /*
15479 * Shut DC down as much and as quickly as possible.
15480 *
15481 * Step 1: Take the link down to OFFLINE. This will cause the
15482 * 8051 to put the Serdes in reset. However, we don't want to
15483 * go through the entire link state machine since we want to
15484 * shutdown ASAP. Furthermore, this is not a graceful shutdown
15485 * but rather an attempt to save the chip.
15486 * Code below is almost the same as quiet_serdes() but avoids
15487 * all the extra work and the sleeps.
15488 */
15489 ppd->driver_link_ready = 0;
15490 ppd->link_enabled = 0;
15491 set_physical_link_state(dd, (OPA_LINKDOWN_REASON_SMA_DISABLED << 8) |
15492 PLS_OFFLINE);
15493 /*
15494 * Step 2: Shutdown LCB and 8051
15495 * After shutdown, do not restore DC_CFG_RESET value.
15496 */
15497 dc_shutdown(dd);
15498 }
15499