1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2 /*
3 * Copyright(c) 2015 - 2019 Intel Corporation.
4 */
5
6 #include <linux/bitfield.h>
7 #include <linux/pci.h>
8 #include <linux/io.h>
9 #include <linux/delay.h>
10 #include <linux/vmalloc.h>
11 #include <linux/aer.h>
12 #include <linux/module.h>
13
14 #include "hfi.h"
15 #include "chip_registers.h"
16 #include "aspm.h"
17
18 /*
19 * This file contains PCIe utility routines.
20 */
21
22 /*
23 * Do all the common PCIe setup and initialization.
24 */
hfi1_pcie_init(struct hfi1_devdata * dd)25 int hfi1_pcie_init(struct hfi1_devdata *dd)
26 {
27 int ret;
28 struct pci_dev *pdev = dd->pcidev;
29
30 ret = pci_enable_device(pdev);
31 if (ret) {
32 /*
33 * This can happen (in theory) iff:
34 * We did a chip reset, and then failed to reprogram the
35 * BAR, or the chip reset due to an internal error. We then
36 * unloaded the driver and reloaded it.
37 *
38 * Both reset cases set the BAR back to initial state. For
39 * the latter case, the AER sticky error bit at offset 0x718
40 * should be set, but the Linux kernel doesn't yet know
41 * about that, it appears. If the original BAR was retained
42 * in the kernel data structures, this may be OK.
43 */
44 dd_dev_err(dd, "pci enable failed: error %d\n", -ret);
45 return ret;
46 }
47
48 ret = pci_request_regions(pdev, DRIVER_NAME);
49 if (ret) {
50 dd_dev_err(dd, "pci_request_regions fails: err %d\n", -ret);
51 goto bail;
52 }
53
54 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
55 if (ret) {
56 /*
57 * If the 64 bit setup fails, try 32 bit. Some systems
58 * do not setup 64 bit maps on systems with 2GB or less
59 * memory installed.
60 */
61 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
62 if (ret) {
63 dd_dev_err(dd, "Unable to set DMA mask: %d\n", ret);
64 goto bail;
65 }
66 }
67
68 pci_set_master(pdev);
69 (void)pci_enable_pcie_error_reporting(pdev);
70 return 0;
71
72 bail:
73 hfi1_pcie_cleanup(pdev);
74 return ret;
75 }
76
77 /*
78 * Clean what was done in hfi1_pcie_init()
79 */
hfi1_pcie_cleanup(struct pci_dev * pdev)80 void hfi1_pcie_cleanup(struct pci_dev *pdev)
81 {
82 pci_disable_device(pdev);
83 /*
84 * Release regions should be called after the disable. OK to
85 * call if request regions has not been called or failed.
86 */
87 pci_release_regions(pdev);
88 }
89
90 /*
91 * Do remaining PCIe setup, once dd is allocated, and save away
92 * fields required to re-initialize after a chip reset, or for
93 * various other purposes
94 */
hfi1_pcie_ddinit(struct hfi1_devdata * dd,struct pci_dev * pdev)95 int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev)
96 {
97 unsigned long len;
98 resource_size_t addr;
99 int ret = 0;
100 u32 rcv_array_count;
101
102 addr = pci_resource_start(pdev, 0);
103 len = pci_resource_len(pdev, 0);
104
105 /*
106 * The TXE PIO buffers are at the tail end of the chip space.
107 * Cut them off and map them separately.
108 */
109
110 /* sanity check vs expectations */
111 if (len != TXE_PIO_SEND + TXE_PIO_SIZE) {
112 dd_dev_err(dd, "chip PIO range does not match\n");
113 return -EINVAL;
114 }
115
116 dd->kregbase1 = ioremap(addr, RCV_ARRAY);
117 if (!dd->kregbase1) {
118 dd_dev_err(dd, "UC mapping of kregbase1 failed\n");
119 return -ENOMEM;
120 }
121 dd_dev_info(dd, "UC base1: %p for %x\n", dd->kregbase1, RCV_ARRAY);
122
123 /* verify that reads actually work, save revision for reset check */
124 dd->revision = readq(dd->kregbase1 + CCE_REVISION);
125 if (dd->revision == ~(u64)0) {
126 dd_dev_err(dd, "Cannot read chip CSRs\n");
127 goto nomem;
128 }
129
130 rcv_array_count = readq(dd->kregbase1 + RCV_ARRAY_CNT);
131 dd_dev_info(dd, "RcvArray count: %u\n", rcv_array_count);
132 dd->base2_start = RCV_ARRAY + rcv_array_count * 8;
133
134 dd->kregbase2 = ioremap(
135 addr + dd->base2_start,
136 TXE_PIO_SEND - dd->base2_start);
137 if (!dd->kregbase2) {
138 dd_dev_err(dd, "UC mapping of kregbase2 failed\n");
139 goto nomem;
140 }
141 dd_dev_info(dd, "UC base2: %p for %x\n", dd->kregbase2,
142 TXE_PIO_SEND - dd->base2_start);
143
144 dd->piobase = ioremap_wc(addr + TXE_PIO_SEND, TXE_PIO_SIZE);
145 if (!dd->piobase) {
146 dd_dev_err(dd, "WC mapping of send buffers failed\n");
147 goto nomem;
148 }
149 dd_dev_info(dd, "WC piobase: %p for %x\n", dd->piobase, TXE_PIO_SIZE);
150
151 dd->physaddr = addr; /* used for io_remap, etc. */
152
153 /*
154 * Map the chip's RcvArray as write-combining to allow us
155 * to write an entire cacheline worth of entries in one shot.
156 */
157 dd->rcvarray_wc = ioremap_wc(addr + RCV_ARRAY,
158 rcv_array_count * 8);
159 if (!dd->rcvarray_wc) {
160 dd_dev_err(dd, "WC mapping of receive array failed\n");
161 goto nomem;
162 }
163 dd_dev_info(dd, "WC RcvArray: %p for %x\n",
164 dd->rcvarray_wc, rcv_array_count * 8);
165
166 dd->flags |= HFI1_PRESENT; /* chip.c CSR routines now work */
167 return 0;
168 nomem:
169 ret = -ENOMEM;
170 hfi1_pcie_ddcleanup(dd);
171 return ret;
172 }
173
174 /*
175 * Do PCIe cleanup related to dd, after chip-specific cleanup, etc. Just prior
176 * to releasing the dd memory.
177 * Void because all of the core pcie cleanup functions are void.
178 */
hfi1_pcie_ddcleanup(struct hfi1_devdata * dd)179 void hfi1_pcie_ddcleanup(struct hfi1_devdata *dd)
180 {
181 dd->flags &= ~HFI1_PRESENT;
182 if (dd->kregbase1)
183 iounmap(dd->kregbase1);
184 dd->kregbase1 = NULL;
185 if (dd->kregbase2)
186 iounmap(dd->kregbase2);
187 dd->kregbase2 = NULL;
188 if (dd->rcvarray_wc)
189 iounmap(dd->rcvarray_wc);
190 dd->rcvarray_wc = NULL;
191 if (dd->piobase)
192 iounmap(dd->piobase);
193 dd->piobase = NULL;
194 }
195
196 /* return the PCIe link speed from the given link status */
extract_speed(u16 linkstat)197 static u32 extract_speed(u16 linkstat)
198 {
199 u32 speed;
200
201 switch (linkstat & PCI_EXP_LNKSTA_CLS) {
202 default: /* not defined, assume Gen1 */
203 case PCI_EXP_LNKSTA_CLS_2_5GB:
204 speed = 2500; /* Gen 1, 2.5GHz */
205 break;
206 case PCI_EXP_LNKSTA_CLS_5_0GB:
207 speed = 5000; /* Gen 2, 5GHz */
208 break;
209 case PCI_EXP_LNKSTA_CLS_8_0GB:
210 speed = 8000; /* Gen 3, 8GHz */
211 break;
212 }
213 return speed;
214 }
215
216 /* read the link status and set dd->{lbus_width,lbus_speed,lbus_info} */
update_lbus_info(struct hfi1_devdata * dd)217 static void update_lbus_info(struct hfi1_devdata *dd)
218 {
219 u16 linkstat;
220 int ret;
221
222 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKSTA, &linkstat);
223 if (ret) {
224 dd_dev_err(dd, "Unable to read from PCI config\n");
225 return;
226 }
227
228 dd->lbus_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, linkstat);
229 dd->lbus_speed = extract_speed(linkstat);
230 snprintf(dd->lbus_info, sizeof(dd->lbus_info),
231 "PCIe,%uMHz,x%u", dd->lbus_speed, dd->lbus_width);
232 }
233
234 /*
235 * Read in the current PCIe link width and speed. Find if the link is
236 * Gen3 capable.
237 */
pcie_speeds(struct hfi1_devdata * dd)238 int pcie_speeds(struct hfi1_devdata *dd)
239 {
240 u32 linkcap;
241 struct pci_dev *parent = dd->pcidev->bus->self;
242 int ret;
243
244 if (!pci_is_pcie(dd->pcidev)) {
245 dd_dev_err(dd, "Can't find PCI Express capability!\n");
246 return -EINVAL;
247 }
248
249 /* find if our max speed is Gen3 and parent supports Gen3 speeds */
250 dd->link_gen3_capable = 1;
251
252 ret = pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &linkcap);
253 if (ret) {
254 dd_dev_err(dd, "Unable to read from PCI config\n");
255 return pcibios_err_to_errno(ret);
256 }
257
258 if ((linkcap & PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_8_0GB) {
259 dd_dev_info(dd,
260 "This HFI is not Gen3 capable, max speed 0x%x, need 0x3\n",
261 linkcap & PCI_EXP_LNKCAP_SLS);
262 dd->link_gen3_capable = 0;
263 }
264
265 /*
266 * bus->max_bus_speed is set from the bridge's linkcap Max Link Speed
267 */
268 if (parent &&
269 (dd->pcidev->bus->max_bus_speed == PCIE_SPEED_2_5GT ||
270 dd->pcidev->bus->max_bus_speed == PCIE_SPEED_5_0GT)) {
271 dd_dev_info(dd, "Parent PCIe bridge does not support Gen3\n");
272 dd->link_gen3_capable = 0;
273 }
274
275 /* obtain the link width and current speed */
276 update_lbus_info(dd);
277
278 dd_dev_info(dd, "%s\n", dd->lbus_info);
279
280 return 0;
281 }
282
283 /*
284 * Restore command and BARs after a reset has wiped them out
285 *
286 * Returns 0 on success, otherwise a negative error value
287 */
restore_pci_variables(struct hfi1_devdata * dd)288 int restore_pci_variables(struct hfi1_devdata *dd)
289 {
290 int ret;
291
292 ret = pci_write_config_word(dd->pcidev, PCI_COMMAND, dd->pci_command);
293 if (ret)
294 goto error;
295
296 ret = pci_write_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0,
297 dd->pcibar0);
298 if (ret)
299 goto error;
300
301 ret = pci_write_config_dword(dd->pcidev, PCI_BASE_ADDRESS_1,
302 dd->pcibar1);
303 if (ret)
304 goto error;
305
306 ret = pci_write_config_dword(dd->pcidev, PCI_ROM_ADDRESS, dd->pci_rom);
307 if (ret)
308 goto error;
309
310 ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_DEVCTL,
311 dd->pcie_devctl);
312 if (ret)
313 goto error;
314
315 ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_LNKCTL,
316 dd->pcie_lnkctl);
317 if (ret)
318 goto error;
319
320 ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_DEVCTL2,
321 dd->pcie_devctl2);
322 if (ret)
323 goto error;
324
325 ret = pci_write_config_dword(dd->pcidev, PCI_CFG_MSIX0, dd->pci_msix0);
326 if (ret)
327 goto error;
328
329 if (pci_find_ext_capability(dd->pcidev, PCI_EXT_CAP_ID_TPH)) {
330 ret = pci_write_config_dword(dd->pcidev, PCIE_CFG_TPH2,
331 dd->pci_tph2);
332 if (ret)
333 goto error;
334 }
335 return 0;
336
337 error:
338 dd_dev_err(dd, "Unable to write to PCI config\n");
339 return pcibios_err_to_errno(ret);
340 }
341
342 /*
343 * Save BARs and command to rewrite after device reset
344 *
345 * Returns 0 on success, otherwise a negative error value
346 */
save_pci_variables(struct hfi1_devdata * dd)347 int save_pci_variables(struct hfi1_devdata *dd)
348 {
349 int ret;
350
351 ret = pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0,
352 &dd->pcibar0);
353 if (ret)
354 goto error;
355
356 ret = pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_1,
357 &dd->pcibar1);
358 if (ret)
359 goto error;
360
361 ret = pci_read_config_dword(dd->pcidev, PCI_ROM_ADDRESS, &dd->pci_rom);
362 if (ret)
363 goto error;
364
365 ret = pci_read_config_word(dd->pcidev, PCI_COMMAND, &dd->pci_command);
366 if (ret)
367 goto error;
368
369 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL,
370 &dd->pcie_devctl);
371 if (ret)
372 goto error;
373
374 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKCTL,
375 &dd->pcie_lnkctl);
376 if (ret)
377 goto error;
378
379 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL2,
380 &dd->pcie_devctl2);
381 if (ret)
382 goto error;
383
384 ret = pci_read_config_dword(dd->pcidev, PCI_CFG_MSIX0, &dd->pci_msix0);
385 if (ret)
386 goto error;
387
388 if (pci_find_ext_capability(dd->pcidev, PCI_EXT_CAP_ID_TPH)) {
389 ret = pci_read_config_dword(dd->pcidev, PCIE_CFG_TPH2,
390 &dd->pci_tph2);
391 if (ret)
392 goto error;
393 }
394 return 0;
395
396 error:
397 dd_dev_err(dd, "Unable to read from PCI config\n");
398 return pcibios_err_to_errno(ret);
399 }
400
401 /*
402 * BIOS may not set PCIe bus-utilization parameters for best performance.
403 * Check and optionally adjust them to maximize our throughput.
404 */
405 static int hfi1_pcie_caps;
406 module_param_named(pcie_caps, hfi1_pcie_caps, int, 0444);
407 MODULE_PARM_DESC(pcie_caps, "Max PCIe tuning: Payload (0..3), ReadReq (4..7)");
408
409 /**
410 * tune_pcie_caps() - Code to adjust PCIe capabilities.
411 * @dd: Valid device data structure
412 *
413 */
tune_pcie_caps(struct hfi1_devdata * dd)414 void tune_pcie_caps(struct hfi1_devdata *dd)
415 {
416 struct pci_dev *parent;
417 u16 rc_mpss, rc_mps, ep_mpss, ep_mps;
418 u16 rc_mrrs, ep_mrrs, max_mrrs, ectl;
419 int ret;
420
421 /*
422 * Turn on extended tags in DevCtl in case the BIOS has turned it off
423 * to improve WFR SDMA bandwidth
424 */
425 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL, &ectl);
426 if ((!ret) && !(ectl & PCI_EXP_DEVCTL_EXT_TAG)) {
427 dd_dev_info(dd, "Enabling PCIe extended tags\n");
428 ectl |= PCI_EXP_DEVCTL_EXT_TAG;
429 ret = pcie_capability_write_word(dd->pcidev,
430 PCI_EXP_DEVCTL, ectl);
431 if (ret)
432 dd_dev_info(dd, "Unable to write to PCI config\n");
433 }
434 /* Find out supported and configured values for parent (root) */
435 parent = dd->pcidev->bus->self;
436 /*
437 * The driver cannot perform the tuning if it does not have
438 * access to the upstream component.
439 */
440 if (!parent) {
441 dd_dev_info(dd, "Parent not found\n");
442 return;
443 }
444 if (!pci_is_root_bus(parent->bus)) {
445 dd_dev_info(dd, "Parent not root\n");
446 return;
447 }
448 if (!pci_is_pcie(parent)) {
449 dd_dev_info(dd, "Parent is not PCI Express capable\n");
450 return;
451 }
452 if (!pci_is_pcie(dd->pcidev)) {
453 dd_dev_info(dd, "PCI device is not PCI Express capable\n");
454 return;
455 }
456 rc_mpss = parent->pcie_mpss;
457 rc_mps = ffs(pcie_get_mps(parent)) - 8;
458 /* Find out supported and configured values for endpoint (us) */
459 ep_mpss = dd->pcidev->pcie_mpss;
460 ep_mps = ffs(pcie_get_mps(dd->pcidev)) - 8;
461
462 /* Find max payload supported by root, endpoint */
463 if (rc_mpss > ep_mpss)
464 rc_mpss = ep_mpss;
465
466 /* If Supported greater than limit in module param, limit it */
467 if (rc_mpss > (hfi1_pcie_caps & 7))
468 rc_mpss = hfi1_pcie_caps & 7;
469 /* If less than (allowed, supported), bump root payload */
470 if (rc_mpss > rc_mps) {
471 rc_mps = rc_mpss;
472 pcie_set_mps(parent, 128 << rc_mps);
473 }
474 /* If less than (allowed, supported), bump endpoint payload */
475 if (rc_mpss > ep_mps) {
476 ep_mps = rc_mpss;
477 pcie_set_mps(dd->pcidev, 128 << ep_mps);
478 }
479
480 /*
481 * Now the Read Request size.
482 * No field for max supported, but PCIe spec limits it to 4096,
483 * which is code '5' (log2(4096) - 7)
484 */
485 max_mrrs = 5;
486 if (max_mrrs > ((hfi1_pcie_caps >> 4) & 7))
487 max_mrrs = (hfi1_pcie_caps >> 4) & 7;
488
489 max_mrrs = 128 << max_mrrs;
490 rc_mrrs = pcie_get_readrq(parent);
491 ep_mrrs = pcie_get_readrq(dd->pcidev);
492
493 if (max_mrrs > rc_mrrs) {
494 rc_mrrs = max_mrrs;
495 pcie_set_readrq(parent, rc_mrrs);
496 }
497 if (max_mrrs > ep_mrrs) {
498 ep_mrrs = max_mrrs;
499 pcie_set_readrq(dd->pcidev, ep_mrrs);
500 }
501 }
502
503 /* End of PCIe capability tuning */
504
505 /*
506 * From here through hfi1_pci_err_handler definition is invoked via
507 * PCI error infrastructure, registered via pci
508 */
509 static pci_ers_result_t
pci_error_detected(struct pci_dev * pdev,pci_channel_state_t state)510 pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
511 {
512 struct hfi1_devdata *dd = pci_get_drvdata(pdev);
513 pci_ers_result_t ret = PCI_ERS_RESULT_RECOVERED;
514
515 switch (state) {
516 case pci_channel_io_normal:
517 dd_dev_info(dd, "State Normal, ignoring\n");
518 break;
519
520 case pci_channel_io_frozen:
521 dd_dev_info(dd, "State Frozen, requesting reset\n");
522 pci_disable_device(pdev);
523 ret = PCI_ERS_RESULT_NEED_RESET;
524 break;
525
526 case pci_channel_io_perm_failure:
527 if (dd) {
528 dd_dev_info(dd, "State Permanent Failure, disabling\n");
529 /* no more register accesses! */
530 dd->flags &= ~HFI1_PRESENT;
531 hfi1_disable_after_error(dd);
532 }
533 /* else early, or other problem */
534 ret = PCI_ERS_RESULT_DISCONNECT;
535 break;
536
537 default: /* shouldn't happen */
538 dd_dev_info(dd, "HFI1 PCI errors detected (state %d)\n",
539 state);
540 break;
541 }
542 return ret;
543 }
544
545 static pci_ers_result_t
pci_mmio_enabled(struct pci_dev * pdev)546 pci_mmio_enabled(struct pci_dev *pdev)
547 {
548 u64 words = 0U;
549 struct hfi1_devdata *dd = pci_get_drvdata(pdev);
550 pci_ers_result_t ret = PCI_ERS_RESULT_RECOVERED;
551
552 if (dd && dd->pport) {
553 words = read_port_cntr(dd->pport, C_RX_WORDS, CNTR_INVALID_VL);
554 if (words == ~0ULL)
555 ret = PCI_ERS_RESULT_NEED_RESET;
556 dd_dev_info(dd,
557 "HFI1 mmio_enabled function called, read wordscntr %llx, returning %d\n",
558 words, ret);
559 }
560 return ret;
561 }
562
563 static pci_ers_result_t
pci_slot_reset(struct pci_dev * pdev)564 pci_slot_reset(struct pci_dev *pdev)
565 {
566 struct hfi1_devdata *dd = pci_get_drvdata(pdev);
567
568 dd_dev_info(dd, "HFI1 slot_reset function called, ignored\n");
569 return PCI_ERS_RESULT_CAN_RECOVER;
570 }
571
572 static void
pci_resume(struct pci_dev * pdev)573 pci_resume(struct pci_dev *pdev)
574 {
575 struct hfi1_devdata *dd = pci_get_drvdata(pdev);
576
577 dd_dev_info(dd, "HFI1 resume function called\n");
578 /*
579 * Running jobs will fail, since it's asynchronous
580 * unlike sysfs-requested reset. Better than
581 * doing nothing.
582 */
583 hfi1_init(dd, 1); /* same as re-init after reset */
584 }
585
586 const struct pci_error_handlers hfi1_pci_err_handler = {
587 .error_detected = pci_error_detected,
588 .mmio_enabled = pci_mmio_enabled,
589 .slot_reset = pci_slot_reset,
590 .resume = pci_resume,
591 };
592
593 /*============================================================================*/
594 /* PCIe Gen3 support */
595
596 /*
597 * This code is separated out because it is expected to be removed in the
598 * final shipping product. If not, then it will be revisited and items
599 * will be moved to more standard locations.
600 */
601
602 /* ASIC_PCI_SD_HOST_STATUS.FW_DNLD_STS field values */
603 #define DL_STATUS_HFI0 0x1 /* hfi0 firmware download complete */
604 #define DL_STATUS_HFI1 0x2 /* hfi1 firmware download complete */
605 #define DL_STATUS_BOTH 0x3 /* hfi0 and hfi1 firmware download complete */
606
607 /* ASIC_PCI_SD_HOST_STATUS.FW_DNLD_ERR field values */
608 #define DL_ERR_NONE 0x0 /* no error */
609 #define DL_ERR_SWAP_PARITY 0x1 /* parity error in SerDes interrupt */
610 /* or response data */
611 #define DL_ERR_DISABLED 0x2 /* hfi disabled */
612 #define DL_ERR_SECURITY 0x3 /* security check failed */
613 #define DL_ERR_SBUS 0x4 /* SBus status error */
614 #define DL_ERR_XFR_PARITY 0x5 /* parity error during ROM transfer*/
615
616 /* gasket block secondary bus reset delay */
617 #define SBR_DELAY_US 200000 /* 200ms */
618
619 static uint pcie_target = 3;
620 module_param(pcie_target, uint, S_IRUGO);
621 MODULE_PARM_DESC(pcie_target, "PCIe target speed (0 skip, 1-3 Gen1-3)");
622
623 static uint pcie_force;
624 module_param(pcie_force, uint, S_IRUGO);
625 MODULE_PARM_DESC(pcie_force, "Force driver to do a PCIe firmware download even if already at target speed");
626
627 static uint pcie_retry = 5;
628 module_param(pcie_retry, uint, S_IRUGO);
629 MODULE_PARM_DESC(pcie_retry, "Driver will try this many times to reach requested speed");
630
631 #define UNSET_PSET 255
632 #define DEFAULT_DISCRETE_PSET 2 /* discrete HFI */
633 #define DEFAULT_MCP_PSET 6 /* MCP HFI */
634 static uint pcie_pset = UNSET_PSET;
635 module_param(pcie_pset, uint, S_IRUGO);
636 MODULE_PARM_DESC(pcie_pset, "PCIe Eq Pset value to use, range is 0-10");
637
638 static uint pcie_ctle = 3; /* discrete on, integrated on */
639 module_param(pcie_ctle, uint, S_IRUGO);
640 MODULE_PARM_DESC(pcie_ctle, "PCIe static CTLE mode, bit 0 - discrete on/off, bit 1 - integrated on/off");
641
642 /* equalization columns */
643 #define PREC 0
644 #define ATTN 1
645 #define POST 2
646
647 /* discrete silicon preliminary equalization values */
648 static const u8 discrete_preliminary_eq[11][3] = {
649 /* prec attn post */
650 { 0x00, 0x00, 0x12 }, /* p0 */
651 { 0x00, 0x00, 0x0c }, /* p1 */
652 { 0x00, 0x00, 0x0f }, /* p2 */
653 { 0x00, 0x00, 0x09 }, /* p3 */
654 { 0x00, 0x00, 0x00 }, /* p4 */
655 { 0x06, 0x00, 0x00 }, /* p5 */
656 { 0x09, 0x00, 0x00 }, /* p6 */
657 { 0x06, 0x00, 0x0f }, /* p7 */
658 { 0x09, 0x00, 0x09 }, /* p8 */
659 { 0x0c, 0x00, 0x00 }, /* p9 */
660 { 0x00, 0x00, 0x18 }, /* p10 */
661 };
662
663 /* integrated silicon preliminary equalization values */
664 static const u8 integrated_preliminary_eq[11][3] = {
665 /* prec attn post */
666 { 0x00, 0x1e, 0x07 }, /* p0 */
667 { 0x00, 0x1e, 0x05 }, /* p1 */
668 { 0x00, 0x1e, 0x06 }, /* p2 */
669 { 0x00, 0x1e, 0x04 }, /* p3 */
670 { 0x00, 0x1e, 0x00 }, /* p4 */
671 { 0x03, 0x1e, 0x00 }, /* p5 */
672 { 0x04, 0x1e, 0x00 }, /* p6 */
673 { 0x03, 0x1e, 0x06 }, /* p7 */
674 { 0x03, 0x1e, 0x04 }, /* p8 */
675 { 0x05, 0x1e, 0x00 }, /* p9 */
676 { 0x00, 0x1e, 0x0a }, /* p10 */
677 };
678
679 static const u8 discrete_ctle_tunings[11][4] = {
680 /* DC LF HF BW */
681 { 0x48, 0x0b, 0x04, 0x04 }, /* p0 */
682 { 0x60, 0x05, 0x0f, 0x0a }, /* p1 */
683 { 0x50, 0x09, 0x06, 0x06 }, /* p2 */
684 { 0x68, 0x05, 0x0f, 0x0a }, /* p3 */
685 { 0x80, 0x05, 0x0f, 0x0a }, /* p4 */
686 { 0x70, 0x05, 0x0f, 0x0a }, /* p5 */
687 { 0x68, 0x05, 0x0f, 0x0a }, /* p6 */
688 { 0x38, 0x0f, 0x00, 0x00 }, /* p7 */
689 { 0x48, 0x09, 0x06, 0x06 }, /* p8 */
690 { 0x60, 0x05, 0x0f, 0x0a }, /* p9 */
691 { 0x38, 0x0f, 0x00, 0x00 }, /* p10 */
692 };
693
694 static const u8 integrated_ctle_tunings[11][4] = {
695 /* DC LF HF BW */
696 { 0x38, 0x0f, 0x00, 0x00 }, /* p0 */
697 { 0x38, 0x0f, 0x00, 0x00 }, /* p1 */
698 { 0x38, 0x0f, 0x00, 0x00 }, /* p2 */
699 { 0x38, 0x0f, 0x00, 0x00 }, /* p3 */
700 { 0x58, 0x0a, 0x05, 0x05 }, /* p4 */
701 { 0x48, 0x0a, 0x05, 0x05 }, /* p5 */
702 { 0x40, 0x0a, 0x05, 0x05 }, /* p6 */
703 { 0x38, 0x0f, 0x00, 0x00 }, /* p7 */
704 { 0x38, 0x0f, 0x00, 0x00 }, /* p8 */
705 { 0x38, 0x09, 0x06, 0x06 }, /* p9 */
706 { 0x38, 0x0e, 0x01, 0x01 }, /* p10 */
707 };
708
709 /* helper to format the value to write to hardware */
710 #define eq_value(pre, curr, post) \
711 ((((u32)(pre)) << \
712 PCIE_CFG_REG_PL102_GEN3_EQ_PRE_CURSOR_PSET_SHIFT) \
713 | (((u32)(curr)) << PCIE_CFG_REG_PL102_GEN3_EQ_CURSOR_PSET_SHIFT) \
714 | (((u32)(post)) << \
715 PCIE_CFG_REG_PL102_GEN3_EQ_POST_CURSOR_PSET_SHIFT))
716
717 /*
718 * Load the given EQ preset table into the PCIe hardware.
719 */
load_eq_table(struct hfi1_devdata * dd,const u8 eq[11][3],u8 fs,u8 div)720 static int load_eq_table(struct hfi1_devdata *dd, const u8 eq[11][3], u8 fs,
721 u8 div)
722 {
723 struct pci_dev *pdev = dd->pcidev;
724 u32 hit_error = 0;
725 u32 violation;
726 u32 i;
727 u8 c_minus1, c0, c_plus1;
728 int ret;
729
730 for (i = 0; i < 11; i++) {
731 /* set index */
732 pci_write_config_dword(pdev, PCIE_CFG_REG_PL103, i);
733 /* write the value */
734 c_minus1 = eq[i][PREC] / div;
735 c0 = fs - (eq[i][PREC] / div) - (eq[i][POST] / div);
736 c_plus1 = eq[i][POST] / div;
737 pci_write_config_dword(pdev, PCIE_CFG_REG_PL102,
738 eq_value(c_minus1, c0, c_plus1));
739 /* check if these coefficients violate EQ rules */
740 ret = pci_read_config_dword(dd->pcidev,
741 PCIE_CFG_REG_PL105, &violation);
742 if (ret) {
743 dd_dev_err(dd, "Unable to read from PCI config\n");
744 hit_error = 1;
745 break;
746 }
747
748 if (violation
749 & PCIE_CFG_REG_PL105_GEN3_EQ_VIOLATE_COEF_RULES_SMASK){
750 if (hit_error == 0) {
751 dd_dev_err(dd,
752 "Gen3 EQ Table Coefficient rule violations\n");
753 dd_dev_err(dd, " prec attn post\n");
754 }
755 dd_dev_err(dd, " p%02d: %02x %02x %02x\n",
756 i, (u32)eq[i][0], (u32)eq[i][1],
757 (u32)eq[i][2]);
758 dd_dev_err(dd, " %02x %02x %02x\n",
759 (u32)c_minus1, (u32)c0, (u32)c_plus1);
760 hit_error = 1;
761 }
762 }
763 if (hit_error)
764 return -EINVAL;
765 return 0;
766 }
767
768 /*
769 * Steps to be done after the PCIe firmware is downloaded and
770 * before the SBR for the Pcie Gen3.
771 * The SBus resource is already being held.
772 */
pcie_post_steps(struct hfi1_devdata * dd)773 static void pcie_post_steps(struct hfi1_devdata *dd)
774 {
775 int i;
776
777 set_sbus_fast_mode(dd);
778 /*
779 * Write to the PCIe PCSes to set the G3_LOCKED_NEXT bits to 1.
780 * This avoids a spurious framing error that can otherwise be
781 * generated by the MAC layer.
782 *
783 * Use individual addresses since no broadcast is set up.
784 */
785 for (i = 0; i < NUM_PCIE_SERDES; i++) {
786 sbus_request(dd, pcie_pcs_addrs[dd->hfi1_id][i],
787 0x03, WRITE_SBUS_RECEIVER, 0x00022132);
788 }
789
790 clear_sbus_fast_mode(dd);
791 }
792
793 /*
794 * Trigger a secondary bus reset (SBR) on ourselves using our parent.
795 *
796 * Based on pci_parent_bus_reset() which is not exported by the
797 * kernel core.
798 */
trigger_sbr(struct hfi1_devdata * dd)799 static int trigger_sbr(struct hfi1_devdata *dd)
800 {
801 struct pci_dev *dev = dd->pcidev;
802 struct pci_dev *pdev;
803
804 /* need a parent */
805 if (!dev->bus->self) {
806 dd_dev_err(dd, "%s: no parent device\n", __func__);
807 return -ENOTTY;
808 }
809
810 /* should not be anyone else on the bus */
811 list_for_each_entry(pdev, &dev->bus->devices, bus_list)
812 if (pdev != dev) {
813 dd_dev_err(dd,
814 "%s: another device is on the same bus\n",
815 __func__);
816 return -ENOTTY;
817 }
818
819 /*
820 * This is an end around to do an SBR during probe time. A new API needs
821 * to be implemented to have cleaner interface but this fixes the
822 * current brokenness
823 */
824 return pci_bridge_secondary_bus_reset(dev->bus->self);
825 }
826
827 /*
828 * Write the given gasket interrupt register.
829 */
write_gasket_interrupt(struct hfi1_devdata * dd,int index,u16 code,u16 data)830 static void write_gasket_interrupt(struct hfi1_devdata *dd, int index,
831 u16 code, u16 data)
832 {
833 write_csr(dd, ASIC_PCIE_SD_INTRPT_LIST + (index * 8),
834 (((u64)code << ASIC_PCIE_SD_INTRPT_LIST_INTRPT_CODE_SHIFT) |
835 ((u64)data << ASIC_PCIE_SD_INTRPT_LIST_INTRPT_DATA_SHIFT)));
836 }
837
838 /*
839 * Tell the gasket logic how to react to the reset.
840 */
arm_gasket_logic(struct hfi1_devdata * dd)841 static void arm_gasket_logic(struct hfi1_devdata *dd)
842 {
843 u64 reg;
844
845 reg = (((u64)1 << dd->hfi1_id) <<
846 ASIC_PCIE_SD_HOST_CMD_INTRPT_CMD_SHIFT) |
847 ((u64)pcie_serdes_broadcast[dd->hfi1_id] <<
848 ASIC_PCIE_SD_HOST_CMD_SBUS_RCVR_ADDR_SHIFT |
849 ASIC_PCIE_SD_HOST_CMD_SBR_MODE_SMASK |
850 ((u64)SBR_DELAY_US & ASIC_PCIE_SD_HOST_CMD_TIMER_MASK) <<
851 ASIC_PCIE_SD_HOST_CMD_TIMER_SHIFT);
852 write_csr(dd, ASIC_PCIE_SD_HOST_CMD, reg);
853 /* read back to push the write */
854 read_csr(dd, ASIC_PCIE_SD_HOST_CMD);
855 }
856
857 /*
858 * CCE_PCIE_CTRL long name helpers
859 * We redefine these shorter macros to use in the code while leaving
860 * chip_registers.h to be autogenerated from the hardware spec.
861 */
862 #define LANE_BUNDLE_MASK CCE_PCIE_CTRL_PCIE_LANE_BUNDLE_MASK
863 #define LANE_BUNDLE_SHIFT CCE_PCIE_CTRL_PCIE_LANE_BUNDLE_SHIFT
864 #define LANE_DELAY_MASK CCE_PCIE_CTRL_PCIE_LANE_DELAY_MASK
865 #define LANE_DELAY_SHIFT CCE_PCIE_CTRL_PCIE_LANE_DELAY_SHIFT
866 #define MARGIN_OVERWRITE_ENABLE_SHIFT CCE_PCIE_CTRL_XMT_MARGIN_OVERWRITE_ENABLE_SHIFT
867 #define MARGIN_SHIFT CCE_PCIE_CTRL_XMT_MARGIN_SHIFT
868 #define MARGIN_G1_G2_OVERWRITE_MASK CCE_PCIE_CTRL_XMT_MARGIN_GEN1_GEN2_OVERWRITE_ENABLE_MASK
869 #define MARGIN_G1_G2_OVERWRITE_SHIFT CCE_PCIE_CTRL_XMT_MARGIN_GEN1_GEN2_OVERWRITE_ENABLE_SHIFT
870 #define MARGIN_GEN1_GEN2_MASK CCE_PCIE_CTRL_XMT_MARGIN_GEN1_GEN2_MASK
871 #define MARGIN_GEN1_GEN2_SHIFT CCE_PCIE_CTRL_XMT_MARGIN_GEN1_GEN2_SHIFT
872
873 /*
874 * Write xmt_margin for full-swing (WFR-B) or half-swing (WFR-C).
875 */
write_xmt_margin(struct hfi1_devdata * dd,const char * fname)876 static void write_xmt_margin(struct hfi1_devdata *dd, const char *fname)
877 {
878 u64 pcie_ctrl;
879 u64 xmt_margin;
880 u64 xmt_margin_oe;
881 u64 lane_delay;
882 u64 lane_bundle;
883
884 pcie_ctrl = read_csr(dd, CCE_PCIE_CTRL);
885
886 /*
887 * For Discrete, use full-swing.
888 * - PCIe TX defaults to full-swing.
889 * Leave this register as default.
890 * For Integrated, use half-swing
891 * - Copy xmt_margin and xmt_margin_oe
892 * from Gen1/Gen2 to Gen3.
893 */
894 if (dd->pcidev->device == PCI_DEVICE_ID_INTEL1) { /* integrated */
895 /* extract initial fields */
896 xmt_margin = (pcie_ctrl >> MARGIN_GEN1_GEN2_SHIFT)
897 & MARGIN_GEN1_GEN2_MASK;
898 xmt_margin_oe = (pcie_ctrl >> MARGIN_G1_G2_OVERWRITE_SHIFT)
899 & MARGIN_G1_G2_OVERWRITE_MASK;
900 lane_delay = (pcie_ctrl >> LANE_DELAY_SHIFT) & LANE_DELAY_MASK;
901 lane_bundle = (pcie_ctrl >> LANE_BUNDLE_SHIFT)
902 & LANE_BUNDLE_MASK;
903
904 /*
905 * For A0, EFUSE values are not set. Override with the
906 * correct values.
907 */
908 if (is_ax(dd)) {
909 /*
910 * xmt_margin and OverwiteEnabel should be the
911 * same for Gen1/Gen2 and Gen3
912 */
913 xmt_margin = 0x5;
914 xmt_margin_oe = 0x1;
915 lane_delay = 0xF; /* Delay 240ns. */
916 lane_bundle = 0x0; /* Set to 1 lane. */
917 }
918
919 /* overwrite existing values */
920 pcie_ctrl = (xmt_margin << MARGIN_GEN1_GEN2_SHIFT)
921 | (xmt_margin_oe << MARGIN_G1_G2_OVERWRITE_SHIFT)
922 | (xmt_margin << MARGIN_SHIFT)
923 | (xmt_margin_oe << MARGIN_OVERWRITE_ENABLE_SHIFT)
924 | (lane_delay << LANE_DELAY_SHIFT)
925 | (lane_bundle << LANE_BUNDLE_SHIFT);
926
927 write_csr(dd, CCE_PCIE_CTRL, pcie_ctrl);
928 }
929
930 dd_dev_dbg(dd, "%s: program XMT margin, CcePcieCtrl 0x%llx\n",
931 fname, pcie_ctrl);
932 }
933
934 /*
935 * Do all the steps needed to transition the PCIe link to Gen3 speed.
936 */
do_pcie_gen3_transition(struct hfi1_devdata * dd)937 int do_pcie_gen3_transition(struct hfi1_devdata *dd)
938 {
939 struct pci_dev *parent = dd->pcidev->bus->self;
940 u64 fw_ctrl;
941 u64 reg, therm;
942 u32 reg32, fs, lf;
943 u32 status, err;
944 int ret;
945 int do_retry, retry_count = 0;
946 int intnum = 0;
947 uint default_pset;
948 uint pset = pcie_pset;
949 u16 target_vector, target_speed;
950 u16 lnkctl2, vendor;
951 u8 div;
952 const u8 (*eq)[3];
953 const u8 (*ctle_tunings)[4];
954 uint static_ctle_mode;
955 int return_error = 0;
956 u32 target_width;
957
958 /* PCIe Gen3 is for the ASIC only */
959 if (dd->icode != ICODE_RTL_SILICON)
960 return 0;
961
962 if (pcie_target == 1) { /* target Gen1 */
963 target_vector = PCI_EXP_LNKCTL2_TLS_2_5GT;
964 target_speed = 2500;
965 } else if (pcie_target == 2) { /* target Gen2 */
966 target_vector = PCI_EXP_LNKCTL2_TLS_5_0GT;
967 target_speed = 5000;
968 } else if (pcie_target == 3) { /* target Gen3 */
969 target_vector = PCI_EXP_LNKCTL2_TLS_8_0GT;
970 target_speed = 8000;
971 } else {
972 /* off or invalid target - skip */
973 dd_dev_info(dd, "%s: Skipping PCIe transition\n", __func__);
974 return 0;
975 }
976
977 /* if already at target speed, done (unless forced) */
978 if (dd->lbus_speed == target_speed) {
979 dd_dev_info(dd, "%s: PCIe already at gen%d, %s\n", __func__,
980 pcie_target,
981 pcie_force ? "re-doing anyway" : "skipping");
982 if (!pcie_force)
983 return 0;
984 }
985
986 /*
987 * The driver cannot do the transition if it has no access to the
988 * upstream component
989 */
990 if (!parent) {
991 dd_dev_info(dd, "%s: No upstream, Can't do gen3 transition\n",
992 __func__);
993 return 0;
994 }
995
996 /* Previous Gen1/Gen2 bus width */
997 target_width = dd->lbus_width;
998
999 /*
1000 * Do the Gen3 transition. Steps are those of the PCIe Gen3
1001 * recipe.
1002 */
1003
1004 /* step 1: pcie link working in gen1/gen2 */
1005
1006 /* step 2: if either side is not capable of Gen3, done */
1007 if (pcie_target == 3 && !dd->link_gen3_capable) {
1008 dd_dev_err(dd, "The PCIe link is not Gen3 capable\n");
1009 ret = -ENOSYS;
1010 goto done_no_mutex;
1011 }
1012
1013 /* hold the SBus resource across the firmware download and SBR */
1014 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
1015 if (ret) {
1016 dd_dev_err(dd, "%s: unable to acquire SBus resource\n",
1017 __func__);
1018 return ret;
1019 }
1020
1021 /* make sure thermal polling is not causing interrupts */
1022 therm = read_csr(dd, ASIC_CFG_THERM_POLL_EN);
1023 if (therm) {
1024 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);
1025 msleep(100);
1026 dd_dev_info(dd, "%s: Disabled therm polling\n",
1027 __func__);
1028 }
1029
1030 retry:
1031 /* the SBus download will reset the spico for thermal */
1032
1033 /* step 3: download SBus Master firmware */
1034 /* step 4: download PCIe Gen3 SerDes firmware */
1035 dd_dev_info(dd, "%s: downloading firmware\n", __func__);
1036 ret = load_pcie_firmware(dd);
1037 if (ret) {
1038 /* do not proceed if the firmware cannot be downloaded */
1039 return_error = 1;
1040 goto done;
1041 }
1042
1043 /* step 5: set up device parameter settings */
1044 dd_dev_info(dd, "%s: setting PCIe registers\n", __func__);
1045
1046 /*
1047 * PcieCfgSpcie1 - Link Control 3
1048 * Leave at reset value. No need to set PerfEq - link equalization
1049 * will be performed automatically after the SBR when the target
1050 * speed is 8GT/s.
1051 */
1052
1053 /* clear all 16 per-lane error bits (PCIe: Lane Error Status) */
1054 pci_write_config_dword(dd->pcidev, PCIE_CFG_SPCIE2, 0xffff);
1055
1056 /* step 5a: Set Synopsys Port Logic registers */
1057
1058 /*
1059 * PcieCfgRegPl2 - Port Force Link
1060 *
1061 * Set the low power field to 0x10 to avoid unnecessary power
1062 * management messages. All other fields are zero.
1063 */
1064 reg32 = 0x10ul << PCIE_CFG_REG_PL2_LOW_PWR_ENT_CNT_SHIFT;
1065 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL2, reg32);
1066
1067 /*
1068 * PcieCfgRegPl100 - Gen3 Control
1069 *
1070 * turn off PcieCfgRegPl100.Gen3ZRxDcNonCompl
1071 * turn on PcieCfgRegPl100.EqEieosCnt
1072 * Everything else zero.
1073 */
1074 reg32 = PCIE_CFG_REG_PL100_EQ_EIEOS_CNT_SMASK;
1075 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL100, reg32);
1076
1077 /*
1078 * PcieCfgRegPl101 - Gen3 EQ FS and LF
1079 * PcieCfgRegPl102 - Gen3 EQ Presets to Coefficients Mapping
1080 * PcieCfgRegPl103 - Gen3 EQ Preset Index
1081 * PcieCfgRegPl105 - Gen3 EQ Status
1082 *
1083 * Give initial EQ settings.
1084 */
1085 if (dd->pcidev->device == PCI_DEVICE_ID_INTEL0) { /* discrete */
1086 /* 1000mV, FS=24, LF = 8 */
1087 fs = 24;
1088 lf = 8;
1089 div = 3;
1090 eq = discrete_preliminary_eq;
1091 default_pset = DEFAULT_DISCRETE_PSET;
1092 ctle_tunings = discrete_ctle_tunings;
1093 /* bit 0 - discrete on/off */
1094 static_ctle_mode = pcie_ctle & 0x1;
1095 } else {
1096 /* 400mV, FS=29, LF = 9 */
1097 fs = 29;
1098 lf = 9;
1099 div = 1;
1100 eq = integrated_preliminary_eq;
1101 default_pset = DEFAULT_MCP_PSET;
1102 ctle_tunings = integrated_ctle_tunings;
1103 /* bit 1 - integrated on/off */
1104 static_ctle_mode = (pcie_ctle >> 1) & 0x1;
1105 }
1106 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL101,
1107 (fs <<
1108 PCIE_CFG_REG_PL101_GEN3_EQ_LOCAL_FS_SHIFT) |
1109 (lf <<
1110 PCIE_CFG_REG_PL101_GEN3_EQ_LOCAL_LF_SHIFT));
1111 ret = load_eq_table(dd, eq, fs, div);
1112 if (ret)
1113 goto done;
1114
1115 /*
1116 * PcieCfgRegPl106 - Gen3 EQ Control
1117 *
1118 * Set Gen3EqPsetReqVec, leave other fields 0.
1119 */
1120 if (pset == UNSET_PSET)
1121 pset = default_pset;
1122 if (pset > 10) { /* valid range is 0-10, inclusive */
1123 dd_dev_err(dd, "%s: Invalid Eq Pset %u, setting to %d\n",
1124 __func__, pset, default_pset);
1125 pset = default_pset;
1126 }
1127 dd_dev_info(dd, "%s: using EQ Pset %u\n", __func__, pset);
1128 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL106,
1129 ((1 << pset) <<
1130 PCIE_CFG_REG_PL106_GEN3_EQ_PSET_REQ_VEC_SHIFT) |
1131 PCIE_CFG_REG_PL106_GEN3_EQ_EVAL2MS_DISABLE_SMASK |
1132 PCIE_CFG_REG_PL106_GEN3_EQ_PHASE23_EXIT_MODE_SMASK);
1133
1134 /*
1135 * step 5b: Do post firmware download steps via SBus
1136 */
1137 dd_dev_info(dd, "%s: doing pcie post steps\n", __func__);
1138 pcie_post_steps(dd);
1139
1140 /*
1141 * step 5c: Program gasket interrupts
1142 */
1143 /* set the Rx Bit Rate to REFCLK ratio */
1144 write_gasket_interrupt(dd, intnum++, 0x0006, 0x0050);
1145 /* disable pCal for PCIe Gen3 RX equalization */
1146 /* select adaptive or static CTLE */
1147 write_gasket_interrupt(dd, intnum++, 0x0026,
1148 0x5b01 | (static_ctle_mode << 3));
1149 /*
1150 * Enable iCal for PCIe Gen3 RX equalization, and set which
1151 * evaluation of RX_EQ_EVAL will launch the iCal procedure.
1152 */
1153 write_gasket_interrupt(dd, intnum++, 0x0026, 0x5202);
1154
1155 if (static_ctle_mode) {
1156 /* apply static CTLE tunings */
1157 u8 pcie_dc, pcie_lf, pcie_hf, pcie_bw;
1158
1159 pcie_dc = ctle_tunings[pset][0];
1160 pcie_lf = ctle_tunings[pset][1];
1161 pcie_hf = ctle_tunings[pset][2];
1162 pcie_bw = ctle_tunings[pset][3];
1163 write_gasket_interrupt(dd, intnum++, 0x0026, 0x0200 | pcie_dc);
1164 write_gasket_interrupt(dd, intnum++, 0x0026, 0x0100 | pcie_lf);
1165 write_gasket_interrupt(dd, intnum++, 0x0026, 0x0000 | pcie_hf);
1166 write_gasket_interrupt(dd, intnum++, 0x0026, 0x5500 | pcie_bw);
1167 }
1168
1169 /* terminate list */
1170 write_gasket_interrupt(dd, intnum++, 0x0000, 0x0000);
1171
1172 /*
1173 * step 5d: program XMT margin
1174 */
1175 write_xmt_margin(dd, __func__);
1176
1177 /*
1178 * step 5e: disable active state power management (ASPM). It
1179 * will be enabled if required later
1180 */
1181 dd_dev_info(dd, "%s: clearing ASPM\n", __func__);
1182 aspm_hw_disable_l1(dd);
1183
1184 /*
1185 * step 5f: clear DirectSpeedChange
1186 * PcieCfgRegPl67.DirectSpeedChange must be zero to prevent the
1187 * change in the speed target from starting before we are ready.
1188 * This field defaults to 0 and we are not changing it, so nothing
1189 * needs to be done.
1190 */
1191
1192 /* step 5g: Set target link speed */
1193 /*
1194 * Set target link speed to be target on both device and parent.
1195 * On setting the parent: Some system BIOSs "helpfully" set the
1196 * parent target speed to Gen2 to match the ASIC's initial speed.
1197 * We can set the target Gen3 because we have already checked
1198 * that it is Gen3 capable earlier.
1199 */
1200 dd_dev_info(dd, "%s: setting parent target link speed\n", __func__);
1201 ret = pcie_capability_read_word(parent, PCI_EXP_LNKCTL2, &lnkctl2);
1202 if (ret) {
1203 dd_dev_err(dd, "Unable to read from PCI config\n");
1204 return_error = 1;
1205 goto done;
1206 }
1207
1208 dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__,
1209 (u32)lnkctl2);
1210 /* only write to parent if target is not as high as ours */
1211 if ((lnkctl2 & PCI_EXP_LNKCTL2_TLS) < target_vector) {
1212 lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS;
1213 lnkctl2 |= target_vector;
1214 dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__,
1215 (u32)lnkctl2);
1216 ret = pcie_capability_write_word(parent,
1217 PCI_EXP_LNKCTL2, lnkctl2);
1218 if (ret) {
1219 dd_dev_err(dd, "Unable to write to PCI config\n");
1220 return_error = 1;
1221 goto done;
1222 }
1223 } else {
1224 dd_dev_info(dd, "%s: ..target speed is OK\n", __func__);
1225 }
1226
1227 dd_dev_info(dd, "%s: setting target link speed\n", __func__);
1228 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKCTL2, &lnkctl2);
1229 if (ret) {
1230 dd_dev_err(dd, "Unable to read from PCI config\n");
1231 return_error = 1;
1232 goto done;
1233 }
1234
1235 dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__,
1236 (u32)lnkctl2);
1237 lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS;
1238 lnkctl2 |= target_vector;
1239 dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__,
1240 (u32)lnkctl2);
1241 ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_LNKCTL2, lnkctl2);
1242 if (ret) {
1243 dd_dev_err(dd, "Unable to write to PCI config\n");
1244 return_error = 1;
1245 goto done;
1246 }
1247
1248 /* step 5h: arm gasket logic */
1249 /* hold DC in reset across the SBR */
1250 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);
1251 (void)read_csr(dd, CCE_DC_CTRL); /* DC reset hold */
1252 /* save firmware control across the SBR */
1253 fw_ctrl = read_csr(dd, MISC_CFG_FW_CTRL);
1254
1255 dd_dev_info(dd, "%s: arming gasket logic\n", __func__);
1256 arm_gasket_logic(dd);
1257
1258 /*
1259 * step 6: quiesce PCIe link
1260 * The chip has already been reset, so there will be no traffic
1261 * from the chip. Linux has no easy way to enforce that it will
1262 * not try to access the device, so we just need to hope it doesn't
1263 * do it while we are doing the reset.
1264 */
1265
1266 /*
1267 * step 7: initiate the secondary bus reset (SBR)
1268 * step 8: hardware brings the links back up
1269 * step 9: wait for link speed transition to be complete
1270 */
1271 dd_dev_info(dd, "%s: calling trigger_sbr\n", __func__);
1272 ret = trigger_sbr(dd);
1273 if (ret)
1274 goto done;
1275
1276 /* step 10: decide what to do next */
1277
1278 /* check if we can read PCI space */
1279 ret = pci_read_config_word(dd->pcidev, PCI_VENDOR_ID, &vendor);
1280 if (ret) {
1281 dd_dev_info(dd,
1282 "%s: read of VendorID failed after SBR, err %d\n",
1283 __func__, ret);
1284 return_error = 1;
1285 goto done;
1286 }
1287 if (vendor == 0xffff) {
1288 dd_dev_info(dd, "%s: VendorID is all 1s after SBR\n", __func__);
1289 return_error = 1;
1290 ret = -EIO;
1291 goto done;
1292 }
1293
1294 /* restore PCI space registers we know were reset */
1295 dd_dev_info(dd, "%s: calling restore_pci_variables\n", __func__);
1296 ret = restore_pci_variables(dd);
1297 if (ret) {
1298 dd_dev_err(dd, "%s: Could not restore PCI variables\n",
1299 __func__);
1300 return_error = 1;
1301 goto done;
1302 }
1303
1304 /* restore firmware control */
1305 write_csr(dd, MISC_CFG_FW_CTRL, fw_ctrl);
1306
1307 /*
1308 * Check the gasket block status.
1309 *
1310 * This is the first CSR read after the SBR. If the read returns
1311 * all 1s (fails), the link did not make it back.
1312 *
1313 * Once we're sure we can read and write, clear the DC reset after
1314 * the SBR. Then check for any per-lane errors. Then look over
1315 * the status.
1316 */
1317 reg = read_csr(dd, ASIC_PCIE_SD_HOST_STATUS);
1318 dd_dev_info(dd, "%s: gasket block status: 0x%llx\n", __func__, reg);
1319 if (reg == ~0ull) { /* PCIe read failed/timeout */
1320 dd_dev_err(dd, "SBR failed - unable to read from device\n");
1321 return_error = 1;
1322 ret = -ENOSYS;
1323 goto done;
1324 }
1325
1326 /* clear the DC reset */
1327 write_csr(dd, CCE_DC_CTRL, 0);
1328
1329 /* Set the LED off */
1330 setextled(dd, 0);
1331
1332 /* check for any per-lane errors */
1333 ret = pci_read_config_dword(dd->pcidev, PCIE_CFG_SPCIE2, ®32);
1334 if (ret) {
1335 dd_dev_err(dd, "Unable to read from PCI config\n");
1336 return_error = 1;
1337 goto done;
1338 }
1339
1340 dd_dev_info(dd, "%s: per-lane errors: 0x%x\n", __func__, reg32);
1341
1342 /* extract status, look for our HFI */
1343 status = (reg >> ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_STS_SHIFT)
1344 & ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_STS_MASK;
1345 if ((status & (1 << dd->hfi1_id)) == 0) {
1346 dd_dev_err(dd,
1347 "%s: gasket status 0x%x, expecting 0x%x\n",
1348 __func__, status, 1 << dd->hfi1_id);
1349 ret = -EIO;
1350 goto done;
1351 }
1352
1353 /* extract error */
1354 err = (reg >> ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_ERR_SHIFT)
1355 & ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_ERR_MASK;
1356 if (err) {
1357 dd_dev_err(dd, "%s: gasket error %d\n", __func__, err);
1358 ret = -EIO;
1359 goto done;
1360 }
1361
1362 /* update our link information cache */
1363 update_lbus_info(dd);
1364 dd_dev_info(dd, "%s: new speed and width: %s\n", __func__,
1365 dd->lbus_info);
1366
1367 if (dd->lbus_speed != target_speed ||
1368 dd->lbus_width < target_width) { /* not target */
1369 /* maybe retry */
1370 do_retry = retry_count < pcie_retry;
1371 dd_dev_err(dd, "PCIe link speed or width did not match target%s\n",
1372 do_retry ? ", retrying" : "");
1373 retry_count++;
1374 if (do_retry) {
1375 msleep(100); /* allow time to settle */
1376 goto retry;
1377 }
1378 ret = -EIO;
1379 }
1380
1381 done:
1382 if (therm) {
1383 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);
1384 msleep(100);
1385 dd_dev_info(dd, "%s: Re-enable therm polling\n",
1386 __func__);
1387 }
1388 release_chip_resource(dd, CR_SBUS);
1389 done_no_mutex:
1390 /* return no error if it is OK to be at current speed */
1391 if (ret && !return_error) {
1392 dd_dev_err(dd, "Proceeding at current speed PCIe speed\n");
1393 ret = 0;
1394 }
1395
1396 dd_dev_info(dd, "%s: done\n", __func__);
1397 return ret;
1398 }
1399