1 /*
2 * Copyright (c) 2012, 2013 Intel Corporation. All rights reserved.
3 * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
4 * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include <linux/pci.h>
36 #include <linux/netdevice.h>
37 #include <linux/vmalloc.h>
38 #include <linux/delay.h>
39 #include <linux/idr.h>
40 #include <linux/module.h>
41 #include <linux/printk.h>
42
43 #include "qib.h"
44 #include "qib_common.h"
45 #include "qib_mad.h"
46
47 #undef pr_fmt
48 #define pr_fmt(fmt) QIB_DRV_NAME ": " fmt
49
50 /*
51 * min buffers we want to have per context, after driver
52 */
53 #define QIB_MIN_USER_CTXT_BUFCNT 7
54
55 #define QLOGIC_IB_R_SOFTWARE_MASK 0xFF
56 #define QLOGIC_IB_R_SOFTWARE_SHIFT 24
57 #define QLOGIC_IB_R_EMULATOR_MASK (1ULL<<62)
58
59 /*
60 * Number of ctxts we are configured to use (to allow for more pio
61 * buffers per ctxt, etc.) Zero means use chip value.
62 */
63 ushort qib_cfgctxts;
64 module_param_named(cfgctxts, qib_cfgctxts, ushort, S_IRUGO);
65 MODULE_PARM_DESC(cfgctxts, "Set max number of contexts to use");
66
67 /*
68 * If set, do not write to any regs if avoidable, hack to allow
69 * check for deranged default register values.
70 */
71 ushort qib_mini_init;
72 module_param_named(mini_init, qib_mini_init, ushort, S_IRUGO);
73 MODULE_PARM_DESC(mini_init, "If set, do minimal diag init");
74
75 unsigned qib_n_krcv_queues;
76 module_param_named(krcvqs, qib_n_krcv_queues, uint, S_IRUGO);
77 MODULE_PARM_DESC(krcvqs, "number of kernel receive queues per IB port");
78
79 unsigned qib_cc_table_size;
80 module_param_named(cc_table_size, qib_cc_table_size, uint, S_IRUGO);
81 MODULE_PARM_DESC(cc_table_size, "Congestion control table entries 0 (CCA disabled - default), min = 128, max = 1984");
82 /*
83 * qib_wc_pat parameter:
84 * 0 is WC via MTRR
85 * 1 is WC via PAT
86 * If PAT initialization fails, code reverts back to MTRR
87 */
88 unsigned qib_wc_pat = 1; /* default (1) is to use PAT, not MTRR */
89 module_param_named(wc_pat, qib_wc_pat, uint, S_IRUGO);
90 MODULE_PARM_DESC(wc_pat, "enable write-combining via PAT mechanism");
91
92 struct workqueue_struct *qib_cq_wq;
93
94 static void verify_interrupt(unsigned long);
95
96 static struct idr qib_unit_table;
97 u32 qib_cpulist_count;
98 unsigned long *qib_cpulist;
99
100 /* set number of contexts we'll actually use */
qib_set_ctxtcnt(struct qib_devdata * dd)101 void qib_set_ctxtcnt(struct qib_devdata *dd)
102 {
103 if (!qib_cfgctxts) {
104 dd->cfgctxts = dd->first_user_ctxt + num_online_cpus();
105 if (dd->cfgctxts > dd->ctxtcnt)
106 dd->cfgctxts = dd->ctxtcnt;
107 } else if (qib_cfgctxts < dd->num_pports)
108 dd->cfgctxts = dd->ctxtcnt;
109 else if (qib_cfgctxts <= dd->ctxtcnt)
110 dd->cfgctxts = qib_cfgctxts;
111 else
112 dd->cfgctxts = dd->ctxtcnt;
113 dd->freectxts = (dd->first_user_ctxt > dd->cfgctxts) ? 0 :
114 dd->cfgctxts - dd->first_user_ctxt;
115 }
116
117 /*
118 * Common code for creating the receive context array.
119 */
qib_create_ctxts(struct qib_devdata * dd)120 int qib_create_ctxts(struct qib_devdata *dd)
121 {
122 unsigned i;
123 int ret;
124
125 /*
126 * Allocate full ctxtcnt array, rather than just cfgctxts, because
127 * cleanup iterates across all possible ctxts.
128 */
129 dd->rcd = kzalloc(sizeof(*dd->rcd) * dd->ctxtcnt, GFP_KERNEL);
130 if (!dd->rcd) {
131 qib_dev_err(dd,
132 "Unable to allocate ctxtdata array, failing\n");
133 ret = -ENOMEM;
134 goto done;
135 }
136
137 /* create (one or more) kctxt */
138 for (i = 0; i < dd->first_user_ctxt; ++i) {
139 struct qib_pportdata *ppd;
140 struct qib_ctxtdata *rcd;
141
142 if (dd->skip_kctxt_mask & (1 << i))
143 continue;
144
145 ppd = dd->pport + (i % dd->num_pports);
146 rcd = qib_create_ctxtdata(ppd, i);
147 if (!rcd) {
148 qib_dev_err(dd,
149 "Unable to allocate ctxtdata for Kernel ctxt, failing\n");
150 ret = -ENOMEM;
151 goto done;
152 }
153 rcd->pkeys[0] = QIB_DEFAULT_P_KEY;
154 rcd->seq_cnt = 1;
155 }
156 ret = 0;
157 done:
158 return ret;
159 }
160
161 /*
162 * Common code for user and kernel context setup.
163 */
qib_create_ctxtdata(struct qib_pportdata * ppd,u32 ctxt)164 struct qib_ctxtdata *qib_create_ctxtdata(struct qib_pportdata *ppd, u32 ctxt)
165 {
166 struct qib_devdata *dd = ppd->dd;
167 struct qib_ctxtdata *rcd;
168
169 rcd = kzalloc(sizeof(*rcd), GFP_KERNEL);
170 if (rcd) {
171 INIT_LIST_HEAD(&rcd->qp_wait_list);
172 rcd->ppd = ppd;
173 rcd->dd = dd;
174 rcd->cnt = 1;
175 rcd->ctxt = ctxt;
176 dd->rcd[ctxt] = rcd;
177
178 dd->f_init_ctxt(rcd);
179
180 /*
181 * To avoid wasting a lot of memory, we allocate 32KB chunks
182 * of physically contiguous memory, advance through it until
183 * used up and then allocate more. Of course, we need
184 * memory to store those extra pointers, now. 32KB seems to
185 * be the most that is "safe" under memory pressure
186 * (creating large files and then copying them over
187 * NFS while doing lots of MPI jobs). The OOM killer can
188 * get invoked, even though we say we can sleep and this can
189 * cause significant system problems....
190 */
191 rcd->rcvegrbuf_size = 0x8000;
192 rcd->rcvegrbufs_perchunk =
193 rcd->rcvegrbuf_size / dd->rcvegrbufsize;
194 rcd->rcvegrbuf_chunks = (rcd->rcvegrcnt +
195 rcd->rcvegrbufs_perchunk - 1) /
196 rcd->rcvegrbufs_perchunk;
197 BUG_ON(!is_power_of_2(rcd->rcvegrbufs_perchunk));
198 rcd->rcvegrbufs_perchunk_shift =
199 ilog2(rcd->rcvegrbufs_perchunk);
200 }
201 return rcd;
202 }
203
204 /*
205 * Common code for initializing the physical port structure.
206 */
qib_init_pportdata(struct qib_pportdata * ppd,struct qib_devdata * dd,u8 hw_pidx,u8 port)207 void qib_init_pportdata(struct qib_pportdata *ppd, struct qib_devdata *dd,
208 u8 hw_pidx, u8 port)
209 {
210 int size;
211 ppd->dd = dd;
212 ppd->hw_pidx = hw_pidx;
213 ppd->port = port; /* IB port number, not index */
214
215 spin_lock_init(&ppd->sdma_lock);
216 spin_lock_init(&ppd->lflags_lock);
217 init_waitqueue_head(&ppd->state_wait);
218
219 init_timer(&ppd->symerr_clear_timer);
220 ppd->symerr_clear_timer.function = qib_clear_symerror_on_linkup;
221 ppd->symerr_clear_timer.data = (unsigned long)ppd;
222
223 ppd->qib_wq = NULL;
224
225 spin_lock_init(&ppd->cc_shadow_lock);
226
227 if (qib_cc_table_size < IB_CCT_MIN_ENTRIES)
228 goto bail;
229
230 ppd->cc_supported_table_entries = min(max_t(int, qib_cc_table_size,
231 IB_CCT_MIN_ENTRIES), IB_CCT_ENTRIES*IB_CC_TABLE_CAP_DEFAULT);
232
233 ppd->cc_max_table_entries =
234 ppd->cc_supported_table_entries/IB_CCT_ENTRIES;
235
236 size = IB_CC_TABLE_CAP_DEFAULT * sizeof(struct ib_cc_table_entry)
237 * IB_CCT_ENTRIES;
238 ppd->ccti_entries = kzalloc(size, GFP_KERNEL);
239 if (!ppd->ccti_entries) {
240 qib_dev_err(dd,
241 "failed to allocate congestion control table for port %d!\n",
242 port);
243 goto bail;
244 }
245
246 size = IB_CC_CCS_ENTRIES * sizeof(struct ib_cc_congestion_entry);
247 ppd->congestion_entries = kzalloc(size, GFP_KERNEL);
248 if (!ppd->congestion_entries) {
249 qib_dev_err(dd,
250 "failed to allocate congestion setting list for port %d!\n",
251 port);
252 goto bail_1;
253 }
254
255 size = sizeof(struct cc_table_shadow);
256 ppd->ccti_entries_shadow = kzalloc(size, GFP_KERNEL);
257 if (!ppd->ccti_entries_shadow) {
258 qib_dev_err(dd,
259 "failed to allocate shadow ccti list for port %d!\n",
260 port);
261 goto bail_2;
262 }
263
264 size = sizeof(struct ib_cc_congestion_setting_attr);
265 ppd->congestion_entries_shadow = kzalloc(size, GFP_KERNEL);
266 if (!ppd->congestion_entries_shadow) {
267 qib_dev_err(dd,
268 "failed to allocate shadow congestion setting list for port %d!\n",
269 port);
270 goto bail_3;
271 }
272
273 return;
274
275 bail_3:
276 kfree(ppd->ccti_entries_shadow);
277 ppd->ccti_entries_shadow = NULL;
278 bail_2:
279 kfree(ppd->congestion_entries);
280 ppd->congestion_entries = NULL;
281 bail_1:
282 kfree(ppd->ccti_entries);
283 ppd->ccti_entries = NULL;
284 bail:
285 /* User is intentionally disabling the congestion control agent */
286 if (!qib_cc_table_size)
287 return;
288
289 if (qib_cc_table_size < IB_CCT_MIN_ENTRIES) {
290 qib_cc_table_size = 0;
291 qib_dev_err(dd,
292 "Congestion Control table size %d less than minimum %d for port %d\n",
293 qib_cc_table_size, IB_CCT_MIN_ENTRIES, port);
294 }
295
296 qib_dev_err(dd, "Congestion Control Agent disabled for port %d\n",
297 port);
298 return;
299 }
300
init_pioavailregs(struct qib_devdata * dd)301 static int init_pioavailregs(struct qib_devdata *dd)
302 {
303 int ret, pidx;
304 u64 *status_page;
305
306 dd->pioavailregs_dma = dma_alloc_coherent(
307 &dd->pcidev->dev, PAGE_SIZE, &dd->pioavailregs_phys,
308 GFP_KERNEL);
309 if (!dd->pioavailregs_dma) {
310 qib_dev_err(dd,
311 "failed to allocate PIOavail reg area in memory\n");
312 ret = -ENOMEM;
313 goto done;
314 }
315
316 /*
317 * We really want L2 cache aligned, but for current CPUs of
318 * interest, they are the same.
319 */
320 status_page = (u64 *)
321 ((char *) dd->pioavailregs_dma +
322 ((2 * L1_CACHE_BYTES +
323 dd->pioavregs * sizeof(u64)) & ~L1_CACHE_BYTES));
324 /* device status comes first, for backwards compatibility */
325 dd->devstatusp = status_page;
326 *status_page++ = 0;
327 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
328 dd->pport[pidx].statusp = status_page;
329 *status_page++ = 0;
330 }
331
332 /*
333 * Setup buffer to hold freeze and other messages, accessible to
334 * apps, following statusp. This is per-unit, not per port.
335 */
336 dd->freezemsg = (char *) status_page;
337 *dd->freezemsg = 0;
338 /* length of msg buffer is "whatever is left" */
339 ret = (char *) status_page - (char *) dd->pioavailregs_dma;
340 dd->freezelen = PAGE_SIZE - ret;
341
342 ret = 0;
343
344 done:
345 return ret;
346 }
347
348 /**
349 * init_shadow_tids - allocate the shadow TID array
350 * @dd: the qlogic_ib device
351 *
352 * allocate the shadow TID array, so we can qib_munlock previous
353 * entries. It may make more sense to move the pageshadow to the
354 * ctxt data structure, so we only allocate memory for ctxts actually
355 * in use, since we at 8k per ctxt, now.
356 * We don't want failures here to prevent use of the driver/chip,
357 * so no return value.
358 */
init_shadow_tids(struct qib_devdata * dd)359 static void init_shadow_tids(struct qib_devdata *dd)
360 {
361 struct page **pages;
362 dma_addr_t *addrs;
363
364 pages = vzalloc(dd->cfgctxts * dd->rcvtidcnt * sizeof(struct page *));
365 if (!pages) {
366 qib_dev_err(dd,
367 "failed to allocate shadow page * array, no expected sends!\n");
368 goto bail;
369 }
370
371 addrs = vzalloc(dd->cfgctxts * dd->rcvtidcnt * sizeof(dma_addr_t));
372 if (!addrs) {
373 qib_dev_err(dd,
374 "failed to allocate shadow dma handle array, no expected sends!\n");
375 goto bail_free;
376 }
377
378 dd->pageshadow = pages;
379 dd->physshadow = addrs;
380 return;
381
382 bail_free:
383 vfree(pages);
384 bail:
385 dd->pageshadow = NULL;
386 }
387
388 /*
389 * Do initialization for device that is only needed on
390 * first detect, not on resets.
391 */
loadtime_init(struct qib_devdata * dd)392 static int loadtime_init(struct qib_devdata *dd)
393 {
394 int ret = 0;
395
396 if (((dd->revision >> QLOGIC_IB_R_SOFTWARE_SHIFT) &
397 QLOGIC_IB_R_SOFTWARE_MASK) != QIB_CHIP_SWVERSION) {
398 qib_dev_err(dd,
399 "Driver only handles version %d, chip swversion is %d (%llx), failng\n",
400 QIB_CHIP_SWVERSION,
401 (int)(dd->revision >>
402 QLOGIC_IB_R_SOFTWARE_SHIFT) &
403 QLOGIC_IB_R_SOFTWARE_MASK,
404 (unsigned long long) dd->revision);
405 ret = -ENOSYS;
406 goto done;
407 }
408
409 if (dd->revision & QLOGIC_IB_R_EMULATOR_MASK)
410 qib_devinfo(dd->pcidev, "%s", dd->boardversion);
411
412 spin_lock_init(&dd->pioavail_lock);
413 spin_lock_init(&dd->sendctrl_lock);
414 spin_lock_init(&dd->uctxt_lock);
415 spin_lock_init(&dd->qib_diag_trans_lock);
416 spin_lock_init(&dd->eep_st_lock);
417 mutex_init(&dd->eep_lock);
418
419 if (qib_mini_init)
420 goto done;
421
422 ret = init_pioavailregs(dd);
423 init_shadow_tids(dd);
424
425 qib_get_eeprom_info(dd);
426
427 /* setup time (don't start yet) to verify we got interrupt */
428 init_timer(&dd->intrchk_timer);
429 dd->intrchk_timer.function = verify_interrupt;
430 dd->intrchk_timer.data = (unsigned long) dd;
431
432 done:
433 return ret;
434 }
435
436 /**
437 * init_after_reset - re-initialize after a reset
438 * @dd: the qlogic_ib device
439 *
440 * sanity check at least some of the values after reset, and
441 * ensure no receive or transmit (explicitly, in case reset
442 * failed
443 */
init_after_reset(struct qib_devdata * dd)444 static int init_after_reset(struct qib_devdata *dd)
445 {
446 int i;
447
448 /*
449 * Ensure chip does no sends or receives, tail updates, or
450 * pioavail updates while we re-initialize. This is mostly
451 * for the driver data structures, not chip registers.
452 */
453 for (i = 0; i < dd->num_pports; ++i) {
454 /*
455 * ctxt == -1 means "all contexts". Only really safe for
456 * _dis_abling things, as here.
457 */
458 dd->f_rcvctrl(dd->pport + i, QIB_RCVCTRL_CTXT_DIS |
459 QIB_RCVCTRL_INTRAVAIL_DIS |
460 QIB_RCVCTRL_TAILUPD_DIS, -1);
461 /* Redundant across ports for some, but no big deal. */
462 dd->f_sendctrl(dd->pport + i, QIB_SENDCTRL_SEND_DIS |
463 QIB_SENDCTRL_AVAIL_DIS);
464 }
465
466 return 0;
467 }
468
enable_chip(struct qib_devdata * dd)469 static void enable_chip(struct qib_devdata *dd)
470 {
471 u64 rcvmask;
472 int i;
473
474 /*
475 * Enable PIO send, and update of PIOavail regs to memory.
476 */
477 for (i = 0; i < dd->num_pports; ++i)
478 dd->f_sendctrl(dd->pport + i, QIB_SENDCTRL_SEND_ENB |
479 QIB_SENDCTRL_AVAIL_ENB);
480 /*
481 * Enable kernel ctxts' receive and receive interrupt.
482 * Other ctxts done as user opens and inits them.
483 */
484 rcvmask = QIB_RCVCTRL_CTXT_ENB | QIB_RCVCTRL_INTRAVAIL_ENB;
485 rcvmask |= (dd->flags & QIB_NODMA_RTAIL) ?
486 QIB_RCVCTRL_TAILUPD_DIS : QIB_RCVCTRL_TAILUPD_ENB;
487 for (i = 0; dd->rcd && i < dd->first_user_ctxt; ++i) {
488 struct qib_ctxtdata *rcd = dd->rcd[i];
489
490 if (rcd)
491 dd->f_rcvctrl(rcd->ppd, rcvmask, i);
492 }
493 }
494
verify_interrupt(unsigned long opaque)495 static void verify_interrupt(unsigned long opaque)
496 {
497 struct qib_devdata *dd = (struct qib_devdata *) opaque;
498
499 if (!dd)
500 return; /* being torn down */
501
502 /*
503 * If we don't have a lid or any interrupts, let the user know and
504 * don't bother checking again.
505 */
506 if (dd->int_counter == 0) {
507 if (!dd->f_intr_fallback(dd))
508 dev_err(&dd->pcidev->dev,
509 "No interrupts detected, not usable.\n");
510 else /* re-arm the timer to see if fallback works */
511 mod_timer(&dd->intrchk_timer, jiffies + HZ/2);
512 }
513 }
514
init_piobuf_state(struct qib_devdata * dd)515 static void init_piobuf_state(struct qib_devdata *dd)
516 {
517 int i, pidx;
518 u32 uctxts;
519
520 /*
521 * Ensure all buffers are free, and fifos empty. Buffers
522 * are common, so only do once for port 0.
523 *
524 * After enable and qib_chg_pioavailkernel so we can safely
525 * enable pioavail updates and PIOENABLE. After this, packets
526 * are ready and able to go out.
527 */
528 dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_ALL);
529 for (pidx = 0; pidx < dd->num_pports; ++pidx)
530 dd->f_sendctrl(dd->pport + pidx, QIB_SENDCTRL_FLUSH);
531
532 /*
533 * If not all sendbufs are used, add the one to each of the lower
534 * numbered contexts. pbufsctxt and lastctxt_piobuf are
535 * calculated in chip-specific code because it may cause some
536 * chip-specific adjustments to be made.
537 */
538 uctxts = dd->cfgctxts - dd->first_user_ctxt;
539 dd->ctxts_extrabuf = dd->pbufsctxt ?
540 dd->lastctxt_piobuf - (dd->pbufsctxt * uctxts) : 0;
541
542 /*
543 * Set up the shadow copies of the piobufavail registers,
544 * which we compare against the chip registers for now, and
545 * the in memory DMA'ed copies of the registers.
546 * By now pioavail updates to memory should have occurred, so
547 * copy them into our working/shadow registers; this is in
548 * case something went wrong with abort, but mostly to get the
549 * initial values of the generation bit correct.
550 */
551 for (i = 0; i < dd->pioavregs; i++) {
552 __le64 tmp;
553
554 tmp = dd->pioavailregs_dma[i];
555 /*
556 * Don't need to worry about pioavailkernel here
557 * because we will call qib_chg_pioavailkernel() later
558 * in initialization, to busy out buffers as needed.
559 */
560 dd->pioavailshadow[i] = le64_to_cpu(tmp);
561 }
562 while (i < ARRAY_SIZE(dd->pioavailshadow))
563 dd->pioavailshadow[i++] = 0; /* for debugging sanity */
564
565 /* after pioavailshadow is setup */
566 qib_chg_pioavailkernel(dd, 0, dd->piobcnt2k + dd->piobcnt4k,
567 TXCHK_CHG_TYPE_KERN, NULL);
568 dd->f_initvl15_bufs(dd);
569 }
570
571 /**
572 * qib_create_workqueues - create per port workqueues
573 * @dd: the qlogic_ib device
574 */
qib_create_workqueues(struct qib_devdata * dd)575 static int qib_create_workqueues(struct qib_devdata *dd)
576 {
577 int pidx;
578 struct qib_pportdata *ppd;
579
580 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
581 ppd = dd->pport + pidx;
582 if (!ppd->qib_wq) {
583 char wq_name[8]; /* 3 + 2 + 1 + 1 + 1 */
584 snprintf(wq_name, sizeof(wq_name), "qib%d_%d",
585 dd->unit, pidx);
586 ppd->qib_wq =
587 create_singlethread_workqueue(wq_name);
588 if (!ppd->qib_wq)
589 goto wq_error;
590 }
591 }
592 return 0;
593 wq_error:
594 pr_err("create_singlethread_workqueue failed for port %d\n",
595 pidx + 1);
596 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
597 ppd = dd->pport + pidx;
598 if (ppd->qib_wq) {
599 destroy_workqueue(ppd->qib_wq);
600 ppd->qib_wq = NULL;
601 }
602 }
603 return -ENOMEM;
604 }
605
606 /**
607 * qib_init - do the actual initialization sequence on the chip
608 * @dd: the qlogic_ib device
609 * @reinit: reinitializing, so don't allocate new memory
610 *
611 * Do the actual initialization sequence on the chip. This is done
612 * both from the init routine called from the PCI infrastructure, and
613 * when we reset the chip, or detect that it was reset internally,
614 * or it's administratively re-enabled.
615 *
616 * Memory allocation here and in called routines is only done in
617 * the first case (reinit == 0). We have to be careful, because even
618 * without memory allocation, we need to re-write all the chip registers
619 * TIDs, etc. after the reset or enable has completed.
620 */
qib_init(struct qib_devdata * dd,int reinit)621 int qib_init(struct qib_devdata *dd, int reinit)
622 {
623 int ret = 0, pidx, lastfail = 0;
624 u32 portok = 0;
625 unsigned i;
626 struct qib_ctxtdata *rcd;
627 struct qib_pportdata *ppd;
628 unsigned long flags;
629
630 /* Set linkstate to unknown, so we can watch for a transition. */
631 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
632 ppd = dd->pport + pidx;
633 spin_lock_irqsave(&ppd->lflags_lock, flags);
634 ppd->lflags &= ~(QIBL_LINKACTIVE | QIBL_LINKARMED |
635 QIBL_LINKDOWN | QIBL_LINKINIT |
636 QIBL_LINKV);
637 spin_unlock_irqrestore(&ppd->lflags_lock, flags);
638 }
639
640 if (reinit)
641 ret = init_after_reset(dd);
642 else
643 ret = loadtime_init(dd);
644 if (ret)
645 goto done;
646
647 /* Bypass most chip-init, to get to device creation */
648 if (qib_mini_init)
649 return 0;
650
651 ret = dd->f_late_initreg(dd);
652 if (ret)
653 goto done;
654
655 /* dd->rcd can be NULL if early init failed */
656 for (i = 0; dd->rcd && i < dd->first_user_ctxt; ++i) {
657 /*
658 * Set up the (kernel) rcvhdr queue and egr TIDs. If doing
659 * re-init, the simplest way to handle this is to free
660 * existing, and re-allocate.
661 * Need to re-create rest of ctxt 0 ctxtdata as well.
662 */
663 rcd = dd->rcd[i];
664 if (!rcd)
665 continue;
666
667 lastfail = qib_create_rcvhdrq(dd, rcd);
668 if (!lastfail)
669 lastfail = qib_setup_eagerbufs(rcd);
670 if (lastfail) {
671 qib_dev_err(dd,
672 "failed to allocate kernel ctxt's rcvhdrq and/or egr bufs\n");
673 continue;
674 }
675 }
676
677 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
678 int mtu;
679 if (lastfail)
680 ret = lastfail;
681 ppd = dd->pport + pidx;
682 mtu = ib_mtu_enum_to_int(qib_ibmtu);
683 if (mtu == -1) {
684 mtu = QIB_DEFAULT_MTU;
685 qib_ibmtu = 0; /* don't leave invalid value */
686 }
687 /* set max we can ever have for this driver load */
688 ppd->init_ibmaxlen = min(mtu > 2048 ?
689 dd->piosize4k : dd->piosize2k,
690 dd->rcvegrbufsize +
691 (dd->rcvhdrentsize << 2));
692 /*
693 * Have to initialize ibmaxlen, but this will normally
694 * change immediately in qib_set_mtu().
695 */
696 ppd->ibmaxlen = ppd->init_ibmaxlen;
697 qib_set_mtu(ppd, mtu);
698
699 spin_lock_irqsave(&ppd->lflags_lock, flags);
700 ppd->lflags |= QIBL_IB_LINK_DISABLED;
701 spin_unlock_irqrestore(&ppd->lflags_lock, flags);
702
703 lastfail = dd->f_bringup_serdes(ppd);
704 if (lastfail) {
705 qib_devinfo(dd->pcidev,
706 "Failed to bringup IB port %u\n", ppd->port);
707 lastfail = -ENETDOWN;
708 continue;
709 }
710
711 portok++;
712 }
713
714 if (!portok) {
715 /* none of the ports initialized */
716 if (!ret && lastfail)
717 ret = lastfail;
718 else if (!ret)
719 ret = -ENETDOWN;
720 /* but continue on, so we can debug cause */
721 }
722
723 enable_chip(dd);
724
725 init_piobuf_state(dd);
726
727 done:
728 if (!ret) {
729 /* chip is OK for user apps; mark it as initialized */
730 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
731 ppd = dd->pport + pidx;
732 /*
733 * Set status even if port serdes is not initialized
734 * so that diags will work.
735 */
736 *ppd->statusp |= QIB_STATUS_CHIP_PRESENT |
737 QIB_STATUS_INITTED;
738 if (!ppd->link_speed_enabled)
739 continue;
740 if (dd->flags & QIB_HAS_SEND_DMA)
741 ret = qib_setup_sdma(ppd);
742 init_timer(&ppd->hol_timer);
743 ppd->hol_timer.function = qib_hol_event;
744 ppd->hol_timer.data = (unsigned long)ppd;
745 ppd->hol_state = QIB_HOL_UP;
746 }
747
748 /* now we can enable all interrupts from the chip */
749 dd->f_set_intr_state(dd, 1);
750
751 /*
752 * Setup to verify we get an interrupt, and fallback
753 * to an alternate if necessary and possible.
754 */
755 mod_timer(&dd->intrchk_timer, jiffies + HZ/2);
756 /* start stats retrieval timer */
757 mod_timer(&dd->stats_timer, jiffies + HZ * ACTIVITY_TIMER);
758 }
759
760 /* if ret is non-zero, we probably should do some cleanup here... */
761 return ret;
762 }
763
764 /*
765 * These next two routines are placeholders in case we don't have per-arch
766 * code for controlling write combining. If explicit control of write
767 * combining is not available, performance will probably be awful.
768 */
769
qib_enable_wc(struct qib_devdata * dd)770 int __attribute__((weak)) qib_enable_wc(struct qib_devdata *dd)
771 {
772 return -EOPNOTSUPP;
773 }
774
qib_disable_wc(struct qib_devdata * dd)775 void __attribute__((weak)) qib_disable_wc(struct qib_devdata *dd)
776 {
777 }
778
__qib_lookup(int unit)779 static inline struct qib_devdata *__qib_lookup(int unit)
780 {
781 return idr_find(&qib_unit_table, unit);
782 }
783
qib_lookup(int unit)784 struct qib_devdata *qib_lookup(int unit)
785 {
786 struct qib_devdata *dd;
787 unsigned long flags;
788
789 spin_lock_irqsave(&qib_devs_lock, flags);
790 dd = __qib_lookup(unit);
791 spin_unlock_irqrestore(&qib_devs_lock, flags);
792
793 return dd;
794 }
795
796 /*
797 * Stop the timers during unit shutdown, or after an error late
798 * in initialization.
799 */
qib_stop_timers(struct qib_devdata * dd)800 static void qib_stop_timers(struct qib_devdata *dd)
801 {
802 struct qib_pportdata *ppd;
803 int pidx;
804
805 if (dd->stats_timer.data) {
806 del_timer_sync(&dd->stats_timer);
807 dd->stats_timer.data = 0;
808 }
809 if (dd->intrchk_timer.data) {
810 del_timer_sync(&dd->intrchk_timer);
811 dd->intrchk_timer.data = 0;
812 }
813 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
814 ppd = dd->pport + pidx;
815 if (ppd->hol_timer.data)
816 del_timer_sync(&ppd->hol_timer);
817 if (ppd->led_override_timer.data) {
818 del_timer_sync(&ppd->led_override_timer);
819 atomic_set(&ppd->led_override_timer_active, 0);
820 }
821 if (ppd->symerr_clear_timer.data)
822 del_timer_sync(&ppd->symerr_clear_timer);
823 }
824 }
825
826 /**
827 * qib_shutdown_device - shut down a device
828 * @dd: the qlogic_ib device
829 *
830 * This is called to make the device quiet when we are about to
831 * unload the driver, and also when the device is administratively
832 * disabled. It does not free any data structures.
833 * Everything it does has to be setup again by qib_init(dd, 1)
834 */
qib_shutdown_device(struct qib_devdata * dd)835 static void qib_shutdown_device(struct qib_devdata *dd)
836 {
837 struct qib_pportdata *ppd;
838 unsigned pidx;
839
840 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
841 ppd = dd->pport + pidx;
842
843 spin_lock_irq(&ppd->lflags_lock);
844 ppd->lflags &= ~(QIBL_LINKDOWN | QIBL_LINKINIT |
845 QIBL_LINKARMED | QIBL_LINKACTIVE |
846 QIBL_LINKV);
847 spin_unlock_irq(&ppd->lflags_lock);
848 *ppd->statusp &= ~(QIB_STATUS_IB_CONF | QIB_STATUS_IB_READY);
849 }
850 dd->flags &= ~QIB_INITTED;
851
852 /* mask interrupts, but not errors */
853 dd->f_set_intr_state(dd, 0);
854
855 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
856 ppd = dd->pport + pidx;
857 dd->f_rcvctrl(ppd, QIB_RCVCTRL_TAILUPD_DIS |
858 QIB_RCVCTRL_CTXT_DIS |
859 QIB_RCVCTRL_INTRAVAIL_DIS |
860 QIB_RCVCTRL_PKEY_ENB, -1);
861 /*
862 * Gracefully stop all sends allowing any in progress to
863 * trickle out first.
864 */
865 dd->f_sendctrl(ppd, QIB_SENDCTRL_CLEAR);
866 }
867
868 /*
869 * Enough for anything that's going to trickle out to have actually
870 * done so.
871 */
872 udelay(20);
873
874 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
875 ppd = dd->pport + pidx;
876 dd->f_setextled(ppd, 0); /* make sure LEDs are off */
877
878 if (dd->flags & QIB_HAS_SEND_DMA)
879 qib_teardown_sdma(ppd);
880
881 dd->f_sendctrl(ppd, QIB_SENDCTRL_AVAIL_DIS |
882 QIB_SENDCTRL_SEND_DIS);
883 /*
884 * Clear SerdesEnable.
885 * We can't count on interrupts since we are stopping.
886 */
887 dd->f_quiet_serdes(ppd);
888
889 if (ppd->qib_wq) {
890 destroy_workqueue(ppd->qib_wq);
891 ppd->qib_wq = NULL;
892 }
893 }
894
895 qib_update_eeprom_log(dd);
896 }
897
898 /**
899 * qib_free_ctxtdata - free a context's allocated data
900 * @dd: the qlogic_ib device
901 * @rcd: the ctxtdata structure
902 *
903 * free up any allocated data for a context
904 * This should not touch anything that would affect a simultaneous
905 * re-allocation of context data, because it is called after qib_mutex
906 * is released (and can be called from reinit as well).
907 * It should never change any chip state, or global driver state.
908 */
qib_free_ctxtdata(struct qib_devdata * dd,struct qib_ctxtdata * rcd)909 void qib_free_ctxtdata(struct qib_devdata *dd, struct qib_ctxtdata *rcd)
910 {
911 if (!rcd)
912 return;
913
914 if (rcd->rcvhdrq) {
915 dma_free_coherent(&dd->pcidev->dev, rcd->rcvhdrq_size,
916 rcd->rcvhdrq, rcd->rcvhdrq_phys);
917 rcd->rcvhdrq = NULL;
918 if (rcd->rcvhdrtail_kvaddr) {
919 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
920 rcd->rcvhdrtail_kvaddr,
921 rcd->rcvhdrqtailaddr_phys);
922 rcd->rcvhdrtail_kvaddr = NULL;
923 }
924 }
925 if (rcd->rcvegrbuf) {
926 unsigned e;
927
928 for (e = 0; e < rcd->rcvegrbuf_chunks; e++) {
929 void *base = rcd->rcvegrbuf[e];
930 size_t size = rcd->rcvegrbuf_size;
931
932 dma_free_coherent(&dd->pcidev->dev, size,
933 base, rcd->rcvegrbuf_phys[e]);
934 }
935 kfree(rcd->rcvegrbuf);
936 rcd->rcvegrbuf = NULL;
937 kfree(rcd->rcvegrbuf_phys);
938 rcd->rcvegrbuf_phys = NULL;
939 rcd->rcvegrbuf_chunks = 0;
940 }
941
942 kfree(rcd->tid_pg_list);
943 vfree(rcd->user_event_mask);
944 vfree(rcd->subctxt_uregbase);
945 vfree(rcd->subctxt_rcvegrbuf);
946 vfree(rcd->subctxt_rcvhdr_base);
947 kfree(rcd);
948 }
949
950 /*
951 * Perform a PIO buffer bandwidth write test, to verify proper system
952 * configuration. Even when all the setup calls work, occasionally
953 * BIOS or other issues can prevent write combining from working, or
954 * can cause other bandwidth problems to the chip.
955 *
956 * This test simply writes the same buffer over and over again, and
957 * measures close to the peak bandwidth to the chip (not testing
958 * data bandwidth to the wire). On chips that use an address-based
959 * trigger to send packets to the wire, this is easy. On chips that
960 * use a count to trigger, we want to make sure that the packet doesn't
961 * go out on the wire, or trigger flow control checks.
962 */
qib_verify_pioperf(struct qib_devdata * dd)963 static void qib_verify_pioperf(struct qib_devdata *dd)
964 {
965 u32 pbnum, cnt, lcnt;
966 u32 __iomem *piobuf;
967 u32 *addr;
968 u64 msecs, emsecs;
969
970 piobuf = dd->f_getsendbuf(dd->pport, 0ULL, &pbnum);
971 if (!piobuf) {
972 qib_devinfo(dd->pcidev,
973 "No PIObufs for checking perf, skipping\n");
974 return;
975 }
976
977 /*
978 * Enough to give us a reasonable test, less than piobuf size, and
979 * likely multiple of store buffer length.
980 */
981 cnt = 1024;
982
983 addr = vmalloc(cnt);
984 if (!addr) {
985 qib_devinfo(dd->pcidev,
986 "Couldn't get memory for checking PIO perf,"
987 " skipping\n");
988 goto done;
989 }
990
991 preempt_disable(); /* we want reasonably accurate elapsed time */
992 msecs = 1 + jiffies_to_msecs(jiffies);
993 for (lcnt = 0; lcnt < 10000U; lcnt++) {
994 /* wait until we cross msec boundary */
995 if (jiffies_to_msecs(jiffies) >= msecs)
996 break;
997 udelay(1);
998 }
999
1000 dd->f_set_armlaunch(dd, 0);
1001
1002 /*
1003 * length 0, no dwords actually sent
1004 */
1005 writeq(0, piobuf);
1006 qib_flush_wc();
1007
1008 /*
1009 * This is only roughly accurate, since even with preempt we
1010 * still take interrupts that could take a while. Running for
1011 * >= 5 msec seems to get us "close enough" to accurate values.
1012 */
1013 msecs = jiffies_to_msecs(jiffies);
1014 for (emsecs = lcnt = 0; emsecs <= 5UL; lcnt++) {
1015 qib_pio_copy(piobuf + 64, addr, cnt >> 2);
1016 emsecs = jiffies_to_msecs(jiffies) - msecs;
1017 }
1018
1019 /* 1 GiB/sec, slightly over IB SDR line rate */
1020 if (lcnt < (emsecs * 1024U))
1021 qib_dev_err(dd,
1022 "Performance problem: bandwidth to PIO buffers is only %u MiB/sec\n",
1023 lcnt / (u32) emsecs);
1024
1025 preempt_enable();
1026
1027 vfree(addr);
1028
1029 done:
1030 /* disarm piobuf, so it's available again */
1031 dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_BUF(pbnum));
1032 qib_sendbuf_done(dd, pbnum);
1033 dd->f_set_armlaunch(dd, 1);
1034 }
1035
1036
qib_free_devdata(struct qib_devdata * dd)1037 void qib_free_devdata(struct qib_devdata *dd)
1038 {
1039 unsigned long flags;
1040
1041 spin_lock_irqsave(&qib_devs_lock, flags);
1042 idr_remove(&qib_unit_table, dd->unit);
1043 list_del(&dd->list);
1044 spin_unlock_irqrestore(&qib_devs_lock, flags);
1045
1046 ib_dealloc_device(&dd->verbs_dev.ibdev);
1047 }
1048
1049 /*
1050 * Allocate our primary per-unit data structure. Must be done via verbs
1051 * allocator, because the verbs cleanup process both does cleanup and
1052 * free of the data structure.
1053 * "extra" is for chip-specific data.
1054 *
1055 * Use the idr mechanism to get a unit number for this unit.
1056 */
qib_alloc_devdata(struct pci_dev * pdev,size_t extra)1057 struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra)
1058 {
1059 unsigned long flags;
1060 struct qib_devdata *dd;
1061 int ret;
1062
1063 dd = (struct qib_devdata *) ib_alloc_device(sizeof(*dd) + extra);
1064 if (!dd) {
1065 dd = ERR_PTR(-ENOMEM);
1066 goto bail;
1067 }
1068
1069 idr_preload(GFP_KERNEL);
1070 spin_lock_irqsave(&qib_devs_lock, flags);
1071
1072 ret = idr_alloc(&qib_unit_table, dd, 0, 0, GFP_NOWAIT);
1073 if (ret >= 0) {
1074 dd->unit = ret;
1075 list_add(&dd->list, &qib_dev_list);
1076 }
1077
1078 spin_unlock_irqrestore(&qib_devs_lock, flags);
1079 idr_preload_end();
1080
1081 if (ret < 0) {
1082 qib_early_err(&pdev->dev,
1083 "Could not allocate unit ID: error %d\n", -ret);
1084 ib_dealloc_device(&dd->verbs_dev.ibdev);
1085 dd = ERR_PTR(ret);
1086 goto bail;
1087 }
1088
1089 if (!qib_cpulist_count) {
1090 u32 count = num_online_cpus();
1091 qib_cpulist = kzalloc(BITS_TO_LONGS(count) *
1092 sizeof(long), GFP_KERNEL);
1093 if (qib_cpulist)
1094 qib_cpulist_count = count;
1095 else
1096 qib_early_err(&pdev->dev,
1097 "Could not alloc cpulist info, cpu affinity might be wrong\n");
1098 }
1099
1100 bail:
1101 return dd;
1102 }
1103
1104 /*
1105 * Called from freeze mode handlers, and from PCI error
1106 * reporting code. Should be paranoid about state of
1107 * system and data structures.
1108 */
qib_disable_after_error(struct qib_devdata * dd)1109 void qib_disable_after_error(struct qib_devdata *dd)
1110 {
1111 if (dd->flags & QIB_INITTED) {
1112 u32 pidx;
1113
1114 dd->flags &= ~QIB_INITTED;
1115 if (dd->pport)
1116 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1117 struct qib_pportdata *ppd;
1118
1119 ppd = dd->pport + pidx;
1120 if (dd->flags & QIB_PRESENT) {
1121 qib_set_linkstate(ppd,
1122 QIB_IB_LINKDOWN_DISABLE);
1123 dd->f_setextled(ppd, 0);
1124 }
1125 *ppd->statusp &= ~QIB_STATUS_IB_READY;
1126 }
1127 }
1128
1129 /*
1130 * Mark as having had an error for driver, and also
1131 * for /sys and status word mapped to user programs.
1132 * This marks unit as not usable, until reset.
1133 */
1134 if (dd->devstatusp)
1135 *dd->devstatusp |= QIB_STATUS_HWERROR;
1136 }
1137
1138 static void qib_remove_one(struct pci_dev *);
1139 static int qib_init_one(struct pci_dev *, const struct pci_device_id *);
1140
1141 #define DRIVER_LOAD_MSG "Intel " QIB_DRV_NAME " loaded: "
1142 #define PFX QIB_DRV_NAME ": "
1143
1144 static DEFINE_PCI_DEVICE_TABLE(qib_pci_tbl) = {
1145 { PCI_DEVICE(PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_QLOGIC_IB_6120) },
1146 { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_IB_7220) },
1147 { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_IB_7322) },
1148 { 0, }
1149 };
1150
1151 MODULE_DEVICE_TABLE(pci, qib_pci_tbl);
1152
1153 struct pci_driver qib_driver = {
1154 .name = QIB_DRV_NAME,
1155 .probe = qib_init_one,
1156 .remove = qib_remove_one,
1157 .id_table = qib_pci_tbl,
1158 .err_handler = &qib_pci_err_handler,
1159 };
1160
1161 /*
1162 * Do all the generic driver unit- and chip-independent memory
1163 * allocation and initialization.
1164 */
qlogic_ib_init(void)1165 static int __init qlogic_ib_init(void)
1166 {
1167 int ret;
1168
1169 ret = qib_dev_init();
1170 if (ret)
1171 goto bail;
1172
1173 qib_cq_wq = create_singlethread_workqueue("qib_cq");
1174 if (!qib_cq_wq) {
1175 ret = -ENOMEM;
1176 goto bail_dev;
1177 }
1178
1179 /*
1180 * These must be called before the driver is registered with
1181 * the PCI subsystem.
1182 */
1183 idr_init(&qib_unit_table);
1184
1185 ret = pci_register_driver(&qib_driver);
1186 if (ret < 0) {
1187 pr_err("Unable to register driver: error %d\n", -ret);
1188 goto bail_unit;
1189 }
1190
1191 /* not fatal if it doesn't work */
1192 if (qib_init_qibfs())
1193 pr_err("Unable to register ipathfs\n");
1194 goto bail; /* all OK */
1195
1196 bail_unit:
1197 idr_destroy(&qib_unit_table);
1198 destroy_workqueue(qib_cq_wq);
1199 bail_dev:
1200 qib_dev_cleanup();
1201 bail:
1202 return ret;
1203 }
1204
1205 module_init(qlogic_ib_init);
1206
1207 /*
1208 * Do the non-unit driver cleanup, memory free, etc. at unload.
1209 */
qlogic_ib_cleanup(void)1210 static void __exit qlogic_ib_cleanup(void)
1211 {
1212 int ret;
1213
1214 ret = qib_exit_qibfs();
1215 if (ret)
1216 pr_err(
1217 "Unable to cleanup counter filesystem: error %d\n",
1218 -ret);
1219
1220 pci_unregister_driver(&qib_driver);
1221
1222 destroy_workqueue(qib_cq_wq);
1223
1224 qib_cpulist_count = 0;
1225 kfree(qib_cpulist);
1226
1227 idr_destroy(&qib_unit_table);
1228 qib_dev_cleanup();
1229 }
1230
1231 module_exit(qlogic_ib_cleanup);
1232
1233 /* this can only be called after a successful initialization */
cleanup_device_data(struct qib_devdata * dd)1234 static void cleanup_device_data(struct qib_devdata *dd)
1235 {
1236 int ctxt;
1237 int pidx;
1238 struct qib_ctxtdata **tmp;
1239 unsigned long flags;
1240
1241 /* users can't do anything more with chip */
1242 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1243 if (dd->pport[pidx].statusp)
1244 *dd->pport[pidx].statusp &= ~QIB_STATUS_CHIP_PRESENT;
1245
1246 spin_lock(&dd->pport[pidx].cc_shadow_lock);
1247
1248 kfree(dd->pport[pidx].congestion_entries);
1249 dd->pport[pidx].congestion_entries = NULL;
1250 kfree(dd->pport[pidx].ccti_entries);
1251 dd->pport[pidx].ccti_entries = NULL;
1252 kfree(dd->pport[pidx].ccti_entries_shadow);
1253 dd->pport[pidx].ccti_entries_shadow = NULL;
1254 kfree(dd->pport[pidx].congestion_entries_shadow);
1255 dd->pport[pidx].congestion_entries_shadow = NULL;
1256
1257 spin_unlock(&dd->pport[pidx].cc_shadow_lock);
1258 }
1259
1260 if (!qib_wc_pat)
1261 qib_disable_wc(dd);
1262
1263 if (dd->pioavailregs_dma) {
1264 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
1265 (void *) dd->pioavailregs_dma,
1266 dd->pioavailregs_phys);
1267 dd->pioavailregs_dma = NULL;
1268 }
1269
1270 if (dd->pageshadow) {
1271 struct page **tmpp = dd->pageshadow;
1272 dma_addr_t *tmpd = dd->physshadow;
1273 int i, cnt = 0;
1274
1275 for (ctxt = 0; ctxt < dd->cfgctxts; ctxt++) {
1276 int ctxt_tidbase = ctxt * dd->rcvtidcnt;
1277 int maxtid = ctxt_tidbase + dd->rcvtidcnt;
1278
1279 for (i = ctxt_tidbase; i < maxtid; i++) {
1280 if (!tmpp[i])
1281 continue;
1282 pci_unmap_page(dd->pcidev, tmpd[i],
1283 PAGE_SIZE, PCI_DMA_FROMDEVICE);
1284 qib_release_user_pages(&tmpp[i], 1);
1285 tmpp[i] = NULL;
1286 cnt++;
1287 }
1288 }
1289
1290 tmpp = dd->pageshadow;
1291 dd->pageshadow = NULL;
1292 vfree(tmpp);
1293 }
1294
1295 /*
1296 * Free any resources still in use (usually just kernel contexts)
1297 * at unload; we do for ctxtcnt, because that's what we allocate.
1298 * We acquire lock to be really paranoid that rcd isn't being
1299 * accessed from some interrupt-related code (that should not happen,
1300 * but best to be sure).
1301 */
1302 spin_lock_irqsave(&dd->uctxt_lock, flags);
1303 tmp = dd->rcd;
1304 dd->rcd = NULL;
1305 spin_unlock_irqrestore(&dd->uctxt_lock, flags);
1306 for (ctxt = 0; tmp && ctxt < dd->ctxtcnt; ctxt++) {
1307 struct qib_ctxtdata *rcd = tmp[ctxt];
1308
1309 tmp[ctxt] = NULL; /* debugging paranoia */
1310 qib_free_ctxtdata(dd, rcd);
1311 }
1312 kfree(tmp);
1313 kfree(dd->boardname);
1314 }
1315
1316 /*
1317 * Clean up on unit shutdown, or error during unit load after
1318 * successful initialization.
1319 */
qib_postinit_cleanup(struct qib_devdata * dd)1320 static void qib_postinit_cleanup(struct qib_devdata *dd)
1321 {
1322 /*
1323 * Clean up chip-specific stuff.
1324 * We check for NULL here, because it's outside
1325 * the kregbase check, and we need to call it
1326 * after the free_irq. Thus it's possible that
1327 * the function pointers were never initialized.
1328 */
1329 if (dd->f_cleanup)
1330 dd->f_cleanup(dd);
1331
1332 qib_pcie_ddcleanup(dd);
1333
1334 cleanup_device_data(dd);
1335
1336 qib_free_devdata(dd);
1337 }
1338
qib_init_one(struct pci_dev * pdev,const struct pci_device_id * ent)1339 static int qib_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1340 {
1341 int ret, j, pidx, initfail;
1342 struct qib_devdata *dd = NULL;
1343
1344 ret = qib_pcie_init(pdev, ent);
1345 if (ret)
1346 goto bail;
1347
1348 /*
1349 * Do device-specific initialiation, function table setup, dd
1350 * allocation, etc.
1351 */
1352 switch (ent->device) {
1353 case PCI_DEVICE_ID_QLOGIC_IB_6120:
1354 #ifdef CONFIG_PCI_MSI
1355 dd = qib_init_iba6120_funcs(pdev, ent);
1356 #else
1357 qib_early_err(&pdev->dev,
1358 "Intel PCIE device 0x%x cannot work if CONFIG_PCI_MSI is not enabled\n",
1359 ent->device);
1360 dd = ERR_PTR(-ENODEV);
1361 #endif
1362 break;
1363
1364 case PCI_DEVICE_ID_QLOGIC_IB_7220:
1365 dd = qib_init_iba7220_funcs(pdev, ent);
1366 break;
1367
1368 case PCI_DEVICE_ID_QLOGIC_IB_7322:
1369 dd = qib_init_iba7322_funcs(pdev, ent);
1370 break;
1371
1372 default:
1373 qib_early_err(&pdev->dev,
1374 "Failing on unknown Intel deviceid 0x%x\n",
1375 ent->device);
1376 ret = -ENODEV;
1377 }
1378
1379 if (IS_ERR(dd))
1380 ret = PTR_ERR(dd);
1381 if (ret)
1382 goto bail; /* error already printed */
1383
1384 ret = qib_create_workqueues(dd);
1385 if (ret)
1386 goto bail;
1387
1388 /* do the generic initialization */
1389 initfail = qib_init(dd, 0);
1390
1391 ret = qib_register_ib_device(dd);
1392
1393 /*
1394 * Now ready for use. this should be cleared whenever we
1395 * detect a reset, or initiate one. If earlier failure,
1396 * we still create devices, so diags, etc. can be used
1397 * to determine cause of problem.
1398 */
1399 if (!qib_mini_init && !initfail && !ret)
1400 dd->flags |= QIB_INITTED;
1401
1402 j = qib_device_create(dd);
1403 if (j)
1404 qib_dev_err(dd, "Failed to create /dev devices: %d\n", -j);
1405 j = qibfs_add(dd);
1406 if (j)
1407 qib_dev_err(dd, "Failed filesystem setup for counters: %d\n",
1408 -j);
1409
1410 if (qib_mini_init || initfail || ret) {
1411 qib_stop_timers(dd);
1412 flush_workqueue(ib_wq);
1413 for (pidx = 0; pidx < dd->num_pports; ++pidx)
1414 dd->f_quiet_serdes(dd->pport + pidx);
1415 if (qib_mini_init)
1416 goto bail;
1417 if (!j) {
1418 (void) qibfs_remove(dd);
1419 qib_device_remove(dd);
1420 }
1421 if (!ret)
1422 qib_unregister_ib_device(dd);
1423 qib_postinit_cleanup(dd);
1424 if (initfail)
1425 ret = initfail;
1426 goto bail;
1427 }
1428
1429 if (!qib_wc_pat) {
1430 ret = qib_enable_wc(dd);
1431 if (ret) {
1432 qib_dev_err(dd,
1433 "Write combining not enabled (err %d): performance may be poor\n",
1434 -ret);
1435 ret = 0;
1436 }
1437 }
1438
1439 qib_verify_pioperf(dd);
1440 bail:
1441 return ret;
1442 }
1443
qib_remove_one(struct pci_dev * pdev)1444 static void qib_remove_one(struct pci_dev *pdev)
1445 {
1446 struct qib_devdata *dd = pci_get_drvdata(pdev);
1447 int ret;
1448
1449 /* unregister from IB core */
1450 qib_unregister_ib_device(dd);
1451
1452 /*
1453 * Disable the IB link, disable interrupts on the device,
1454 * clear dma engines, etc.
1455 */
1456 if (!qib_mini_init)
1457 qib_shutdown_device(dd);
1458
1459 qib_stop_timers(dd);
1460
1461 /* wait until all of our (qsfp) queue_work() calls complete */
1462 flush_workqueue(ib_wq);
1463
1464 ret = qibfs_remove(dd);
1465 if (ret)
1466 qib_dev_err(dd, "Failed counters filesystem cleanup: %d\n",
1467 -ret);
1468
1469 qib_device_remove(dd);
1470
1471 qib_postinit_cleanup(dd);
1472 }
1473
1474 /**
1475 * qib_create_rcvhdrq - create a receive header queue
1476 * @dd: the qlogic_ib device
1477 * @rcd: the context data
1478 *
1479 * This must be contiguous memory (from an i/o perspective), and must be
1480 * DMA'able (which means for some systems, it will go through an IOMMU,
1481 * or be forced into a low address range).
1482 */
qib_create_rcvhdrq(struct qib_devdata * dd,struct qib_ctxtdata * rcd)1483 int qib_create_rcvhdrq(struct qib_devdata *dd, struct qib_ctxtdata *rcd)
1484 {
1485 unsigned amt;
1486
1487 if (!rcd->rcvhdrq) {
1488 dma_addr_t phys_hdrqtail;
1489 gfp_t gfp_flags;
1490
1491 amt = ALIGN(dd->rcvhdrcnt * dd->rcvhdrentsize *
1492 sizeof(u32), PAGE_SIZE);
1493 gfp_flags = (rcd->ctxt >= dd->first_user_ctxt) ?
1494 GFP_USER : GFP_KERNEL;
1495 rcd->rcvhdrq = dma_alloc_coherent(
1496 &dd->pcidev->dev, amt, &rcd->rcvhdrq_phys,
1497 gfp_flags | __GFP_COMP);
1498
1499 if (!rcd->rcvhdrq) {
1500 qib_dev_err(dd,
1501 "attempt to allocate %d bytes for ctxt %u rcvhdrq failed\n",
1502 amt, rcd->ctxt);
1503 goto bail;
1504 }
1505
1506 if (rcd->ctxt >= dd->first_user_ctxt) {
1507 rcd->user_event_mask = vmalloc_user(PAGE_SIZE);
1508 if (!rcd->user_event_mask)
1509 goto bail_free_hdrq;
1510 }
1511
1512 if (!(dd->flags & QIB_NODMA_RTAIL)) {
1513 rcd->rcvhdrtail_kvaddr = dma_alloc_coherent(
1514 &dd->pcidev->dev, PAGE_SIZE, &phys_hdrqtail,
1515 gfp_flags);
1516 if (!rcd->rcvhdrtail_kvaddr)
1517 goto bail_free;
1518 rcd->rcvhdrqtailaddr_phys = phys_hdrqtail;
1519 }
1520
1521 rcd->rcvhdrq_size = amt;
1522 }
1523
1524 /* clear for security and sanity on each use */
1525 memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size);
1526 if (rcd->rcvhdrtail_kvaddr)
1527 memset(rcd->rcvhdrtail_kvaddr, 0, PAGE_SIZE);
1528 return 0;
1529
1530 bail_free:
1531 qib_dev_err(dd,
1532 "attempt to allocate 1 page for ctxt %u rcvhdrqtailaddr failed\n",
1533 rcd->ctxt);
1534 vfree(rcd->user_event_mask);
1535 rcd->user_event_mask = NULL;
1536 bail_free_hdrq:
1537 dma_free_coherent(&dd->pcidev->dev, amt, rcd->rcvhdrq,
1538 rcd->rcvhdrq_phys);
1539 rcd->rcvhdrq = NULL;
1540 bail:
1541 return -ENOMEM;
1542 }
1543
1544 /**
1545 * allocate eager buffers, both kernel and user contexts.
1546 * @rcd: the context we are setting up.
1547 *
1548 * Allocate the eager TID buffers and program them into hip.
1549 * They are no longer completely contiguous, we do multiple allocation
1550 * calls. Otherwise we get the OOM code involved, by asking for too
1551 * much per call, with disastrous results on some kernels.
1552 */
qib_setup_eagerbufs(struct qib_ctxtdata * rcd)1553 int qib_setup_eagerbufs(struct qib_ctxtdata *rcd)
1554 {
1555 struct qib_devdata *dd = rcd->dd;
1556 unsigned e, egrcnt, egrperchunk, chunk, egrsize, egroff;
1557 size_t size;
1558 gfp_t gfp_flags;
1559
1560 /*
1561 * GFP_USER, but without GFP_FS, so buffer cache can be
1562 * coalesced (we hope); otherwise, even at order 4,
1563 * heavy filesystem activity makes these fail, and we can
1564 * use compound pages.
1565 */
1566 gfp_flags = __GFP_WAIT | __GFP_IO | __GFP_COMP;
1567
1568 egrcnt = rcd->rcvegrcnt;
1569 egroff = rcd->rcvegr_tid_base;
1570 egrsize = dd->rcvegrbufsize;
1571
1572 chunk = rcd->rcvegrbuf_chunks;
1573 egrperchunk = rcd->rcvegrbufs_perchunk;
1574 size = rcd->rcvegrbuf_size;
1575 if (!rcd->rcvegrbuf) {
1576 rcd->rcvegrbuf =
1577 kzalloc(chunk * sizeof(rcd->rcvegrbuf[0]),
1578 GFP_KERNEL);
1579 if (!rcd->rcvegrbuf)
1580 goto bail;
1581 }
1582 if (!rcd->rcvegrbuf_phys) {
1583 rcd->rcvegrbuf_phys =
1584 kmalloc(chunk * sizeof(rcd->rcvegrbuf_phys[0]),
1585 GFP_KERNEL);
1586 if (!rcd->rcvegrbuf_phys)
1587 goto bail_rcvegrbuf;
1588 }
1589 for (e = 0; e < rcd->rcvegrbuf_chunks; e++) {
1590 if (rcd->rcvegrbuf[e])
1591 continue;
1592 rcd->rcvegrbuf[e] =
1593 dma_alloc_coherent(&dd->pcidev->dev, size,
1594 &rcd->rcvegrbuf_phys[e],
1595 gfp_flags);
1596 if (!rcd->rcvegrbuf[e])
1597 goto bail_rcvegrbuf_phys;
1598 }
1599
1600 rcd->rcvegr_phys = rcd->rcvegrbuf_phys[0];
1601
1602 for (e = chunk = 0; chunk < rcd->rcvegrbuf_chunks; chunk++) {
1603 dma_addr_t pa = rcd->rcvegrbuf_phys[chunk];
1604 unsigned i;
1605
1606 /* clear for security and sanity on each use */
1607 memset(rcd->rcvegrbuf[chunk], 0, size);
1608
1609 for (i = 0; e < egrcnt && i < egrperchunk; e++, i++) {
1610 dd->f_put_tid(dd, e + egroff +
1611 (u64 __iomem *)
1612 ((char __iomem *)
1613 dd->kregbase +
1614 dd->rcvegrbase),
1615 RCVHQ_RCV_TYPE_EAGER, pa);
1616 pa += egrsize;
1617 }
1618 cond_resched(); /* don't hog the cpu */
1619 }
1620
1621 return 0;
1622
1623 bail_rcvegrbuf_phys:
1624 for (e = 0; e < rcd->rcvegrbuf_chunks && rcd->rcvegrbuf[e]; e++)
1625 dma_free_coherent(&dd->pcidev->dev, size,
1626 rcd->rcvegrbuf[e], rcd->rcvegrbuf_phys[e]);
1627 kfree(rcd->rcvegrbuf_phys);
1628 rcd->rcvegrbuf_phys = NULL;
1629 bail_rcvegrbuf:
1630 kfree(rcd->rcvegrbuf);
1631 rcd->rcvegrbuf = NULL;
1632 bail:
1633 return -ENOMEM;
1634 }
1635
1636 /*
1637 * Note: Changes to this routine should be mirrored
1638 * for the diagnostics routine qib_remap_ioaddr32().
1639 * There is also related code for VL15 buffers in qib_init_7322_variables().
1640 * The teardown code that unmaps is in qib_pcie_ddcleanup()
1641 */
init_chip_wc_pat(struct qib_devdata * dd,u32 vl15buflen)1642 int init_chip_wc_pat(struct qib_devdata *dd, u32 vl15buflen)
1643 {
1644 u64 __iomem *qib_kregbase = NULL;
1645 void __iomem *qib_piobase = NULL;
1646 u64 __iomem *qib_userbase = NULL;
1647 u64 qib_kreglen;
1648 u64 qib_pio2koffset = dd->piobufbase & 0xffffffff;
1649 u64 qib_pio4koffset = dd->piobufbase >> 32;
1650 u64 qib_pio2klen = dd->piobcnt2k * dd->palign;
1651 u64 qib_pio4klen = dd->piobcnt4k * dd->align4k;
1652 u64 qib_physaddr = dd->physaddr;
1653 u64 qib_piolen;
1654 u64 qib_userlen = 0;
1655
1656 /*
1657 * Free the old mapping because the kernel will try to reuse the
1658 * old mapping and not create a new mapping with the
1659 * write combining attribute.
1660 */
1661 iounmap(dd->kregbase);
1662 dd->kregbase = NULL;
1663
1664 /*
1665 * Assumes chip address space looks like:
1666 * - kregs + sregs + cregs + uregs (in any order)
1667 * - piobufs (2K and 4K bufs in either order)
1668 * or:
1669 * - kregs + sregs + cregs (in any order)
1670 * - piobufs (2K and 4K bufs in either order)
1671 * - uregs
1672 */
1673 if (dd->piobcnt4k == 0) {
1674 qib_kreglen = qib_pio2koffset;
1675 qib_piolen = qib_pio2klen;
1676 } else if (qib_pio2koffset < qib_pio4koffset) {
1677 qib_kreglen = qib_pio2koffset;
1678 qib_piolen = qib_pio4koffset + qib_pio4klen - qib_kreglen;
1679 } else {
1680 qib_kreglen = qib_pio4koffset;
1681 qib_piolen = qib_pio2koffset + qib_pio2klen - qib_kreglen;
1682 }
1683 qib_piolen += vl15buflen;
1684 /* Map just the configured ports (not all hw ports) */
1685 if (dd->uregbase > qib_kreglen)
1686 qib_userlen = dd->ureg_align * dd->cfgctxts;
1687
1688 /* Sanity checks passed, now create the new mappings */
1689 qib_kregbase = ioremap_nocache(qib_physaddr, qib_kreglen);
1690 if (!qib_kregbase)
1691 goto bail;
1692
1693 qib_piobase = ioremap_wc(qib_physaddr + qib_kreglen, qib_piolen);
1694 if (!qib_piobase)
1695 goto bail_kregbase;
1696
1697 if (qib_userlen) {
1698 qib_userbase = ioremap_nocache(qib_physaddr + dd->uregbase,
1699 qib_userlen);
1700 if (!qib_userbase)
1701 goto bail_piobase;
1702 }
1703
1704 dd->kregbase = qib_kregbase;
1705 dd->kregend = (u64 __iomem *)
1706 ((char __iomem *) qib_kregbase + qib_kreglen);
1707 dd->piobase = qib_piobase;
1708 dd->pio2kbase = (void __iomem *)
1709 (((char __iomem *) dd->piobase) +
1710 qib_pio2koffset - qib_kreglen);
1711 if (dd->piobcnt4k)
1712 dd->pio4kbase = (void __iomem *)
1713 (((char __iomem *) dd->piobase) +
1714 qib_pio4koffset - qib_kreglen);
1715 if (qib_userlen)
1716 /* ureg will now be accessed relative to dd->userbase */
1717 dd->userbase = qib_userbase;
1718 return 0;
1719
1720 bail_piobase:
1721 iounmap(qib_piobase);
1722 bail_kregbase:
1723 iounmap(qib_kregbase);
1724 bail:
1725 return -ENOMEM;
1726 }
1727