1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2021 Broadcom. All Rights Reserved. The term
4 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
5 */
6
7 #include "efct_driver.h"
8 #include "efct_hw.h"
9 #include "efct_unsol.h"
10
11 struct efct_hw_link_stat_cb_arg {
12 void (*cb)(int status, u32 num_counters,
13 struct efct_hw_link_stat_counts *counters, void *arg);
14 void *arg;
15 };
16
17 struct efct_hw_host_stat_cb_arg {
18 void (*cb)(int status, u32 num_counters,
19 struct efct_hw_host_stat_counts *counters, void *arg);
20 void *arg;
21 };
22
23 struct efct_hw_fw_wr_cb_arg {
24 void (*cb)(int status, u32 bytes_written, u32 change_status, void *arg);
25 void *arg;
26 };
27
28 struct efct_mbox_rqst_ctx {
29 int (*callback)(struct efc *efc, int status, u8 *mqe, void *arg);
30 void *arg;
31 };
32
33 static int
efct_hw_link_event_init(struct efct_hw * hw)34 efct_hw_link_event_init(struct efct_hw *hw)
35 {
36 hw->link.status = SLI4_LINK_STATUS_MAX;
37 hw->link.topology = SLI4_LINK_TOPO_NONE;
38 hw->link.medium = SLI4_LINK_MEDIUM_MAX;
39 hw->link.speed = 0;
40 hw->link.loop_map = NULL;
41 hw->link.fc_id = U32_MAX;
42
43 return 0;
44 }
45
46 static int
efct_hw_read_max_dump_size(struct efct_hw * hw)47 efct_hw_read_max_dump_size(struct efct_hw *hw)
48 {
49 u8 buf[SLI4_BMBX_SIZE];
50 struct efct *efct = hw->os;
51 int rc = 0;
52 struct sli4_rsp_cmn_set_dump_location *rsp;
53
54 /* attempt to detemine the dump size for function 0 only. */
55 if (PCI_FUNC(efct->pci->devfn) != 0)
56 return rc;
57
58 if (sli_cmd_common_set_dump_location(&hw->sli, buf, 1, 0, NULL, 0))
59 return -EIO;
60
61 rsp = (struct sli4_rsp_cmn_set_dump_location *)
62 (buf + offsetof(struct sli4_cmd_sli_config, payload.embed));
63
64 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);
65 if (rc != 0) {
66 efc_log_debug(hw->os, "set dump location cmd failed\n");
67 return rc;
68 }
69
70 hw->dump_size =
71 le32_to_cpu(rsp->buffer_length_dword) & SLI4_CMN_SET_DUMP_BUFFER_LEN;
72
73 efc_log_debug(hw->os, "Dump size %x\n", hw->dump_size);
74
75 return rc;
76 }
77
78 static int
__efct_read_topology_cb(struct efct_hw * hw,int status,u8 * mqe,void * arg)79 __efct_read_topology_cb(struct efct_hw *hw, int status, u8 *mqe, void *arg)
80 {
81 struct sli4_cmd_read_topology *read_topo =
82 (struct sli4_cmd_read_topology *)mqe;
83 u8 speed;
84 struct efc_domain_record drec = {0};
85 struct efct *efct = hw->os;
86
87 if (status || le16_to_cpu(read_topo->hdr.status)) {
88 efc_log_debug(hw->os, "bad status cqe=%#x mqe=%#x\n", status,
89 le16_to_cpu(read_topo->hdr.status));
90 return -EIO;
91 }
92
93 switch (le32_to_cpu(read_topo->dw2_attentype) &
94 SLI4_READTOPO_ATTEN_TYPE) {
95 case SLI4_READ_TOPOLOGY_LINK_UP:
96 hw->link.status = SLI4_LINK_STATUS_UP;
97 break;
98 case SLI4_READ_TOPOLOGY_LINK_DOWN:
99 hw->link.status = SLI4_LINK_STATUS_DOWN;
100 break;
101 case SLI4_READ_TOPOLOGY_LINK_NO_ALPA:
102 hw->link.status = SLI4_LINK_STATUS_NO_ALPA;
103 break;
104 default:
105 hw->link.status = SLI4_LINK_STATUS_MAX;
106 break;
107 }
108
109 switch (read_topo->topology) {
110 case SLI4_READ_TOPO_NON_FC_AL:
111 hw->link.topology = SLI4_LINK_TOPO_NON_FC_AL;
112 break;
113 case SLI4_READ_TOPO_FC_AL:
114 hw->link.topology = SLI4_LINK_TOPO_FC_AL;
115 if (hw->link.status == SLI4_LINK_STATUS_UP)
116 hw->link.loop_map = hw->loop_map.virt;
117 hw->link.fc_id = read_topo->acquired_al_pa;
118 break;
119 default:
120 hw->link.topology = SLI4_LINK_TOPO_MAX;
121 break;
122 }
123
124 hw->link.medium = SLI4_LINK_MEDIUM_FC;
125
126 speed = (le32_to_cpu(read_topo->currlink_state) &
127 SLI4_READTOPO_LINKSTATE_SPEED) >> 8;
128 switch (speed) {
129 case SLI4_READ_TOPOLOGY_SPEED_1G:
130 hw->link.speed = 1 * 1000;
131 break;
132 case SLI4_READ_TOPOLOGY_SPEED_2G:
133 hw->link.speed = 2 * 1000;
134 break;
135 case SLI4_READ_TOPOLOGY_SPEED_4G:
136 hw->link.speed = 4 * 1000;
137 break;
138 case SLI4_READ_TOPOLOGY_SPEED_8G:
139 hw->link.speed = 8 * 1000;
140 break;
141 case SLI4_READ_TOPOLOGY_SPEED_16G:
142 hw->link.speed = 16 * 1000;
143 break;
144 case SLI4_READ_TOPOLOGY_SPEED_32G:
145 hw->link.speed = 32 * 1000;
146 break;
147 case SLI4_READ_TOPOLOGY_SPEED_64G:
148 hw->link.speed = 64 * 1000;
149 break;
150 case SLI4_READ_TOPOLOGY_SPEED_128G:
151 hw->link.speed = 128 * 1000;
152 break;
153 }
154
155 drec.speed = hw->link.speed;
156 drec.fc_id = hw->link.fc_id;
157 drec.is_nport = true;
158 efc_domain_cb(efct->efcport, EFC_HW_DOMAIN_FOUND, &drec);
159
160 return 0;
161 }
162
163 static int
efct_hw_cb_link(void * ctx,void * e)164 efct_hw_cb_link(void *ctx, void *e)
165 {
166 struct efct_hw *hw = ctx;
167 struct sli4_link_event *event = e;
168 struct efc_domain *d = NULL;
169 int rc = 0;
170 struct efct *efct = hw->os;
171
172 efct_hw_link_event_init(hw);
173
174 switch (event->status) {
175 case SLI4_LINK_STATUS_UP:
176
177 hw->link = *event;
178 efct->efcport->link_status = EFC_LINK_STATUS_UP;
179
180 if (event->topology == SLI4_LINK_TOPO_NON_FC_AL) {
181 struct efc_domain_record drec = {0};
182
183 efc_log_info(hw->os, "Link Up, NPORT, speed is %d\n",
184 event->speed);
185 drec.speed = event->speed;
186 drec.fc_id = event->fc_id;
187 drec.is_nport = true;
188 efc_domain_cb(efct->efcport, EFC_HW_DOMAIN_FOUND,
189 &drec);
190 } else if (event->topology == SLI4_LINK_TOPO_FC_AL) {
191 u8 buf[SLI4_BMBX_SIZE];
192
193 efc_log_info(hw->os, "Link Up, LOOP, speed is %d\n",
194 event->speed);
195
196 if (!sli_cmd_read_topology(&hw->sli, buf,
197 &hw->loop_map)) {
198 rc = efct_hw_command(hw, buf, EFCT_CMD_NOWAIT,
199 __efct_read_topology_cb, NULL);
200 }
201
202 if (rc)
203 efc_log_debug(hw->os, "READ_TOPOLOGY failed\n");
204 } else {
205 efc_log_info(hw->os, "%s(%#x), speed is %d\n",
206 "Link Up, unsupported topology ",
207 event->topology, event->speed);
208 }
209 break;
210 case SLI4_LINK_STATUS_DOWN:
211 efc_log_info(hw->os, "Link down\n");
212
213 hw->link.status = event->status;
214 efct->efcport->link_status = EFC_LINK_STATUS_DOWN;
215
216 d = efct->efcport->domain;
217 if (d)
218 efc_domain_cb(efct->efcport, EFC_HW_DOMAIN_LOST, d);
219 break;
220 default:
221 efc_log_debug(hw->os, "unhandled link status %#x\n",
222 event->status);
223 break;
224 }
225
226 return 0;
227 }
228
229 int
efct_hw_setup(struct efct_hw * hw,void * os,struct pci_dev * pdev)230 efct_hw_setup(struct efct_hw *hw, void *os, struct pci_dev *pdev)
231 {
232 u32 i, max_sgl, cpus;
233
234 if (hw->hw_setup_called)
235 return 0;
236
237 /*
238 * efct_hw_init() relies on NULL pointers indicating that a structure
239 * needs allocation. If a structure is non-NULL, efct_hw_init() won't
240 * free/realloc that memory
241 */
242 memset(hw, 0, sizeof(struct efct_hw));
243
244 hw->hw_setup_called = true;
245
246 hw->os = os;
247
248 mutex_init(&hw->bmbx_lock);
249 spin_lock_init(&hw->cmd_lock);
250 INIT_LIST_HEAD(&hw->cmd_head);
251 INIT_LIST_HEAD(&hw->cmd_pending);
252 hw->cmd_head_count = 0;
253
254 /* Create mailbox command ctx pool */
255 hw->cmd_ctx_pool = mempool_create_kmalloc_pool(EFCT_CMD_CTX_POOL_SZ,
256 sizeof(struct efct_command_ctx));
257 if (!hw->cmd_ctx_pool) {
258 efc_log_err(hw->os, "failed to allocate mailbox buffer pool\n");
259 return -EIO;
260 }
261
262 /* Create mailbox request ctx pool for library callback */
263 hw->mbox_rqst_pool = mempool_create_kmalloc_pool(EFCT_CMD_CTX_POOL_SZ,
264 sizeof(struct efct_mbox_rqst_ctx));
265 if (!hw->mbox_rqst_pool) {
266 efc_log_err(hw->os, "failed to allocate mbox request pool\n");
267 return -EIO;
268 }
269
270 spin_lock_init(&hw->io_lock);
271 INIT_LIST_HEAD(&hw->io_inuse);
272 INIT_LIST_HEAD(&hw->io_free);
273 INIT_LIST_HEAD(&hw->io_wait_free);
274
275 atomic_set(&hw->io_alloc_failed_count, 0);
276
277 hw->config.speed = SLI4_LINK_SPEED_AUTO_16_8_4;
278 if (sli_setup(&hw->sli, hw->os, pdev, ((struct efct *)os)->reg)) {
279 efc_log_err(hw->os, "SLI setup failed\n");
280 return -EIO;
281 }
282
283 efct_hw_link_event_init(hw);
284
285 sli_callback(&hw->sli, SLI4_CB_LINK, efct_hw_cb_link, hw);
286
287 /*
288 * Set all the queue sizes to the maximum allowed.
289 */
290 for (i = 0; i < ARRAY_SIZE(hw->num_qentries); i++)
291 hw->num_qentries[i] = hw->sli.qinfo.max_qentries[i];
292 /*
293 * Adjust the size of the WQs so that the CQ is twice as big as
294 * the WQ to allow for 2 completions per IO. This allows us to
295 * handle multi-phase as well as aborts.
296 */
297 hw->num_qentries[SLI4_QTYPE_WQ] = hw->num_qentries[SLI4_QTYPE_CQ] / 2;
298
299 /*
300 * The RQ assignment for RQ pair mode.
301 */
302
303 hw->config.rq_default_buffer_size = EFCT_HW_RQ_SIZE_PAYLOAD;
304 hw->config.n_io = hw->sli.ext[SLI4_RSRC_XRI].size;
305
306 cpus = num_possible_cpus();
307 hw->config.n_eq = cpus > EFCT_HW_MAX_NUM_EQ ? EFCT_HW_MAX_NUM_EQ : cpus;
308
309 max_sgl = sli_get_max_sgl(&hw->sli) - SLI4_SGE_MAX_RESERVED;
310 max_sgl = (max_sgl > EFCT_FC_MAX_SGL) ? EFCT_FC_MAX_SGL : max_sgl;
311 hw->config.n_sgl = max_sgl;
312
313 (void)efct_hw_read_max_dump_size(hw);
314
315 return 0;
316 }
317
318 static void
efct_logfcfi(struct efct_hw * hw,u32 j,u32 i,u32 id)319 efct_logfcfi(struct efct_hw *hw, u32 j, u32 i, u32 id)
320 {
321 efc_log_info(hw->os,
322 "REG_FCFI: filter[%d] %08X -> RQ[%d] id=%d\n",
323 j, hw->config.filter_def[j], i, id);
324 }
325
326 static inline void
efct_hw_init_free_io(struct efct_hw_io * io)327 efct_hw_init_free_io(struct efct_hw_io *io)
328 {
329 /*
330 * Set io->done to NULL, to avoid any callbacks, should
331 * a completion be received for one of these IOs
332 */
333 io->done = NULL;
334 io->abort_done = NULL;
335 io->status_saved = false;
336 io->abort_in_progress = false;
337 io->type = 0xFFFF;
338 io->wq = NULL;
339 }
340
efct_hw_iotype_is_originator(u16 io_type)341 static bool efct_hw_iotype_is_originator(u16 io_type)
342 {
343 switch (io_type) {
344 case EFCT_HW_FC_CT:
345 case EFCT_HW_ELS_REQ:
346 return true;
347 default:
348 return false;
349 }
350 }
351
352 static void
efct_hw_io_restore_sgl(struct efct_hw * hw,struct efct_hw_io * io)353 efct_hw_io_restore_sgl(struct efct_hw *hw, struct efct_hw_io *io)
354 {
355 /* Restore the default */
356 io->sgl = &io->def_sgl;
357 io->sgl_count = io->def_sgl_count;
358 }
359
360 static void
efct_hw_wq_process_io(void * arg,u8 * cqe,int status)361 efct_hw_wq_process_io(void *arg, u8 *cqe, int status)
362 {
363 struct efct_hw_io *io = arg;
364 struct efct_hw *hw = io->hw;
365 struct sli4_fc_wcqe *wcqe = (void *)cqe;
366 u32 len = 0;
367 u32 ext = 0;
368
369 /* clear xbusy flag if WCQE[XB] is clear */
370 if (io->xbusy && (wcqe->flags & SLI4_WCQE_XB) == 0)
371 io->xbusy = false;
372
373 /* get extended CQE status */
374 switch (io->type) {
375 case EFCT_HW_BLS_ACC:
376 case EFCT_HW_BLS_RJT:
377 break;
378 case EFCT_HW_ELS_REQ:
379 sli_fc_els_did(&hw->sli, cqe, &ext);
380 len = sli_fc_response_length(&hw->sli, cqe);
381 break;
382 case EFCT_HW_ELS_RSP:
383 case EFCT_HW_FC_CT_RSP:
384 break;
385 case EFCT_HW_FC_CT:
386 len = sli_fc_response_length(&hw->sli, cqe);
387 break;
388 case EFCT_HW_IO_TARGET_WRITE:
389 len = sli_fc_io_length(&hw->sli, cqe);
390 break;
391 case EFCT_HW_IO_TARGET_READ:
392 len = sli_fc_io_length(&hw->sli, cqe);
393 break;
394 case EFCT_HW_IO_TARGET_RSP:
395 break;
396 case EFCT_HW_IO_DNRX_REQUEUE:
397 /* release the count for re-posting the buffer */
398 /* efct_hw_io_free(hw, io); */
399 break;
400 default:
401 efc_log_err(hw->os, "unhandled io type %#x for XRI 0x%x\n",
402 io->type, io->indicator);
403 break;
404 }
405 if (status) {
406 ext = sli_fc_ext_status(&hw->sli, cqe);
407 /*
408 * If we're not an originator IO, and XB is set, then issue
409 * abort for the IO from within the HW
410 */
411 if (efct_hw_iotype_is_originator(io->type) &&
412 wcqe->flags & SLI4_WCQE_XB) {
413 int rc;
414
415 efc_log_debug(hw->os, "aborting xri=%#x tag=%#x\n",
416 io->indicator, io->reqtag);
417
418 /*
419 * Because targets may send a response when the IO
420 * completes using the same XRI, we must wait for the
421 * XRI_ABORTED CQE to issue the IO callback
422 */
423 rc = efct_hw_io_abort(hw, io, false, NULL, NULL);
424 if (rc == 0) {
425 /*
426 * latch status to return after abort is
427 * complete
428 */
429 io->status_saved = true;
430 io->saved_status = status;
431 io->saved_ext = ext;
432 io->saved_len = len;
433 goto exit_efct_hw_wq_process_io;
434 } else if (rc == -EINPROGRESS) {
435 /*
436 * Already being aborted by someone else (ABTS
437 * perhaps). Just return original
438 * error.
439 */
440 efc_log_debug(hw->os, "%s%#x tag=%#x\n",
441 "abort in progress xri=",
442 io->indicator, io->reqtag);
443
444 } else {
445 /* Failed to abort for some other reason, log
446 * error
447 */
448 efc_log_debug(hw->os, "%s%#x tag=%#x rc=%d\n",
449 "Failed to abort xri=",
450 io->indicator, io->reqtag, rc);
451 }
452 }
453 }
454
455 if (io->done) {
456 efct_hw_done_t done = io->done;
457
458 io->done = NULL;
459
460 if (io->status_saved) {
461 /* use latched status if exists */
462 status = io->saved_status;
463 len = io->saved_len;
464 ext = io->saved_ext;
465 io->status_saved = false;
466 }
467
468 /* Restore default SGL */
469 efct_hw_io_restore_sgl(hw, io);
470 done(io, len, status, ext, io->arg);
471 }
472
473 exit_efct_hw_wq_process_io:
474 return;
475 }
476
477 static int
efct_hw_setup_io(struct efct_hw * hw)478 efct_hw_setup_io(struct efct_hw *hw)
479 {
480 u32 i = 0;
481 struct efct_hw_io *io = NULL;
482 uintptr_t xfer_virt = 0;
483 uintptr_t xfer_phys = 0;
484 u32 index;
485 bool new_alloc = true;
486 struct efc_dma *dma;
487 struct efct *efct = hw->os;
488
489 if (!hw->io) {
490 hw->io = kmalloc_array(hw->config.n_io, sizeof(io), GFP_KERNEL);
491 if (!hw->io)
492 return -ENOMEM;
493
494 memset(hw->io, 0, hw->config.n_io * sizeof(io));
495
496 for (i = 0; i < hw->config.n_io; i++) {
497 hw->io[i] = kzalloc(sizeof(*io), GFP_KERNEL);
498 if (!hw->io[i])
499 goto error;
500 }
501
502 /* Create WQE buffs for IO */
503 hw->wqe_buffs = kzalloc((hw->config.n_io * hw->sli.wqe_size),
504 GFP_KERNEL);
505 if (!hw->wqe_buffs) {
506 kfree(hw->io);
507 return -ENOMEM;
508 }
509
510 } else {
511 /* re-use existing IOs, including SGLs */
512 new_alloc = false;
513 }
514
515 if (new_alloc) {
516 dma = &hw->xfer_rdy;
517 dma->size = sizeof(struct fcp_txrdy) * hw->config.n_io;
518 dma->virt = dma_alloc_coherent(&efct->pci->dev,
519 dma->size, &dma->phys, GFP_DMA);
520 if (!dma->virt)
521 return -ENOMEM;
522 }
523 xfer_virt = (uintptr_t)hw->xfer_rdy.virt;
524 xfer_phys = hw->xfer_rdy.phys;
525
526 /* Initialize the pool of HW IO objects */
527 for (i = 0; i < hw->config.n_io; i++) {
528 struct hw_wq_callback *wqcb;
529
530 io = hw->io[i];
531
532 /* initialize IO fields */
533 io->hw = hw;
534
535 /* Assign a WQE buff */
536 io->wqe.wqebuf = &hw->wqe_buffs[i * hw->sli.wqe_size];
537
538 /* Allocate the request tag for this IO */
539 wqcb = efct_hw_reqtag_alloc(hw, efct_hw_wq_process_io, io);
540 if (!wqcb) {
541 efc_log_err(hw->os, "can't allocate request tag\n");
542 return -ENOSPC;
543 }
544 io->reqtag = wqcb->instance_index;
545
546 /* Now for the fields that are initialized on each free */
547 efct_hw_init_free_io(io);
548
549 /* The XB flag isn't cleared on IO free, so init to zero */
550 io->xbusy = 0;
551
552 if (sli_resource_alloc(&hw->sli, SLI4_RSRC_XRI,
553 &io->indicator, &index)) {
554 efc_log_err(hw->os,
555 "sli_resource_alloc failed @ %d\n", i);
556 return -ENOMEM;
557 }
558
559 if (new_alloc) {
560 dma = &io->def_sgl;
561 dma->size = hw->config.n_sgl *
562 sizeof(struct sli4_sge);
563 dma->virt = dma_alloc_coherent(&efct->pci->dev,
564 dma->size, &dma->phys,
565 GFP_DMA);
566 if (!dma->virt) {
567 efc_log_err(hw->os, "dma_alloc fail %d\n", i);
568 memset(&io->def_sgl, 0,
569 sizeof(struct efc_dma));
570 return -ENOMEM;
571 }
572 }
573 io->def_sgl_count = hw->config.n_sgl;
574 io->sgl = &io->def_sgl;
575 io->sgl_count = io->def_sgl_count;
576
577 if (hw->xfer_rdy.size) {
578 io->xfer_rdy.virt = (void *)xfer_virt;
579 io->xfer_rdy.phys = xfer_phys;
580 io->xfer_rdy.size = sizeof(struct fcp_txrdy);
581
582 xfer_virt += sizeof(struct fcp_txrdy);
583 xfer_phys += sizeof(struct fcp_txrdy);
584 }
585 }
586
587 return 0;
588 error:
589 for (i = 0; i < hw->config.n_io && hw->io[i]; i++) {
590 kfree(hw->io[i]);
591 hw->io[i] = NULL;
592 }
593
594 kfree(hw->io);
595 hw->io = NULL;
596
597 return -ENOMEM;
598 }
599
600 static int
efct_hw_init_prereg_io(struct efct_hw * hw)601 efct_hw_init_prereg_io(struct efct_hw *hw)
602 {
603 u32 i, idx = 0;
604 struct efct_hw_io *io = NULL;
605 u8 cmd[SLI4_BMBX_SIZE];
606 int rc = 0;
607 u32 n_rem;
608 u32 n = 0;
609 u32 sgls_per_request = 256;
610 struct efc_dma **sgls = NULL;
611 struct efc_dma req;
612 struct efct *efct = hw->os;
613
614 sgls = kmalloc_array(sgls_per_request, sizeof(*sgls), GFP_KERNEL);
615 if (!sgls)
616 return -ENOMEM;
617
618 memset(&req, 0, sizeof(struct efc_dma));
619 req.size = 32 + sgls_per_request * 16;
620 req.virt = dma_alloc_coherent(&efct->pci->dev, req.size, &req.phys,
621 GFP_DMA);
622 if (!req.virt) {
623 kfree(sgls);
624 return -ENOMEM;
625 }
626
627 for (n_rem = hw->config.n_io; n_rem; n_rem -= n) {
628 /* Copy address of SGL's into local sgls[] array, break
629 * out if the xri is not contiguous.
630 */
631 u32 min = (sgls_per_request < n_rem) ? sgls_per_request : n_rem;
632
633 for (n = 0; n < min; n++) {
634 /* Check that we have contiguous xri values */
635 if (n > 0) {
636 if (hw->io[idx + n]->indicator !=
637 hw->io[idx + n - 1]->indicator + 1)
638 break;
639 }
640
641 sgls[n] = hw->io[idx + n]->sgl;
642 }
643
644 if (sli_cmd_post_sgl_pages(&hw->sli, cmd,
645 hw->io[idx]->indicator, n, sgls, NULL, &req)) {
646 rc = -EIO;
647 break;
648 }
649
650 rc = efct_hw_command(hw, cmd, EFCT_CMD_POLL, NULL, NULL);
651 if (rc) {
652 efc_log_err(hw->os, "SGL post failed, rc=%d\n", rc);
653 break;
654 }
655
656 /* Add to tail if successful */
657 for (i = 0; i < n; i++, idx++) {
658 io = hw->io[idx];
659 io->state = EFCT_HW_IO_STATE_FREE;
660 INIT_LIST_HEAD(&io->list_entry);
661 list_add_tail(&io->list_entry, &hw->io_free);
662 }
663 }
664
665 dma_free_coherent(&efct->pci->dev, req.size, req.virt, req.phys);
666 memset(&req, 0, sizeof(struct efc_dma));
667 kfree(sgls);
668
669 return rc;
670 }
671
672 static int
efct_hw_init_io(struct efct_hw * hw)673 efct_hw_init_io(struct efct_hw *hw)
674 {
675 u32 i, idx = 0;
676 bool prereg = false;
677 struct efct_hw_io *io = NULL;
678 int rc = 0;
679
680 prereg = hw->sli.params.sgl_pre_registered;
681
682 if (prereg)
683 return efct_hw_init_prereg_io(hw);
684
685 for (i = 0; i < hw->config.n_io; i++, idx++) {
686 io = hw->io[idx];
687 io->state = EFCT_HW_IO_STATE_FREE;
688 INIT_LIST_HEAD(&io->list_entry);
689 list_add_tail(&io->list_entry, &hw->io_free);
690 }
691
692 return rc;
693 }
694
695 static int
efct_hw_config_set_fdt_xfer_hint(struct efct_hw * hw,u32 fdt_xfer_hint)696 efct_hw_config_set_fdt_xfer_hint(struct efct_hw *hw, u32 fdt_xfer_hint)
697 {
698 int rc = 0;
699 u8 buf[SLI4_BMBX_SIZE];
700 struct sli4_rqst_cmn_set_features_set_fdt_xfer_hint param;
701
702 memset(¶m, 0, sizeof(param));
703 param.fdt_xfer_hint = cpu_to_le32(fdt_xfer_hint);
704 /* build the set_features command */
705 sli_cmd_common_set_features(&hw->sli, buf,
706 SLI4_SET_FEATURES_SET_FTD_XFER_HINT, sizeof(param), ¶m);
707
708 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);
709 if (rc)
710 efc_log_warn(hw->os, "set FDT hint %d failed: %d\n",
711 fdt_xfer_hint, rc);
712 else
713 efc_log_info(hw->os, "Set FTD transfer hint to %d\n",
714 le32_to_cpu(param.fdt_xfer_hint));
715
716 return rc;
717 }
718
719 static int
efct_hw_config_rq(struct efct_hw * hw)720 efct_hw_config_rq(struct efct_hw *hw)
721 {
722 u32 min_rq_count, i, rc;
723 struct sli4_cmd_rq_cfg rq_cfg[SLI4_CMD_REG_FCFI_NUM_RQ_CFG];
724 u8 buf[SLI4_BMBX_SIZE];
725
726 efc_log_info(hw->os, "using REG_FCFI standard\n");
727
728 /*
729 * Set the filter match/mask values from hw's
730 * filter_def values
731 */
732 for (i = 0; i < SLI4_CMD_REG_FCFI_NUM_RQ_CFG; i++) {
733 rq_cfg[i].rq_id = cpu_to_le16(0xffff);
734 rq_cfg[i].r_ctl_mask = (u8)hw->config.filter_def[i];
735 rq_cfg[i].r_ctl_match = (u8)(hw->config.filter_def[i] >> 8);
736 rq_cfg[i].type_mask = (u8)(hw->config.filter_def[i] >> 16);
737 rq_cfg[i].type_match = (u8)(hw->config.filter_def[i] >> 24);
738 }
739
740 /*
741 * Update the rq_id's of the FCF configuration
742 * (don't update more than the number of rq_cfg
743 * elements)
744 */
745 min_rq_count = (hw->hw_rq_count < SLI4_CMD_REG_FCFI_NUM_RQ_CFG) ?
746 hw->hw_rq_count : SLI4_CMD_REG_FCFI_NUM_RQ_CFG;
747 for (i = 0; i < min_rq_count; i++) {
748 struct hw_rq *rq = hw->hw_rq[i];
749 u32 j;
750
751 for (j = 0; j < SLI4_CMD_REG_FCFI_NUM_RQ_CFG; j++) {
752 u32 mask = (rq->filter_mask != 0) ?
753 rq->filter_mask : 1;
754
755 if (!(mask & (1U << j)))
756 continue;
757
758 rq_cfg[i].rq_id = cpu_to_le16(rq->hdr->id);
759 efct_logfcfi(hw, j, i, rq->hdr->id);
760 }
761 }
762
763 rc = -EIO;
764 if (!sli_cmd_reg_fcfi(&hw->sli, buf, 0, rq_cfg))
765 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);
766
767 if (rc != 0) {
768 efc_log_err(hw->os, "FCFI registration failed\n");
769 return rc;
770 }
771 hw->fcf_indicator =
772 le16_to_cpu(((struct sli4_cmd_reg_fcfi *)buf)->fcfi);
773
774 return rc;
775 }
776
777 static int
efct_hw_config_mrq(struct efct_hw * hw,u8 mode,u16 fcf_index)778 efct_hw_config_mrq(struct efct_hw *hw, u8 mode, u16 fcf_index)
779 {
780 u8 buf[SLI4_BMBX_SIZE], mrq_bitmask = 0;
781 struct hw_rq *rq;
782 struct sli4_cmd_reg_fcfi_mrq *rsp = NULL;
783 struct sli4_cmd_rq_cfg rq_filter[SLI4_CMD_REG_FCFI_MRQ_NUM_RQ_CFG];
784 u32 rc, i;
785
786 if (mode == SLI4_CMD_REG_FCFI_SET_FCFI_MODE)
787 goto issue_cmd;
788
789 /* Set the filter match/mask values from hw's filter_def values */
790 for (i = 0; i < SLI4_CMD_REG_FCFI_NUM_RQ_CFG; i++) {
791 rq_filter[i].rq_id = cpu_to_le16(0xffff);
792 rq_filter[i].type_mask = (u8)hw->config.filter_def[i];
793 rq_filter[i].type_match = (u8)(hw->config.filter_def[i] >> 8);
794 rq_filter[i].r_ctl_mask = (u8)(hw->config.filter_def[i] >> 16);
795 rq_filter[i].r_ctl_match = (u8)(hw->config.filter_def[i] >> 24);
796 }
797
798 rq = hw->hw_rq[0];
799 rq_filter[0].rq_id = cpu_to_le16(rq->hdr->id);
800 rq_filter[1].rq_id = cpu_to_le16(rq->hdr->id);
801
802 mrq_bitmask = 0x2;
803 issue_cmd:
804 efc_log_debug(hw->os, "Issue reg_fcfi_mrq count:%d policy:%d mode:%d\n",
805 hw->hw_rq_count, hw->config.rq_selection_policy, mode);
806 /* Invoke REG_FCFI_MRQ */
807 rc = sli_cmd_reg_fcfi_mrq(&hw->sli, buf, mode, fcf_index,
808 hw->config.rq_selection_policy, mrq_bitmask,
809 hw->hw_mrq_count, rq_filter);
810 if (rc) {
811 efc_log_err(hw->os, "sli_cmd_reg_fcfi_mrq() failed\n");
812 return -EIO;
813 }
814
815 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);
816
817 rsp = (struct sli4_cmd_reg_fcfi_mrq *)buf;
818
819 if ((rc) || (le16_to_cpu(rsp->hdr.status))) {
820 efc_log_err(hw->os, "FCFI MRQ reg failed. cmd=%x status=%x\n",
821 rsp->hdr.command, le16_to_cpu(rsp->hdr.status));
822 return -EIO;
823 }
824
825 if (mode == SLI4_CMD_REG_FCFI_SET_FCFI_MODE)
826 hw->fcf_indicator = le16_to_cpu(rsp->fcfi);
827
828 return 0;
829 }
830
831 static void
efct_hw_queue_hash_add(struct efct_queue_hash * hash,u16 id,u16 index)832 efct_hw_queue_hash_add(struct efct_queue_hash *hash,
833 u16 id, u16 index)
834 {
835 u32 hash_index = id & (EFCT_HW_Q_HASH_SIZE - 1);
836
837 /*
838 * Since the hash is always bigger than the number of queues, then we
839 * never have to worry about an infinite loop.
840 */
841 while (hash[hash_index].in_use)
842 hash_index = (hash_index + 1) & (EFCT_HW_Q_HASH_SIZE - 1);
843
844 /* not used, claim the entry */
845 hash[hash_index].id = id;
846 hash[hash_index].in_use = true;
847 hash[hash_index].index = index;
848 }
849
850 static int
efct_hw_config_sli_port_health_check(struct efct_hw * hw,u8 query,u8 enable)851 efct_hw_config_sli_port_health_check(struct efct_hw *hw, u8 query, u8 enable)
852 {
853 int rc = 0;
854 u8 buf[SLI4_BMBX_SIZE];
855 struct sli4_rqst_cmn_set_features_health_check param;
856 u32 health_check_flag = 0;
857
858 memset(¶m, 0, sizeof(param));
859
860 if (enable)
861 health_check_flag |= SLI4_RQ_HEALTH_CHECK_ENABLE;
862
863 if (query)
864 health_check_flag |= SLI4_RQ_HEALTH_CHECK_QUERY;
865
866 param.health_check_dword = cpu_to_le32(health_check_flag);
867
868 /* build the set_features command */
869 sli_cmd_common_set_features(&hw->sli, buf,
870 SLI4_SET_FEATURES_SLI_PORT_HEALTH_CHECK, sizeof(param), ¶m);
871
872 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);
873 if (rc)
874 efc_log_err(hw->os, "efct_hw_command returns %d\n", rc);
875 else
876 efc_log_debug(hw->os, "SLI Port Health Check is enabled\n");
877
878 return rc;
879 }
880
881 int
efct_hw_init(struct efct_hw * hw)882 efct_hw_init(struct efct_hw *hw)
883 {
884 int rc;
885 u32 i = 0;
886 int rem_count;
887 unsigned long flags = 0;
888 struct efct_hw_io *temp;
889 struct efc_dma *dma;
890
891 /*
892 * Make sure the command lists are empty. If this is start-of-day,
893 * they'll be empty since they were just initialized in efct_hw_setup.
894 * If we've just gone through a reset, the command and command pending
895 * lists should have been cleaned up as part of the reset
896 * (efct_hw_reset()).
897 */
898 spin_lock_irqsave(&hw->cmd_lock, flags);
899 if (!list_empty(&hw->cmd_head)) {
900 spin_unlock_irqrestore(&hw->cmd_lock, flags);
901 efc_log_err(hw->os, "command found on cmd list\n");
902 return -EIO;
903 }
904 if (!list_empty(&hw->cmd_pending)) {
905 spin_unlock_irqrestore(&hw->cmd_lock, flags);
906 efc_log_err(hw->os, "command found on pending list\n");
907 return -EIO;
908 }
909 spin_unlock_irqrestore(&hw->cmd_lock, flags);
910
911 /* Free RQ buffers if prevously allocated */
912 efct_hw_rx_free(hw);
913
914 /*
915 * The IO queues must be initialized here for the reset case. The
916 * efct_hw_init_io() function will re-add the IOs to the free list.
917 * The cmd_head list should be OK since we free all entries in
918 * efct_hw_command_cancel() that is called in the efct_hw_reset().
919 */
920
921 /* If we are in this function due to a reset, there may be stale items
922 * on lists that need to be removed. Clean them up.
923 */
924 rem_count = 0;
925 while ((!list_empty(&hw->io_wait_free))) {
926 rem_count++;
927 temp = list_first_entry(&hw->io_wait_free, struct efct_hw_io,
928 list_entry);
929 list_del_init(&temp->list_entry);
930 }
931 if (rem_count > 0)
932 efc_log_debug(hw->os, "rmvd %d items from io_wait_free list\n",
933 rem_count);
934
935 rem_count = 0;
936 while ((!list_empty(&hw->io_inuse))) {
937 rem_count++;
938 temp = list_first_entry(&hw->io_inuse, struct efct_hw_io,
939 list_entry);
940 list_del_init(&temp->list_entry);
941 }
942 if (rem_count > 0)
943 efc_log_debug(hw->os, "rmvd %d items from io_inuse list\n",
944 rem_count);
945
946 rem_count = 0;
947 while ((!list_empty(&hw->io_free))) {
948 rem_count++;
949 temp = list_first_entry(&hw->io_free, struct efct_hw_io,
950 list_entry);
951 list_del_init(&temp->list_entry);
952 }
953 if (rem_count > 0)
954 efc_log_debug(hw->os, "rmvd %d items from io_free list\n",
955 rem_count);
956
957 /* If MRQ not required, Make sure we dont request feature. */
958 if (hw->config.n_rq == 1)
959 hw->sli.features &= (~SLI4_REQFEAT_MRQP);
960
961 if (sli_init(&hw->sli)) {
962 efc_log_err(hw->os, "SLI failed to initialize\n");
963 return -EIO;
964 }
965
966 if (hw->sliport_healthcheck) {
967 rc = efct_hw_config_sli_port_health_check(hw, 0, 1);
968 if (rc != 0) {
969 efc_log_err(hw->os, "Enable port Health check fail\n");
970 return rc;
971 }
972 }
973
974 /*
975 * Set FDT transfer hint, only works on Lancer
976 */
977 if (hw->sli.if_type == SLI4_INTF_IF_TYPE_2) {
978 /*
979 * Non-fatal error. In particular, we can disregard failure to
980 * set EFCT_HW_FDT_XFER_HINT on devices with legacy firmware
981 * that do not support EFCT_HW_FDT_XFER_HINT feature.
982 */
983 efct_hw_config_set_fdt_xfer_hint(hw, EFCT_HW_FDT_XFER_HINT);
984 }
985
986 /* zero the hashes */
987 memset(hw->cq_hash, 0, sizeof(hw->cq_hash));
988 efc_log_debug(hw->os, "Max CQs %d, hash size = %d\n",
989 EFCT_HW_MAX_NUM_CQ, EFCT_HW_Q_HASH_SIZE);
990
991 memset(hw->rq_hash, 0, sizeof(hw->rq_hash));
992 efc_log_debug(hw->os, "Max RQs %d, hash size = %d\n",
993 EFCT_HW_MAX_NUM_RQ, EFCT_HW_Q_HASH_SIZE);
994
995 memset(hw->wq_hash, 0, sizeof(hw->wq_hash));
996 efc_log_debug(hw->os, "Max WQs %d, hash size = %d\n",
997 EFCT_HW_MAX_NUM_WQ, EFCT_HW_Q_HASH_SIZE);
998
999 rc = efct_hw_init_queues(hw);
1000 if (rc)
1001 return rc;
1002
1003 rc = efct_hw_map_wq_cpu(hw);
1004 if (rc)
1005 return rc;
1006
1007 /* Allocate and p_st RQ buffers */
1008 rc = efct_hw_rx_allocate(hw);
1009 if (rc) {
1010 efc_log_err(hw->os, "rx_allocate failed\n");
1011 return rc;
1012 }
1013
1014 rc = efct_hw_rx_post(hw);
1015 if (rc) {
1016 efc_log_err(hw->os, "WARNING - error posting RQ buffers\n");
1017 return rc;
1018 }
1019
1020 if (hw->config.n_eq == 1) {
1021 rc = efct_hw_config_rq(hw);
1022 if (rc) {
1023 efc_log_err(hw->os, "config rq failed %d\n", rc);
1024 return rc;
1025 }
1026 } else {
1027 rc = efct_hw_config_mrq(hw, SLI4_CMD_REG_FCFI_SET_FCFI_MODE, 0);
1028 if (rc != 0) {
1029 efc_log_err(hw->os, "REG_FCFI_MRQ FCFI reg failed\n");
1030 return rc;
1031 }
1032
1033 rc = efct_hw_config_mrq(hw, SLI4_CMD_REG_FCFI_SET_MRQ_MODE, 0);
1034 if (rc != 0) {
1035 efc_log_err(hw->os, "REG_FCFI_MRQ MRQ reg failed\n");
1036 return rc;
1037 }
1038 }
1039
1040 /*
1041 * Allocate the WQ request tag pool, if not previously allocated
1042 * (the request tag value is 16 bits, thus the pool allocation size
1043 * of 64k)
1044 */
1045 hw->wq_reqtag_pool = efct_hw_reqtag_pool_alloc(hw);
1046 if (!hw->wq_reqtag_pool) {
1047 efc_log_err(hw->os, "efct_hw_reqtag_pool_alloc failed\n");
1048 return -ENOMEM;
1049 }
1050
1051 rc = efct_hw_setup_io(hw);
1052 if (rc) {
1053 efc_log_err(hw->os, "IO allocation failure\n");
1054 return rc;
1055 }
1056
1057 rc = efct_hw_init_io(hw);
1058 if (rc) {
1059 efc_log_err(hw->os, "IO initialization failure\n");
1060 return rc;
1061 }
1062
1063 dma = &hw->loop_map;
1064 dma->size = SLI4_MIN_LOOP_MAP_BYTES;
1065 dma->virt = dma_alloc_coherent(&hw->os->pci->dev, dma->size, &dma->phys,
1066 GFP_DMA);
1067 if (!dma->virt)
1068 return -EIO;
1069
1070 /*
1071 * Arming the EQ allows (e.g.) interrupts when CQ completions write EQ
1072 * entries
1073 */
1074 for (i = 0; i < hw->eq_count; i++)
1075 sli_queue_arm(&hw->sli, &hw->eq[i], true);
1076
1077 /*
1078 * Initialize RQ hash
1079 */
1080 for (i = 0; i < hw->rq_count; i++)
1081 efct_hw_queue_hash_add(hw->rq_hash, hw->rq[i].id, i);
1082
1083 /*
1084 * Initialize WQ hash
1085 */
1086 for (i = 0; i < hw->wq_count; i++)
1087 efct_hw_queue_hash_add(hw->wq_hash, hw->wq[i].id, i);
1088
1089 /*
1090 * Arming the CQ allows (e.g.) MQ completions to write CQ entries
1091 */
1092 for (i = 0; i < hw->cq_count; i++) {
1093 efct_hw_queue_hash_add(hw->cq_hash, hw->cq[i].id, i);
1094 sli_queue_arm(&hw->sli, &hw->cq[i], true);
1095 }
1096
1097 /* Set RQ process limit*/
1098 for (i = 0; i < hw->hw_rq_count; i++) {
1099 struct hw_rq *rq = hw->hw_rq[i];
1100
1101 hw->cq[rq->cq->instance].proc_limit = hw->config.n_io / 2;
1102 }
1103
1104 /* record the fact that the queues are functional */
1105 hw->state = EFCT_HW_STATE_ACTIVE;
1106 /*
1107 * Allocate a HW IOs for send frame.
1108 */
1109 hw->hw_wq[0]->send_frame_io = efct_hw_io_alloc(hw);
1110 if (!hw->hw_wq[0]->send_frame_io)
1111 efc_log_err(hw->os, "alloc for send_frame_io failed\n");
1112
1113 /* Initialize send frame sequence id */
1114 atomic_set(&hw->send_frame_seq_id, 0);
1115
1116 return 0;
1117 }
1118
1119 int
efct_hw_parse_filter(struct efct_hw * hw,void * value)1120 efct_hw_parse_filter(struct efct_hw *hw, void *value)
1121 {
1122 int rc = 0;
1123 char *p = NULL;
1124 char *token;
1125 u32 idx = 0;
1126
1127 for (idx = 0; idx < ARRAY_SIZE(hw->config.filter_def); idx++)
1128 hw->config.filter_def[idx] = 0;
1129
1130 p = kstrdup(value, GFP_KERNEL);
1131 if (!p || !*p) {
1132 efc_log_err(hw->os, "p is NULL\n");
1133 return -ENOMEM;
1134 }
1135
1136 idx = 0;
1137 while ((token = strsep(&p, ",")) && *token) {
1138 if (kstrtou32(token, 0, &hw->config.filter_def[idx++]))
1139 efc_log_err(hw->os, "kstrtoint failed\n");
1140
1141 if (!p || !*p)
1142 break;
1143
1144 if (idx == ARRAY_SIZE(hw->config.filter_def))
1145 break;
1146 }
1147 kfree(p);
1148
1149 return rc;
1150 }
1151
1152 u64
efct_get_wwnn(struct efct_hw * hw)1153 efct_get_wwnn(struct efct_hw *hw)
1154 {
1155 struct sli4 *sli = &hw->sli;
1156 u8 p[8];
1157
1158 memcpy(p, sli->wwnn, sizeof(p));
1159 return get_unaligned_be64(p);
1160 }
1161
1162 u64
efct_get_wwpn(struct efct_hw * hw)1163 efct_get_wwpn(struct efct_hw *hw)
1164 {
1165 struct sli4 *sli = &hw->sli;
1166 u8 p[8];
1167
1168 memcpy(p, sli->wwpn, sizeof(p));
1169 return get_unaligned_be64(p);
1170 }
1171
1172 static struct efc_hw_rq_buffer *
efct_hw_rx_buffer_alloc(struct efct_hw * hw,u32 rqindex,u32 count,u32 size)1173 efct_hw_rx_buffer_alloc(struct efct_hw *hw, u32 rqindex, u32 count,
1174 u32 size)
1175 {
1176 struct efct *efct = hw->os;
1177 struct efc_hw_rq_buffer *rq_buf = NULL;
1178 struct efc_hw_rq_buffer *prq;
1179 u32 i;
1180
1181 if (!count)
1182 return NULL;
1183
1184 rq_buf = kmalloc_array(count, sizeof(*rq_buf), GFP_KERNEL);
1185 if (!rq_buf)
1186 return NULL;
1187 memset(rq_buf, 0, sizeof(*rq_buf) * count);
1188
1189 for (i = 0, prq = rq_buf; i < count; i ++, prq++) {
1190 prq->rqindex = rqindex;
1191 prq->dma.size = size;
1192 prq->dma.virt = dma_alloc_coherent(&efct->pci->dev,
1193 prq->dma.size,
1194 &prq->dma.phys,
1195 GFP_DMA);
1196 if (!prq->dma.virt) {
1197 efc_log_err(hw->os, "DMA allocation failed\n");
1198 kfree(rq_buf);
1199 return NULL;
1200 }
1201 }
1202 return rq_buf;
1203 }
1204
1205 static void
efct_hw_rx_buffer_free(struct efct_hw * hw,struct efc_hw_rq_buffer * rq_buf,u32 count)1206 efct_hw_rx_buffer_free(struct efct_hw *hw,
1207 struct efc_hw_rq_buffer *rq_buf,
1208 u32 count)
1209 {
1210 struct efct *efct = hw->os;
1211 u32 i;
1212 struct efc_hw_rq_buffer *prq;
1213
1214 if (rq_buf) {
1215 for (i = 0, prq = rq_buf; i < count; i++, prq++) {
1216 dma_free_coherent(&efct->pci->dev,
1217 prq->dma.size, prq->dma.virt,
1218 prq->dma.phys);
1219 memset(&prq->dma, 0, sizeof(struct efc_dma));
1220 }
1221
1222 kfree(rq_buf);
1223 }
1224 }
1225
1226 int
efct_hw_rx_allocate(struct efct_hw * hw)1227 efct_hw_rx_allocate(struct efct_hw *hw)
1228 {
1229 struct efct *efct = hw->os;
1230 u32 i;
1231 int rc = 0;
1232 u32 rqindex = 0;
1233 u32 hdr_size = EFCT_HW_RQ_SIZE_HDR;
1234 u32 payload_size = hw->config.rq_default_buffer_size;
1235
1236 rqindex = 0;
1237
1238 for (i = 0; i < hw->hw_rq_count; i++) {
1239 struct hw_rq *rq = hw->hw_rq[i];
1240
1241 /* Allocate header buffers */
1242 rq->hdr_buf = efct_hw_rx_buffer_alloc(hw, rqindex,
1243 rq->entry_count,
1244 hdr_size);
1245 if (!rq->hdr_buf) {
1246 efc_log_err(efct, "rx_buffer_alloc hdr_buf failed\n");
1247 rc = -EIO;
1248 break;
1249 }
1250
1251 efc_log_debug(hw->os,
1252 "rq[%2d] rq_id %02d header %4d by %4d bytes\n",
1253 i, rq->hdr->id, rq->entry_count, hdr_size);
1254
1255 rqindex++;
1256
1257 /* Allocate payload buffers */
1258 rq->payload_buf = efct_hw_rx_buffer_alloc(hw, rqindex,
1259 rq->entry_count,
1260 payload_size);
1261 if (!rq->payload_buf) {
1262 efc_log_err(efct, "rx_buffer_alloc fb_buf failed\n");
1263 rc = -EIO;
1264 break;
1265 }
1266 efc_log_debug(hw->os,
1267 "rq[%2d] rq_id %02d default %4d by %4d bytes\n",
1268 i, rq->data->id, rq->entry_count, payload_size);
1269 rqindex++;
1270 }
1271
1272 return rc ? -EIO : 0;
1273 }
1274
1275 int
efct_hw_rx_post(struct efct_hw * hw)1276 efct_hw_rx_post(struct efct_hw *hw)
1277 {
1278 u32 i;
1279 u32 idx;
1280 u32 rq_idx;
1281 int rc = 0;
1282
1283 if (!hw->seq_pool) {
1284 u32 count = 0;
1285
1286 for (i = 0; i < hw->hw_rq_count; i++)
1287 count += hw->hw_rq[i]->entry_count;
1288
1289 hw->seq_pool = kmalloc_array(count,
1290 sizeof(struct efc_hw_sequence), GFP_KERNEL);
1291 if (!hw->seq_pool)
1292 return -ENOMEM;
1293 }
1294
1295 /*
1296 * In RQ pair mode, we MUST post the header and payload buffer at the
1297 * same time.
1298 */
1299 for (rq_idx = 0, idx = 0; rq_idx < hw->hw_rq_count; rq_idx++) {
1300 struct hw_rq *rq = hw->hw_rq[rq_idx];
1301
1302 for (i = 0; i < rq->entry_count - 1; i++) {
1303 struct efc_hw_sequence *seq;
1304
1305 seq = hw->seq_pool + idx;
1306 idx++;
1307 seq->header = &rq->hdr_buf[i];
1308 seq->payload = &rq->payload_buf[i];
1309 rc = efct_hw_sequence_free(hw, seq);
1310 if (rc)
1311 break;
1312 }
1313 if (rc)
1314 break;
1315 }
1316
1317 if (rc && hw->seq_pool)
1318 kfree(hw->seq_pool);
1319
1320 return rc;
1321 }
1322
1323 void
efct_hw_rx_free(struct efct_hw * hw)1324 efct_hw_rx_free(struct efct_hw *hw)
1325 {
1326 u32 i;
1327
1328 /* Free hw_rq buffers */
1329 for (i = 0; i < hw->hw_rq_count; i++) {
1330 struct hw_rq *rq = hw->hw_rq[i];
1331
1332 if (rq) {
1333 efct_hw_rx_buffer_free(hw, rq->hdr_buf,
1334 rq->entry_count);
1335 rq->hdr_buf = NULL;
1336 efct_hw_rx_buffer_free(hw, rq->payload_buf,
1337 rq->entry_count);
1338 rq->payload_buf = NULL;
1339 }
1340 }
1341 }
1342
1343 static int
efct_hw_cmd_submit_pending(struct efct_hw * hw)1344 efct_hw_cmd_submit_pending(struct efct_hw *hw)
1345 {
1346 int rc = 0;
1347
1348 /* Assumes lock held */
1349
1350 /* Only submit MQE if there's room */
1351 while (hw->cmd_head_count < (EFCT_HW_MQ_DEPTH - 1) &&
1352 !list_empty(&hw->cmd_pending)) {
1353 struct efct_command_ctx *ctx;
1354
1355 ctx = list_first_entry(&hw->cmd_pending,
1356 struct efct_command_ctx, list_entry);
1357 if (!ctx)
1358 break;
1359
1360 list_del_init(&ctx->list_entry);
1361
1362 list_add_tail(&ctx->list_entry, &hw->cmd_head);
1363 hw->cmd_head_count++;
1364 if (sli_mq_write(&hw->sli, hw->mq, ctx->buf) < 0) {
1365 efc_log_debug(hw->os,
1366 "sli_queue_write failed: %d\n", rc);
1367 rc = -EIO;
1368 break;
1369 }
1370 }
1371 return rc;
1372 }
1373
1374 int
efct_hw_command(struct efct_hw * hw,u8 * cmd,u32 opts,void * cb,void * arg)1375 efct_hw_command(struct efct_hw *hw, u8 *cmd, u32 opts, void *cb, void *arg)
1376 {
1377 int rc = -EIO;
1378 unsigned long flags = 0;
1379 void *bmbx = NULL;
1380
1381 /*
1382 * If the chip is in an error state (UE'd) then reject this mailbox
1383 * command.
1384 */
1385 if (sli_fw_error_status(&hw->sli) > 0) {
1386 efc_log_crit(hw->os, "Chip in an error state - reset needed\n");
1387 efc_log_crit(hw->os, "status=%#x error1=%#x error2=%#x\n",
1388 sli_reg_read_status(&hw->sli),
1389 sli_reg_read_err1(&hw->sli),
1390 sli_reg_read_err2(&hw->sli));
1391
1392 return -EIO;
1393 }
1394
1395 /*
1396 * Send a mailbox command to the hardware, and either wait for
1397 * a completion (EFCT_CMD_POLL) or get an optional asynchronous
1398 * completion (EFCT_CMD_NOWAIT).
1399 */
1400
1401 if (opts == EFCT_CMD_POLL) {
1402 mutex_lock(&hw->bmbx_lock);
1403 bmbx = hw->sli.bmbx.virt;
1404
1405 memset(bmbx, 0, SLI4_BMBX_SIZE);
1406 memcpy(bmbx, cmd, SLI4_BMBX_SIZE);
1407
1408 if (sli_bmbx_command(&hw->sli) == 0) {
1409 rc = 0;
1410 memcpy(cmd, bmbx, SLI4_BMBX_SIZE);
1411 }
1412 mutex_unlock(&hw->bmbx_lock);
1413 } else if (opts == EFCT_CMD_NOWAIT) {
1414 struct efct_command_ctx *ctx = NULL;
1415
1416 if (hw->state != EFCT_HW_STATE_ACTIVE) {
1417 efc_log_err(hw->os, "Can't send command, HW state=%d\n",
1418 hw->state);
1419 return -EIO;
1420 }
1421
1422 ctx = mempool_alloc(hw->cmd_ctx_pool, GFP_ATOMIC);
1423 if (!ctx)
1424 return -ENOSPC;
1425
1426 memset(ctx, 0, sizeof(struct efct_command_ctx));
1427
1428 if (cb) {
1429 ctx->cb = cb;
1430 ctx->arg = arg;
1431 }
1432
1433 memcpy(ctx->buf, cmd, SLI4_BMBX_SIZE);
1434 ctx->ctx = hw;
1435
1436 spin_lock_irqsave(&hw->cmd_lock, flags);
1437
1438 /* Add to pending list */
1439 INIT_LIST_HEAD(&ctx->list_entry);
1440 list_add_tail(&ctx->list_entry, &hw->cmd_pending);
1441
1442 /* Submit as much of the pending list as we can */
1443 rc = efct_hw_cmd_submit_pending(hw);
1444
1445 spin_unlock_irqrestore(&hw->cmd_lock, flags);
1446 }
1447
1448 return rc;
1449 }
1450
1451 static int
efct_hw_command_process(struct efct_hw * hw,int status,u8 * mqe,size_t size)1452 efct_hw_command_process(struct efct_hw *hw, int status, u8 *mqe,
1453 size_t size)
1454 {
1455 struct efct_command_ctx *ctx = NULL;
1456 unsigned long flags = 0;
1457
1458 spin_lock_irqsave(&hw->cmd_lock, flags);
1459 if (!list_empty(&hw->cmd_head)) {
1460 ctx = list_first_entry(&hw->cmd_head,
1461 struct efct_command_ctx, list_entry);
1462 list_del_init(&ctx->list_entry);
1463 }
1464 if (!ctx) {
1465 efc_log_err(hw->os, "no command context\n");
1466 spin_unlock_irqrestore(&hw->cmd_lock, flags);
1467 return -EIO;
1468 }
1469
1470 hw->cmd_head_count--;
1471
1472 /* Post any pending requests */
1473 efct_hw_cmd_submit_pending(hw);
1474
1475 spin_unlock_irqrestore(&hw->cmd_lock, flags);
1476
1477 if (ctx->cb) {
1478 memcpy(ctx->buf, mqe, size);
1479 ctx->cb(hw, status, ctx->buf, ctx->arg);
1480 }
1481
1482 mempool_free(ctx, hw->cmd_ctx_pool);
1483
1484 return 0;
1485 }
1486
1487 static int
efct_hw_mq_process(struct efct_hw * hw,int status,struct sli4_queue * mq)1488 efct_hw_mq_process(struct efct_hw *hw,
1489 int status, struct sli4_queue *mq)
1490 {
1491 u8 mqe[SLI4_BMBX_SIZE];
1492 int rc;
1493
1494 rc = sli_mq_read(&hw->sli, mq, mqe);
1495 if (!rc)
1496 rc = efct_hw_command_process(hw, status, mqe, mq->size);
1497
1498 return rc;
1499 }
1500
1501 static int
efct_hw_command_cancel(struct efct_hw * hw)1502 efct_hw_command_cancel(struct efct_hw *hw)
1503 {
1504 unsigned long flags = 0;
1505 int rc = 0;
1506
1507 spin_lock_irqsave(&hw->cmd_lock, flags);
1508
1509 /*
1510 * Manually clean up remaining commands. Note: since this calls
1511 * efct_hw_command_process(), we'll also process the cmd_pending
1512 * list, so no need to manually clean that out.
1513 */
1514 while (!list_empty(&hw->cmd_head)) {
1515 u8 mqe[SLI4_BMBX_SIZE] = { 0 };
1516 struct efct_command_ctx *ctx;
1517
1518 ctx = list_first_entry(&hw->cmd_head,
1519 struct efct_command_ctx, list_entry);
1520
1521 efc_log_debug(hw->os, "hung command %08x\n",
1522 !ctx ? U32_MAX : *((u32 *)ctx->buf));
1523 spin_unlock_irqrestore(&hw->cmd_lock, flags);
1524 rc = efct_hw_command_process(hw, -1, mqe, SLI4_BMBX_SIZE);
1525 spin_lock_irqsave(&hw->cmd_lock, flags);
1526 }
1527
1528 spin_unlock_irqrestore(&hw->cmd_lock, flags);
1529
1530 return rc;
1531 }
1532
1533 static void
efct_mbox_rsp_cb(struct efct_hw * hw,int status,u8 * mqe,void * arg)1534 efct_mbox_rsp_cb(struct efct_hw *hw, int status, u8 *mqe, void *arg)
1535 {
1536 struct efct_mbox_rqst_ctx *ctx = arg;
1537
1538 if (ctx) {
1539 if (ctx->callback)
1540 (*ctx->callback)(hw->os->efcport, status, mqe,
1541 ctx->arg);
1542
1543 mempool_free(ctx, hw->mbox_rqst_pool);
1544 }
1545 }
1546
1547 int
efct_issue_mbox_rqst(void * base,void * cmd,void * cb,void * arg)1548 efct_issue_mbox_rqst(void *base, void *cmd, void *cb, void *arg)
1549 {
1550 struct efct_mbox_rqst_ctx *ctx;
1551 struct efct *efct = base;
1552 struct efct_hw *hw = &efct->hw;
1553 int rc;
1554
1555 /*
1556 * Allocate a callback context (which includes the mbox cmd buffer),
1557 * we need this to be persistent as the mbox cmd submission may be
1558 * queued and executed later execution.
1559 */
1560 ctx = mempool_alloc(hw->mbox_rqst_pool, GFP_ATOMIC);
1561 if (!ctx)
1562 return -EIO;
1563
1564 ctx->callback = cb;
1565 ctx->arg = arg;
1566
1567 rc = efct_hw_command(hw, cmd, EFCT_CMD_NOWAIT, efct_mbox_rsp_cb, ctx);
1568 if (rc) {
1569 efc_log_err(efct, "issue mbox rqst failure rc:%d\n", rc);
1570 mempool_free(ctx, hw->mbox_rqst_pool);
1571 return -EIO;
1572 }
1573
1574 return 0;
1575 }
1576
1577 static inline struct efct_hw_io *
_efct_hw_io_alloc(struct efct_hw * hw)1578 _efct_hw_io_alloc(struct efct_hw *hw)
1579 {
1580 struct efct_hw_io *io = NULL;
1581
1582 if (!list_empty(&hw->io_free)) {
1583 io = list_first_entry(&hw->io_free, struct efct_hw_io,
1584 list_entry);
1585 list_del(&io->list_entry);
1586 }
1587 if (io) {
1588 INIT_LIST_HEAD(&io->list_entry);
1589 list_add_tail(&io->list_entry, &hw->io_inuse);
1590 io->state = EFCT_HW_IO_STATE_INUSE;
1591 io->abort_reqtag = U32_MAX;
1592 io->wq = hw->wq_cpu_array[raw_smp_processor_id()];
1593 if (!io->wq) {
1594 efc_log_err(hw->os, "WQ not assigned for cpu:%d\n",
1595 raw_smp_processor_id());
1596 io->wq = hw->hw_wq[0];
1597 }
1598 kref_init(&io->ref);
1599 io->release = efct_hw_io_free_internal;
1600 } else {
1601 atomic_add(1, &hw->io_alloc_failed_count);
1602 }
1603
1604 return io;
1605 }
1606
1607 struct efct_hw_io *
efct_hw_io_alloc(struct efct_hw * hw)1608 efct_hw_io_alloc(struct efct_hw *hw)
1609 {
1610 struct efct_hw_io *io = NULL;
1611 unsigned long flags = 0;
1612
1613 spin_lock_irqsave(&hw->io_lock, flags);
1614 io = _efct_hw_io_alloc(hw);
1615 spin_unlock_irqrestore(&hw->io_lock, flags);
1616
1617 return io;
1618 }
1619
1620 static void
efct_hw_io_free_move_correct_list(struct efct_hw * hw,struct efct_hw_io * io)1621 efct_hw_io_free_move_correct_list(struct efct_hw *hw,
1622 struct efct_hw_io *io)
1623 {
1624 /*
1625 * When an IO is freed, depending on the exchange busy flag,
1626 * move it to the correct list.
1627 */
1628 if (io->xbusy) {
1629 /*
1630 * add to wait_free list and wait for XRI_ABORTED CQEs to clean
1631 * up
1632 */
1633 INIT_LIST_HEAD(&io->list_entry);
1634 list_add_tail(&io->list_entry, &hw->io_wait_free);
1635 io->state = EFCT_HW_IO_STATE_WAIT_FREE;
1636 } else {
1637 /* IO not busy, add to free list */
1638 INIT_LIST_HEAD(&io->list_entry);
1639 list_add_tail(&io->list_entry, &hw->io_free);
1640 io->state = EFCT_HW_IO_STATE_FREE;
1641 }
1642 }
1643
1644 static inline void
efct_hw_io_free_common(struct efct_hw * hw,struct efct_hw_io * io)1645 efct_hw_io_free_common(struct efct_hw *hw, struct efct_hw_io *io)
1646 {
1647 /* initialize IO fields */
1648 efct_hw_init_free_io(io);
1649
1650 /* Restore default SGL */
1651 efct_hw_io_restore_sgl(hw, io);
1652 }
1653
1654 void
efct_hw_io_free_internal(struct kref * arg)1655 efct_hw_io_free_internal(struct kref *arg)
1656 {
1657 unsigned long flags = 0;
1658 struct efct_hw_io *io = container_of(arg, struct efct_hw_io, ref);
1659 struct efct_hw *hw = io->hw;
1660
1661 /* perform common cleanup */
1662 efct_hw_io_free_common(hw, io);
1663
1664 spin_lock_irqsave(&hw->io_lock, flags);
1665 /* remove from in-use list */
1666 if (!list_empty(&io->list_entry) && !list_empty(&hw->io_inuse)) {
1667 list_del_init(&io->list_entry);
1668 efct_hw_io_free_move_correct_list(hw, io);
1669 }
1670 spin_unlock_irqrestore(&hw->io_lock, flags);
1671 }
1672
1673 int
efct_hw_io_free(struct efct_hw * hw,struct efct_hw_io * io)1674 efct_hw_io_free(struct efct_hw *hw, struct efct_hw_io *io)
1675 {
1676 return kref_put(&io->ref, io->release);
1677 }
1678
1679 struct efct_hw_io *
efct_hw_io_lookup(struct efct_hw * hw,u32 xri)1680 efct_hw_io_lookup(struct efct_hw *hw, u32 xri)
1681 {
1682 u32 ioindex;
1683
1684 ioindex = xri - hw->sli.ext[SLI4_RSRC_XRI].base[0];
1685 return hw->io[ioindex];
1686 }
1687
1688 int
efct_hw_io_init_sges(struct efct_hw * hw,struct efct_hw_io * io,enum efct_hw_io_type type)1689 efct_hw_io_init_sges(struct efct_hw *hw, struct efct_hw_io *io,
1690 enum efct_hw_io_type type)
1691 {
1692 struct sli4_sge *data = NULL;
1693 u32 i = 0;
1694 u32 skips = 0;
1695 u32 sge_flags = 0;
1696
1697 if (!io) {
1698 efc_log_err(hw->os, "bad parameter hw=%p io=%p\n", hw, io);
1699 return -EIO;
1700 }
1701
1702 /* Clear / reset the scatter-gather list */
1703 io->sgl = &io->def_sgl;
1704 io->sgl_count = io->def_sgl_count;
1705 io->first_data_sge = 0;
1706
1707 memset(io->sgl->virt, 0, 2 * sizeof(struct sli4_sge));
1708 io->n_sge = 0;
1709 io->sge_offset = 0;
1710
1711 io->type = type;
1712
1713 data = io->sgl->virt;
1714
1715 /*
1716 * Some IO types have underlying hardware requirements on the order
1717 * of SGEs. Process all special entries here.
1718 */
1719 switch (type) {
1720 case EFCT_HW_IO_TARGET_WRITE:
1721
1722 /* populate host resident XFER_RDY buffer */
1723 sge_flags = le32_to_cpu(data->dw2_flags);
1724 sge_flags &= (~SLI4_SGE_TYPE_MASK);
1725 sge_flags |= (SLI4_SGE_TYPE_DATA << SLI4_SGE_TYPE_SHIFT);
1726 data->buffer_address_high =
1727 cpu_to_le32(upper_32_bits(io->xfer_rdy.phys));
1728 data->buffer_address_low =
1729 cpu_to_le32(lower_32_bits(io->xfer_rdy.phys));
1730 data->buffer_length = cpu_to_le32(io->xfer_rdy.size);
1731 data->dw2_flags = cpu_to_le32(sge_flags);
1732 data++;
1733
1734 skips = EFCT_TARGET_WRITE_SKIPS;
1735
1736 io->n_sge = 1;
1737 break;
1738 case EFCT_HW_IO_TARGET_READ:
1739 /*
1740 * For FCP_TSEND64, the first 2 entries are SKIP SGE's
1741 */
1742 skips = EFCT_TARGET_READ_SKIPS;
1743 break;
1744 case EFCT_HW_IO_TARGET_RSP:
1745 /*
1746 * No skips, etc. for FCP_TRSP64
1747 */
1748 break;
1749 default:
1750 efc_log_err(hw->os, "unsupported IO type %#x\n", type);
1751 return -EIO;
1752 }
1753
1754 /*
1755 * Write skip entries
1756 */
1757 for (i = 0; i < skips; i++) {
1758 sge_flags = le32_to_cpu(data->dw2_flags);
1759 sge_flags &= (~SLI4_SGE_TYPE_MASK);
1760 sge_flags |= (SLI4_SGE_TYPE_SKIP << SLI4_SGE_TYPE_SHIFT);
1761 data->dw2_flags = cpu_to_le32(sge_flags);
1762 data++;
1763 }
1764
1765 io->n_sge += skips;
1766
1767 /*
1768 * Set last
1769 */
1770 sge_flags = le32_to_cpu(data->dw2_flags);
1771 sge_flags |= SLI4_SGE_LAST;
1772 data->dw2_flags = cpu_to_le32(sge_flags);
1773
1774 return 0;
1775 }
1776
1777 int
efct_hw_io_add_sge(struct efct_hw * hw,struct efct_hw_io * io,uintptr_t addr,u32 length)1778 efct_hw_io_add_sge(struct efct_hw *hw, struct efct_hw_io *io,
1779 uintptr_t addr, u32 length)
1780 {
1781 struct sli4_sge *data = NULL;
1782 u32 sge_flags = 0;
1783
1784 if (!io || !addr || !length) {
1785 efc_log_err(hw->os,
1786 "bad parameter hw=%p io=%p addr=%lx length=%u\n",
1787 hw, io, addr, length);
1788 return -EIO;
1789 }
1790
1791 if (length > hw->sli.sge_supported_length) {
1792 efc_log_err(hw->os,
1793 "length of SGE %d bigger than allowed %d\n",
1794 length, hw->sli.sge_supported_length);
1795 return -EIO;
1796 }
1797
1798 data = io->sgl->virt;
1799 data += io->n_sge;
1800
1801 sge_flags = le32_to_cpu(data->dw2_flags);
1802 sge_flags &= ~SLI4_SGE_TYPE_MASK;
1803 sge_flags |= SLI4_SGE_TYPE_DATA << SLI4_SGE_TYPE_SHIFT;
1804 sge_flags &= ~SLI4_SGE_DATA_OFFSET_MASK;
1805 sge_flags |= SLI4_SGE_DATA_OFFSET_MASK & io->sge_offset;
1806
1807 data->buffer_address_high = cpu_to_le32(upper_32_bits(addr));
1808 data->buffer_address_low = cpu_to_le32(lower_32_bits(addr));
1809 data->buffer_length = cpu_to_le32(length);
1810
1811 /*
1812 * Always assume this is the last entry and mark as such.
1813 * If this is not the first entry unset the "last SGE"
1814 * indication for the previous entry
1815 */
1816 sge_flags |= SLI4_SGE_LAST;
1817 data->dw2_flags = cpu_to_le32(sge_flags);
1818
1819 if (io->n_sge) {
1820 sge_flags = le32_to_cpu(data[-1].dw2_flags);
1821 sge_flags &= ~SLI4_SGE_LAST;
1822 data[-1].dw2_flags = cpu_to_le32(sge_flags);
1823 }
1824
1825 /* Set first_data_bde if not previously set */
1826 if (io->first_data_sge == 0)
1827 io->first_data_sge = io->n_sge;
1828
1829 io->sge_offset += length;
1830 io->n_sge++;
1831
1832 return 0;
1833 }
1834
1835 void
efct_hw_io_abort_all(struct efct_hw * hw)1836 efct_hw_io_abort_all(struct efct_hw *hw)
1837 {
1838 struct efct_hw_io *io_to_abort = NULL;
1839 struct efct_hw_io *next_io = NULL;
1840
1841 list_for_each_entry_safe(io_to_abort, next_io,
1842 &hw->io_inuse, list_entry) {
1843 efct_hw_io_abort(hw, io_to_abort, true, NULL, NULL);
1844 }
1845 }
1846
1847 static void
efct_hw_wq_process_abort(void * arg,u8 * cqe,int status)1848 efct_hw_wq_process_abort(void *arg, u8 *cqe, int status)
1849 {
1850 struct efct_hw_io *io = arg;
1851 struct efct_hw *hw = io->hw;
1852 u32 ext = 0;
1853 u32 len = 0;
1854 struct hw_wq_callback *wqcb;
1855
1856 /*
1857 * For IOs that were aborted internally, we may need to issue the
1858 * callback here depending on whether a XRI_ABORTED CQE is expected ot
1859 * not. If the status is Local Reject/No XRI, then
1860 * issue the callback now.
1861 */
1862 ext = sli_fc_ext_status(&hw->sli, cqe);
1863 if (status == SLI4_FC_WCQE_STATUS_LOCAL_REJECT &&
1864 ext == SLI4_FC_LOCAL_REJECT_NO_XRI && io->done) {
1865 efct_hw_done_t done = io->done;
1866
1867 io->done = NULL;
1868
1869 /*
1870 * Use latched status as this is always saved for an internal
1871 * abort Note: We won't have both a done and abort_done
1872 * function, so don't worry about
1873 * clobbering the len, status and ext fields.
1874 */
1875 status = io->saved_status;
1876 len = io->saved_len;
1877 ext = io->saved_ext;
1878 io->status_saved = false;
1879 done(io, len, status, ext, io->arg);
1880 }
1881
1882 if (io->abort_done) {
1883 efct_hw_done_t done = io->abort_done;
1884
1885 io->abort_done = NULL;
1886 done(io, len, status, ext, io->abort_arg);
1887 }
1888
1889 /* clear abort bit to indicate abort is complete */
1890 io->abort_in_progress = false;
1891
1892 /* Free the WQ callback */
1893 if (io->abort_reqtag == U32_MAX) {
1894 efc_log_err(hw->os, "HW IO already freed\n");
1895 return;
1896 }
1897
1898 wqcb = efct_hw_reqtag_get_instance(hw, io->abort_reqtag);
1899 efct_hw_reqtag_free(hw, wqcb);
1900
1901 /*
1902 * Call efct_hw_io_free() because this releases the WQ reservation as
1903 * well as doing the refcount put. Don't duplicate the code here.
1904 */
1905 (void)efct_hw_io_free(hw, io);
1906 }
1907
1908 static void
efct_hw_fill_abort_wqe(struct efct_hw * hw,struct efct_hw_wqe * wqe)1909 efct_hw_fill_abort_wqe(struct efct_hw *hw, struct efct_hw_wqe *wqe)
1910 {
1911 struct sli4_abort_wqe *abort = (void *)wqe->wqebuf;
1912
1913 memset(abort, 0, hw->sli.wqe_size);
1914
1915 abort->criteria = SLI4_ABORT_CRITERIA_XRI_TAG;
1916 abort->ia_ir_byte |= wqe->send_abts ? 0 : 1;
1917
1918 /* Suppress ABTS retries */
1919 abort->ia_ir_byte |= SLI4_ABRT_WQE_IR;
1920
1921 abort->t_tag = cpu_to_le32(wqe->id);
1922 abort->command = SLI4_WQE_ABORT;
1923 abort->request_tag = cpu_to_le16(wqe->abort_reqtag);
1924
1925 abort->dw10w0_flags = cpu_to_le16(SLI4_ABRT_WQE_QOSD);
1926
1927 abort->cq_id = cpu_to_le16(SLI4_CQ_DEFAULT);
1928 }
1929
1930 int
efct_hw_io_abort(struct efct_hw * hw,struct efct_hw_io * io_to_abort,bool send_abts,void * cb,void * arg)1931 efct_hw_io_abort(struct efct_hw *hw, struct efct_hw_io *io_to_abort,
1932 bool send_abts, void *cb, void *arg)
1933 {
1934 struct hw_wq_callback *wqcb;
1935 unsigned long flags = 0;
1936
1937 if (!io_to_abort) {
1938 efc_log_err(hw->os, "bad parameter hw=%p io=%p\n",
1939 hw, io_to_abort);
1940 return -EIO;
1941 }
1942
1943 if (hw->state != EFCT_HW_STATE_ACTIVE) {
1944 efc_log_err(hw->os, "cannot send IO abort, HW state=%d\n",
1945 hw->state);
1946 return -EIO;
1947 }
1948
1949 /* take a reference on IO being aborted */
1950 if (kref_get_unless_zero(&io_to_abort->ref) == 0) {
1951 /* command no longer active */
1952 efc_log_debug(hw->os,
1953 "io not active xri=0x%x tag=0x%x\n",
1954 io_to_abort->indicator, io_to_abort->reqtag);
1955 return -ENOENT;
1956 }
1957
1958 /* Must have a valid WQ reference */
1959 if (!io_to_abort->wq) {
1960 efc_log_debug(hw->os, "io_to_abort xri=0x%x not active on WQ\n",
1961 io_to_abort->indicator);
1962 /* efct_ref_get(): same function */
1963 kref_put(&io_to_abort->ref, io_to_abort->release);
1964 return -ENOENT;
1965 }
1966
1967 /*
1968 * Validation checks complete; now check to see if already being
1969 * aborted, if not set the flag.
1970 */
1971 if (cmpxchg(&io_to_abort->abort_in_progress, false, true)) {
1972 /* efct_ref_get(): same function */
1973 kref_put(&io_to_abort->ref, io_to_abort->release);
1974 efc_log_debug(hw->os,
1975 "io already being aborted xri=0x%x tag=0x%x\n",
1976 io_to_abort->indicator, io_to_abort->reqtag);
1977 return -EINPROGRESS;
1978 }
1979
1980 /*
1981 * If we got here, the possibilities are:
1982 * - host owned xri
1983 * - io_to_abort->wq_index != U32_MAX
1984 * - submit ABORT_WQE to same WQ
1985 * - port owned xri:
1986 * - rxri: io_to_abort->wq_index == U32_MAX
1987 * - submit ABORT_WQE to any WQ
1988 * - non-rxri
1989 * - io_to_abort->index != U32_MAX
1990 * - submit ABORT_WQE to same WQ
1991 * - io_to_abort->index == U32_MAX
1992 * - submit ABORT_WQE to any WQ
1993 */
1994 io_to_abort->abort_done = cb;
1995 io_to_abort->abort_arg = arg;
1996
1997 /* Allocate a request tag for the abort portion of this IO */
1998 wqcb = efct_hw_reqtag_alloc(hw, efct_hw_wq_process_abort, io_to_abort);
1999 if (!wqcb) {
2000 efc_log_err(hw->os, "can't allocate request tag\n");
2001 return -ENOSPC;
2002 }
2003
2004 io_to_abort->abort_reqtag = wqcb->instance_index;
2005 io_to_abort->wqe.send_abts = send_abts;
2006 io_to_abort->wqe.id = io_to_abort->indicator;
2007 io_to_abort->wqe.abort_reqtag = io_to_abort->abort_reqtag;
2008
2009 /*
2010 * If the wqe is on the pending list, then set this wqe to be
2011 * aborted when the IO's wqe is removed from the list.
2012 */
2013 if (io_to_abort->wq) {
2014 spin_lock_irqsave(&io_to_abort->wq->queue->lock, flags);
2015 if (io_to_abort->wqe.list_entry.next) {
2016 io_to_abort->wqe.abort_wqe_submit_needed = true;
2017 spin_unlock_irqrestore(&io_to_abort->wq->queue->lock,
2018 flags);
2019 return 0;
2020 }
2021 spin_unlock_irqrestore(&io_to_abort->wq->queue->lock, flags);
2022 }
2023
2024 efct_hw_fill_abort_wqe(hw, &io_to_abort->wqe);
2025
2026 /* ABORT_WQE does not actually utilize an XRI on the Port,
2027 * therefore, keep xbusy as-is to track the exchange's state,
2028 * not the ABORT_WQE's state
2029 */
2030 if (efct_hw_wq_write(io_to_abort->wq, &io_to_abort->wqe)) {
2031 io_to_abort->abort_in_progress = false;
2032 /* efct_ref_get(): same function */
2033 kref_put(&io_to_abort->ref, io_to_abort->release);
2034 return -EIO;
2035 }
2036
2037 return 0;
2038 }
2039
2040 void
efct_hw_reqtag_pool_free(struct efct_hw * hw)2041 efct_hw_reqtag_pool_free(struct efct_hw *hw)
2042 {
2043 u32 i;
2044 struct reqtag_pool *reqtag_pool = hw->wq_reqtag_pool;
2045 struct hw_wq_callback *wqcb = NULL;
2046
2047 if (reqtag_pool) {
2048 for (i = 0; i < U16_MAX; i++) {
2049 wqcb = reqtag_pool->tags[i];
2050 if (!wqcb)
2051 continue;
2052
2053 kfree(wqcb);
2054 }
2055 kfree(reqtag_pool);
2056 hw->wq_reqtag_pool = NULL;
2057 }
2058 }
2059
2060 struct reqtag_pool *
efct_hw_reqtag_pool_alloc(struct efct_hw * hw)2061 efct_hw_reqtag_pool_alloc(struct efct_hw *hw)
2062 {
2063 u32 i = 0;
2064 struct reqtag_pool *reqtag_pool;
2065 struct hw_wq_callback *wqcb;
2066
2067 reqtag_pool = kzalloc(sizeof(*reqtag_pool), GFP_KERNEL);
2068 if (!reqtag_pool)
2069 return NULL;
2070
2071 INIT_LIST_HEAD(&reqtag_pool->freelist);
2072 /* initialize reqtag pool lock */
2073 spin_lock_init(&reqtag_pool->lock);
2074 for (i = 0; i < U16_MAX; i++) {
2075 wqcb = kmalloc(sizeof(*wqcb), GFP_KERNEL);
2076 if (!wqcb)
2077 break;
2078
2079 reqtag_pool->tags[i] = wqcb;
2080 wqcb->instance_index = i;
2081 wqcb->callback = NULL;
2082 wqcb->arg = NULL;
2083 INIT_LIST_HEAD(&wqcb->list_entry);
2084 list_add_tail(&wqcb->list_entry, &reqtag_pool->freelist);
2085 }
2086
2087 return reqtag_pool;
2088 }
2089
2090 struct hw_wq_callback *
efct_hw_reqtag_alloc(struct efct_hw * hw,void (* callback)(void * arg,u8 * cqe,int status),void * arg)2091 efct_hw_reqtag_alloc(struct efct_hw *hw,
2092 void (*callback)(void *arg, u8 *cqe, int status),
2093 void *arg)
2094 {
2095 struct hw_wq_callback *wqcb = NULL;
2096 struct reqtag_pool *reqtag_pool = hw->wq_reqtag_pool;
2097 unsigned long flags = 0;
2098
2099 if (!callback)
2100 return wqcb;
2101
2102 spin_lock_irqsave(&reqtag_pool->lock, flags);
2103
2104 if (!list_empty(&reqtag_pool->freelist)) {
2105 wqcb = list_first_entry(&reqtag_pool->freelist,
2106 struct hw_wq_callback, list_entry);
2107 }
2108
2109 if (wqcb) {
2110 list_del_init(&wqcb->list_entry);
2111 spin_unlock_irqrestore(&reqtag_pool->lock, flags);
2112 wqcb->callback = callback;
2113 wqcb->arg = arg;
2114 } else {
2115 spin_unlock_irqrestore(&reqtag_pool->lock, flags);
2116 }
2117
2118 return wqcb;
2119 }
2120
2121 void
efct_hw_reqtag_free(struct efct_hw * hw,struct hw_wq_callback * wqcb)2122 efct_hw_reqtag_free(struct efct_hw *hw, struct hw_wq_callback *wqcb)
2123 {
2124 unsigned long flags = 0;
2125 struct reqtag_pool *reqtag_pool = hw->wq_reqtag_pool;
2126
2127 if (!wqcb->callback)
2128 efc_log_err(hw->os, "WQCB is already freed\n");
2129
2130 spin_lock_irqsave(&reqtag_pool->lock, flags);
2131 wqcb->callback = NULL;
2132 wqcb->arg = NULL;
2133 INIT_LIST_HEAD(&wqcb->list_entry);
2134 list_add(&wqcb->list_entry, &hw->wq_reqtag_pool->freelist);
2135 spin_unlock_irqrestore(&reqtag_pool->lock, flags);
2136 }
2137
2138 struct hw_wq_callback *
efct_hw_reqtag_get_instance(struct efct_hw * hw,u32 instance_index)2139 efct_hw_reqtag_get_instance(struct efct_hw *hw, u32 instance_index)
2140 {
2141 struct hw_wq_callback *wqcb;
2142
2143 wqcb = hw->wq_reqtag_pool->tags[instance_index];
2144 if (!wqcb)
2145 efc_log_err(hw->os, "wqcb for instance %d is null\n",
2146 instance_index);
2147
2148 return wqcb;
2149 }
2150
2151 int
efct_hw_queue_hash_find(struct efct_queue_hash * hash,u16 id)2152 efct_hw_queue_hash_find(struct efct_queue_hash *hash, u16 id)
2153 {
2154 int index = -1;
2155 int i = id & (EFCT_HW_Q_HASH_SIZE - 1);
2156
2157 /*
2158 * Since the hash is always bigger than the maximum number of Qs, then
2159 * we never have to worry about an infinite loop. We will always find
2160 * an unused entry.
2161 */
2162 do {
2163 if (hash[i].in_use && hash[i].id == id)
2164 index = hash[i].index;
2165 else
2166 i = (i + 1) & (EFCT_HW_Q_HASH_SIZE - 1);
2167 } while (index == -1 && hash[i].in_use);
2168
2169 return index;
2170 }
2171
2172 int
efct_hw_process(struct efct_hw * hw,u32 vector,u32 max_isr_time_msec)2173 efct_hw_process(struct efct_hw *hw, u32 vector,
2174 u32 max_isr_time_msec)
2175 {
2176 struct hw_eq *eq;
2177
2178 /*
2179 * The caller should disable interrupts if they wish to prevent us
2180 * from processing during a shutdown. The following states are defined:
2181 * EFCT_HW_STATE_UNINITIALIZED - No queues allocated
2182 * EFCT_HW_STATE_QUEUES_ALLOCATED - The state after a chip reset,
2183 * queues are cleared.
2184 * EFCT_HW_STATE_ACTIVE - Chip and queues are operational
2185 * EFCT_HW_STATE_RESET_IN_PROGRESS - reset, we still want completions
2186 * EFCT_HW_STATE_TEARDOWN_IN_PROGRESS - We still want mailbox
2187 * completions.
2188 */
2189 if (hw->state == EFCT_HW_STATE_UNINITIALIZED)
2190 return 0;
2191
2192 /* Get pointer to struct hw_eq */
2193 eq = hw->hw_eq[vector];
2194 if (!eq)
2195 return 0;
2196
2197 eq->use_count++;
2198
2199 return efct_hw_eq_process(hw, eq, max_isr_time_msec);
2200 }
2201
2202 int
efct_hw_eq_process(struct efct_hw * hw,struct hw_eq * eq,u32 max_isr_time_msec)2203 efct_hw_eq_process(struct efct_hw *hw, struct hw_eq *eq,
2204 u32 max_isr_time_msec)
2205 {
2206 u8 eqe[sizeof(struct sli4_eqe)] = { 0 };
2207 u32 tcheck_count;
2208 u64 tstart;
2209 u64 telapsed;
2210 bool done = false;
2211
2212 tcheck_count = EFCT_HW_TIMECHECK_ITERATIONS;
2213 tstart = jiffies_to_msecs(jiffies);
2214
2215 while (!done && !sli_eq_read(&hw->sli, eq->queue, eqe)) {
2216 u16 cq_id = 0;
2217 int rc;
2218
2219 rc = sli_eq_parse(&hw->sli, eqe, &cq_id);
2220 if (unlikely(rc)) {
2221 if (rc == SLI4_EQE_STATUS_EQ_FULL) {
2222 u32 i;
2223
2224 /*
2225 * Received a sentinel EQE indicating the
2226 * EQ is full. Process all CQs
2227 */
2228 for (i = 0; i < hw->cq_count; i++)
2229 efct_hw_cq_process(hw, hw->hw_cq[i]);
2230 continue;
2231 } else {
2232 return rc;
2233 }
2234 } else {
2235 int index;
2236
2237 index = efct_hw_queue_hash_find(hw->cq_hash, cq_id);
2238
2239 if (likely(index >= 0))
2240 efct_hw_cq_process(hw, hw->hw_cq[index]);
2241 else
2242 efc_log_err(hw->os, "bad CQ_ID %#06x\n", cq_id);
2243 }
2244
2245 if (eq->queue->n_posted > eq->queue->posted_limit)
2246 sli_queue_arm(&hw->sli, eq->queue, false);
2247
2248 if (tcheck_count && (--tcheck_count == 0)) {
2249 tcheck_count = EFCT_HW_TIMECHECK_ITERATIONS;
2250 telapsed = jiffies_to_msecs(jiffies) - tstart;
2251 if (telapsed >= max_isr_time_msec)
2252 done = true;
2253 }
2254 }
2255 sli_queue_eq_arm(&hw->sli, eq->queue, true);
2256
2257 return 0;
2258 }
2259
2260 static int
_efct_hw_wq_write(struct hw_wq * wq,struct efct_hw_wqe * wqe)2261 _efct_hw_wq_write(struct hw_wq *wq, struct efct_hw_wqe *wqe)
2262 {
2263 int queue_rc;
2264
2265 /* Every so often, set the wqec bit to generate comsummed completions */
2266 if (wq->wqec_count)
2267 wq->wqec_count--;
2268
2269 if (wq->wqec_count == 0) {
2270 struct sli4_generic_wqe *genwqe = (void *)wqe->wqebuf;
2271
2272 genwqe->cmdtype_wqec_byte |= SLI4_GEN_WQE_WQEC;
2273 wq->wqec_count = wq->wqec_set_count;
2274 }
2275
2276 /* Decrement WQ free count */
2277 wq->free_count--;
2278
2279 queue_rc = sli_wq_write(&wq->hw->sli, wq->queue, wqe->wqebuf);
2280
2281 return (queue_rc < 0) ? -EIO : 0;
2282 }
2283
2284 static void
hw_wq_submit_pending(struct hw_wq * wq,u32 update_free_count)2285 hw_wq_submit_pending(struct hw_wq *wq, u32 update_free_count)
2286 {
2287 struct efct_hw_wqe *wqe;
2288 unsigned long flags = 0;
2289
2290 spin_lock_irqsave(&wq->queue->lock, flags);
2291
2292 /* Update free count with value passed in */
2293 wq->free_count += update_free_count;
2294
2295 while ((wq->free_count > 0) && (!list_empty(&wq->pending_list))) {
2296 wqe = list_first_entry(&wq->pending_list,
2297 struct efct_hw_wqe, list_entry);
2298 list_del_init(&wqe->list_entry);
2299 _efct_hw_wq_write(wq, wqe);
2300
2301 if (wqe->abort_wqe_submit_needed) {
2302 wqe->abort_wqe_submit_needed = false;
2303 efct_hw_fill_abort_wqe(wq->hw, wqe);
2304 INIT_LIST_HEAD(&wqe->list_entry);
2305 list_add_tail(&wqe->list_entry, &wq->pending_list);
2306 wq->wq_pending_count++;
2307 }
2308 }
2309
2310 spin_unlock_irqrestore(&wq->queue->lock, flags);
2311 }
2312
2313 void
efct_hw_cq_process(struct efct_hw * hw,struct hw_cq * cq)2314 efct_hw_cq_process(struct efct_hw *hw, struct hw_cq *cq)
2315 {
2316 u8 cqe[sizeof(struct sli4_mcqe)];
2317 u16 rid = U16_MAX;
2318 /* completion type */
2319 enum sli4_qentry ctype;
2320 u32 n_processed = 0;
2321 u32 tstart, telapsed;
2322
2323 tstart = jiffies_to_msecs(jiffies);
2324
2325 while (!sli_cq_read(&hw->sli, cq->queue, cqe)) {
2326 int status;
2327
2328 status = sli_cq_parse(&hw->sli, cq->queue, cqe, &ctype, &rid);
2329 /*
2330 * The sign of status is significant. If status is:
2331 * == 0 : call completed correctly and
2332 * the CQE indicated success
2333 * > 0 : call completed correctly and
2334 * the CQE indicated an error
2335 * < 0 : call failed and no information is available about the
2336 * CQE
2337 */
2338 if (status < 0) {
2339 if (status == SLI4_MCQE_STATUS_NOT_COMPLETED)
2340 /*
2341 * Notification that an entry was consumed,
2342 * but not completed
2343 */
2344 continue;
2345
2346 break;
2347 }
2348
2349 switch (ctype) {
2350 case SLI4_QENTRY_ASYNC:
2351 sli_cqe_async(&hw->sli, cqe);
2352 break;
2353 case SLI4_QENTRY_MQ:
2354 /*
2355 * Process MQ entry. Note there is no way to determine
2356 * the MQ_ID from the completion entry.
2357 */
2358 efct_hw_mq_process(hw, status, hw->mq);
2359 break;
2360 case SLI4_QENTRY_WQ:
2361 efct_hw_wq_process(hw, cq, cqe, status, rid);
2362 break;
2363 case SLI4_QENTRY_WQ_RELEASE: {
2364 u32 wq_id = rid;
2365 int index;
2366 struct hw_wq *wq = NULL;
2367
2368 index = efct_hw_queue_hash_find(hw->wq_hash, wq_id);
2369
2370 if (likely(index >= 0)) {
2371 wq = hw->hw_wq[index];
2372 } else {
2373 efc_log_err(hw->os, "bad WQ_ID %#06x\n", wq_id);
2374 break;
2375 }
2376 /* Submit any HW IOs that are on the WQ pending list */
2377 hw_wq_submit_pending(wq, wq->wqec_set_count);
2378
2379 break;
2380 }
2381
2382 case SLI4_QENTRY_RQ:
2383 efct_hw_rqpair_process_rq(hw, cq, cqe);
2384 break;
2385 case SLI4_QENTRY_XABT: {
2386 efct_hw_xabt_process(hw, cq, cqe, rid);
2387 break;
2388 }
2389 default:
2390 efc_log_debug(hw->os, "unhandled ctype=%#x rid=%#x\n",
2391 ctype, rid);
2392 break;
2393 }
2394
2395 n_processed++;
2396 if (n_processed == cq->queue->proc_limit)
2397 break;
2398
2399 if (cq->queue->n_posted >= cq->queue->posted_limit)
2400 sli_queue_arm(&hw->sli, cq->queue, false);
2401 }
2402
2403 sli_queue_arm(&hw->sli, cq->queue, true);
2404
2405 if (n_processed > cq->queue->max_num_processed)
2406 cq->queue->max_num_processed = n_processed;
2407 telapsed = jiffies_to_msecs(jiffies) - tstart;
2408 if (telapsed > cq->queue->max_process_time)
2409 cq->queue->max_process_time = telapsed;
2410 }
2411
2412 void
efct_hw_wq_process(struct efct_hw * hw,struct hw_cq * cq,u8 * cqe,int status,u16 rid)2413 efct_hw_wq_process(struct efct_hw *hw, struct hw_cq *cq,
2414 u8 *cqe, int status, u16 rid)
2415 {
2416 struct hw_wq_callback *wqcb;
2417
2418 if (rid == EFCT_HW_REQUE_XRI_REGTAG) {
2419 if (status)
2420 efc_log_err(hw->os, "reque xri failed, status = %d\n",
2421 status);
2422 return;
2423 }
2424
2425 wqcb = efct_hw_reqtag_get_instance(hw, rid);
2426 if (!wqcb) {
2427 efc_log_err(hw->os, "invalid request tag: x%x\n", rid);
2428 return;
2429 }
2430
2431 if (!wqcb->callback) {
2432 efc_log_err(hw->os, "wqcb callback is NULL\n");
2433 return;
2434 }
2435
2436 (*wqcb->callback)(wqcb->arg, cqe, status);
2437 }
2438
2439 void
efct_hw_xabt_process(struct efct_hw * hw,struct hw_cq * cq,u8 * cqe,u16 rid)2440 efct_hw_xabt_process(struct efct_hw *hw, struct hw_cq *cq,
2441 u8 *cqe, u16 rid)
2442 {
2443 /* search IOs wait free list */
2444 struct efct_hw_io *io = NULL;
2445 unsigned long flags = 0;
2446
2447 io = efct_hw_io_lookup(hw, rid);
2448 if (!io) {
2449 /* IO lookup failure should never happen */
2450 efc_log_err(hw->os, "xabt io lookup failed rid=%#x\n", rid);
2451 return;
2452 }
2453
2454 if (!io->xbusy)
2455 efc_log_debug(hw->os, "xabt io not busy rid=%#x\n", rid);
2456 else
2457 /* mark IO as no longer busy */
2458 io->xbusy = false;
2459
2460 /*
2461 * For IOs that were aborted internally, we need to issue any pending
2462 * callback here.
2463 */
2464 if (io->done) {
2465 efct_hw_done_t done = io->done;
2466 void *arg = io->arg;
2467
2468 /*
2469 * Use latched status as this is always saved for an internal
2470 * abort
2471 */
2472 int status = io->saved_status;
2473 u32 len = io->saved_len;
2474 u32 ext = io->saved_ext;
2475
2476 io->done = NULL;
2477 io->status_saved = false;
2478
2479 done(io, len, status, ext, arg);
2480 }
2481
2482 spin_lock_irqsave(&hw->io_lock, flags);
2483 if (io->state == EFCT_HW_IO_STATE_INUSE ||
2484 io->state == EFCT_HW_IO_STATE_WAIT_FREE) {
2485 /* if on wait_free list, caller has already freed IO;
2486 * remove from wait_free list and add to free list.
2487 * if on in-use list, already marked as no longer busy;
2488 * just leave there and wait for caller to free.
2489 */
2490 if (io->state == EFCT_HW_IO_STATE_WAIT_FREE) {
2491 io->state = EFCT_HW_IO_STATE_FREE;
2492 list_del_init(&io->list_entry);
2493 efct_hw_io_free_move_correct_list(hw, io);
2494 }
2495 }
2496 spin_unlock_irqrestore(&hw->io_lock, flags);
2497 }
2498
2499 static int
efct_hw_flush(struct efct_hw * hw)2500 efct_hw_flush(struct efct_hw *hw)
2501 {
2502 u32 i = 0;
2503
2504 /* Process any remaining completions */
2505 for (i = 0; i < hw->eq_count; i++)
2506 efct_hw_process(hw, i, ~0);
2507
2508 return 0;
2509 }
2510
2511 int
efct_hw_wq_write(struct hw_wq * wq,struct efct_hw_wqe * wqe)2512 efct_hw_wq_write(struct hw_wq *wq, struct efct_hw_wqe *wqe)
2513 {
2514 int rc = 0;
2515 unsigned long flags = 0;
2516
2517 spin_lock_irqsave(&wq->queue->lock, flags);
2518 if (list_empty(&wq->pending_list)) {
2519 if (wq->free_count > 0) {
2520 rc = _efct_hw_wq_write(wq, wqe);
2521 } else {
2522 INIT_LIST_HEAD(&wqe->list_entry);
2523 list_add_tail(&wqe->list_entry, &wq->pending_list);
2524 wq->wq_pending_count++;
2525 }
2526
2527 spin_unlock_irqrestore(&wq->queue->lock, flags);
2528 return rc;
2529 }
2530
2531 INIT_LIST_HEAD(&wqe->list_entry);
2532 list_add_tail(&wqe->list_entry, &wq->pending_list);
2533 wq->wq_pending_count++;
2534 while (wq->free_count > 0) {
2535 wqe = list_first_entry(&wq->pending_list, struct efct_hw_wqe,
2536 list_entry);
2537 if (!wqe)
2538 break;
2539
2540 list_del_init(&wqe->list_entry);
2541 rc = _efct_hw_wq_write(wq, wqe);
2542 if (rc)
2543 break;
2544
2545 if (wqe->abort_wqe_submit_needed) {
2546 wqe->abort_wqe_submit_needed = false;
2547 efct_hw_fill_abort_wqe(wq->hw, wqe);
2548
2549 INIT_LIST_HEAD(&wqe->list_entry);
2550 list_add_tail(&wqe->list_entry, &wq->pending_list);
2551 wq->wq_pending_count++;
2552 }
2553 }
2554
2555 spin_unlock_irqrestore(&wq->queue->lock, flags);
2556
2557 return rc;
2558 }
2559
2560 int
efct_efc_bls_send(struct efc * efc,u32 type,struct sli_bls_params * bls)2561 efct_efc_bls_send(struct efc *efc, u32 type, struct sli_bls_params *bls)
2562 {
2563 struct efct *efct = efc->base;
2564
2565 return efct_hw_bls_send(efct, type, bls, NULL, NULL);
2566 }
2567
2568 int
efct_hw_bls_send(struct efct * efct,u32 type,struct sli_bls_params * bls_params,void * cb,void * arg)2569 efct_hw_bls_send(struct efct *efct, u32 type, struct sli_bls_params *bls_params,
2570 void *cb, void *arg)
2571 {
2572 struct efct_hw *hw = &efct->hw;
2573 struct efct_hw_io *hio;
2574 struct sli_bls_payload bls;
2575 int rc;
2576
2577 if (hw->state != EFCT_HW_STATE_ACTIVE) {
2578 efc_log_err(hw->os,
2579 "cannot send BLS, HW state=%d\n", hw->state);
2580 return -EIO;
2581 }
2582
2583 hio = efct_hw_io_alloc(hw);
2584 if (!hio) {
2585 efc_log_err(hw->os, "HIO allocation failed\n");
2586 return -EIO;
2587 }
2588
2589 hio->done = cb;
2590 hio->arg = arg;
2591
2592 bls_params->xri = hio->indicator;
2593 bls_params->tag = hio->reqtag;
2594
2595 if (type == FC_RCTL_BA_ACC) {
2596 hio->type = EFCT_HW_BLS_ACC;
2597 bls.type = SLI4_SLI_BLS_ACC;
2598 memcpy(&bls.u.acc, bls_params->payload, sizeof(bls.u.acc));
2599 } else {
2600 hio->type = EFCT_HW_BLS_RJT;
2601 bls.type = SLI4_SLI_BLS_RJT;
2602 memcpy(&bls.u.rjt, bls_params->payload, sizeof(bls.u.rjt));
2603 }
2604
2605 bls.ox_id = cpu_to_le16(bls_params->ox_id);
2606 bls.rx_id = cpu_to_le16(bls_params->rx_id);
2607
2608 if (sli_xmit_bls_rsp64_wqe(&hw->sli, hio->wqe.wqebuf,
2609 &bls, bls_params)) {
2610 efc_log_err(hw->os, "XMIT_BLS_RSP64 WQE error\n");
2611 return -EIO;
2612 }
2613
2614 hio->xbusy = true;
2615
2616 /*
2617 * Add IO to active io wqe list before submitting, in case the
2618 * wcqe processing preempts this thread.
2619 */
2620 hio->wq->use_count++;
2621 rc = efct_hw_wq_write(hio->wq, &hio->wqe);
2622 if (rc >= 0) {
2623 /* non-negative return is success */
2624 rc = 0;
2625 } else {
2626 /* failed to write wqe, remove from active wqe list */
2627 efc_log_err(hw->os,
2628 "sli_queue_write failed: %d\n", rc);
2629 hio->xbusy = false;
2630 }
2631
2632 return rc;
2633 }
2634
2635 static int
efct_els_ssrs_send_cb(struct efct_hw_io * hio,u32 length,int status,u32 ext_status,void * arg)2636 efct_els_ssrs_send_cb(struct efct_hw_io *hio, u32 length, int status,
2637 u32 ext_status, void *arg)
2638 {
2639 struct efc_disc_io *io = arg;
2640
2641 efc_disc_io_complete(io, length, status, ext_status);
2642 return 0;
2643 }
2644
2645 static inline void
efct_fill_els_params(struct efc_disc_io * io,struct sli_els_params * params)2646 efct_fill_els_params(struct efc_disc_io *io, struct sli_els_params *params)
2647 {
2648 u8 *cmd = io->req.virt;
2649
2650 params->cmd = *cmd;
2651 params->s_id = io->s_id;
2652 params->d_id = io->d_id;
2653 params->ox_id = io->iparam.els.ox_id;
2654 params->rpi = io->rpi;
2655 params->vpi = io->vpi;
2656 params->rpi_registered = io->rpi_registered;
2657 params->xmit_len = io->xmit_len;
2658 params->rsp_len = io->rsp_len;
2659 params->timeout = io->iparam.els.timeout;
2660 }
2661
2662 static inline void
efct_fill_ct_params(struct efc_disc_io * io,struct sli_ct_params * params)2663 efct_fill_ct_params(struct efc_disc_io *io, struct sli_ct_params *params)
2664 {
2665 params->r_ctl = io->iparam.ct.r_ctl;
2666 params->type = io->iparam.ct.type;
2667 params->df_ctl = io->iparam.ct.df_ctl;
2668 params->d_id = io->d_id;
2669 params->ox_id = io->iparam.ct.ox_id;
2670 params->rpi = io->rpi;
2671 params->vpi = io->vpi;
2672 params->rpi_registered = io->rpi_registered;
2673 params->xmit_len = io->xmit_len;
2674 params->rsp_len = io->rsp_len;
2675 params->timeout = io->iparam.ct.timeout;
2676 }
2677
2678 /**
2679 * efct_els_hw_srrs_send() - Send a single request and response cmd.
2680 * @efc: efc library structure
2681 * @io: Discovery IO used to hold els and ct cmd context.
2682 *
2683 * This routine supports communication sequences consisting of a single
2684 * request and single response between two endpoints. Examples include:
2685 * - Sending an ELS request.
2686 * - Sending an ELS response - To send an ELS response, the caller must provide
2687 * the OX_ID from the received request.
2688 * - Sending a FC Common Transport (FC-CT) request - To send a FC-CT request,
2689 * the caller must provide the R_CTL, TYPE, and DF_CTL
2690 * values to place in the FC frame header.
2691 *
2692 * Return: Status of the request.
2693 */
2694 int
efct_els_hw_srrs_send(struct efc * efc,struct efc_disc_io * io)2695 efct_els_hw_srrs_send(struct efc *efc, struct efc_disc_io *io)
2696 {
2697 struct efct *efct = efc->base;
2698 struct efct_hw_io *hio;
2699 struct efct_hw *hw = &efct->hw;
2700 struct efc_dma *send = &io->req;
2701 struct efc_dma *receive = &io->rsp;
2702 struct sli4_sge *sge = NULL;
2703 int rc = 0;
2704 u32 len = io->xmit_len;
2705 u32 sge0_flags;
2706 u32 sge1_flags;
2707
2708 hio = efct_hw_io_alloc(hw);
2709 if (!hio) {
2710 pr_err("HIO alloc failed\n");
2711 return -EIO;
2712 }
2713
2714 if (hw->state != EFCT_HW_STATE_ACTIVE) {
2715 efc_log_debug(hw->os,
2716 "cannot send SRRS, HW state=%d\n", hw->state);
2717 return -EIO;
2718 }
2719
2720 hio->done = efct_els_ssrs_send_cb;
2721 hio->arg = io;
2722
2723 sge = hio->sgl->virt;
2724
2725 /* clear both SGE */
2726 memset(hio->sgl->virt, 0, 2 * sizeof(struct sli4_sge));
2727
2728 sge0_flags = le32_to_cpu(sge[0].dw2_flags);
2729 sge1_flags = le32_to_cpu(sge[1].dw2_flags);
2730 if (send->size) {
2731 sge[0].buffer_address_high =
2732 cpu_to_le32(upper_32_bits(send->phys));
2733 sge[0].buffer_address_low =
2734 cpu_to_le32(lower_32_bits(send->phys));
2735
2736 sge0_flags |= (SLI4_SGE_TYPE_DATA << SLI4_SGE_TYPE_SHIFT);
2737
2738 sge[0].buffer_length = cpu_to_le32(len);
2739 }
2740
2741 if (io->io_type == EFC_DISC_IO_ELS_REQ ||
2742 io->io_type == EFC_DISC_IO_CT_REQ) {
2743 sge[1].buffer_address_high =
2744 cpu_to_le32(upper_32_bits(receive->phys));
2745 sge[1].buffer_address_low =
2746 cpu_to_le32(lower_32_bits(receive->phys));
2747
2748 sge1_flags |= (SLI4_SGE_TYPE_DATA << SLI4_SGE_TYPE_SHIFT);
2749 sge1_flags |= SLI4_SGE_LAST;
2750
2751 sge[1].buffer_length = cpu_to_le32(receive->size);
2752 } else {
2753 sge0_flags |= SLI4_SGE_LAST;
2754 }
2755
2756 sge[0].dw2_flags = cpu_to_le32(sge0_flags);
2757 sge[1].dw2_flags = cpu_to_le32(sge1_flags);
2758
2759 switch (io->io_type) {
2760 case EFC_DISC_IO_ELS_REQ: {
2761 struct sli_els_params els_params;
2762
2763 hio->type = EFCT_HW_ELS_REQ;
2764 efct_fill_els_params(io, &els_params);
2765 els_params.xri = hio->indicator;
2766 els_params.tag = hio->reqtag;
2767
2768 if (sli_els_request64_wqe(&hw->sli, hio->wqe.wqebuf, hio->sgl,
2769 &els_params)) {
2770 efc_log_err(hw->os, "REQ WQE error\n");
2771 rc = -EIO;
2772 }
2773 break;
2774 }
2775 case EFC_DISC_IO_ELS_RESP: {
2776 struct sli_els_params els_params;
2777
2778 hio->type = EFCT_HW_ELS_RSP;
2779 efct_fill_els_params(io, &els_params);
2780 els_params.xri = hio->indicator;
2781 els_params.tag = hio->reqtag;
2782 if (sli_xmit_els_rsp64_wqe(&hw->sli, hio->wqe.wqebuf, send,
2783 &els_params)){
2784 efc_log_err(hw->os, "RSP WQE error\n");
2785 rc = -EIO;
2786 }
2787 break;
2788 }
2789 case EFC_DISC_IO_CT_REQ: {
2790 struct sli_ct_params ct_params;
2791
2792 hio->type = EFCT_HW_FC_CT;
2793 efct_fill_ct_params(io, &ct_params);
2794 ct_params.xri = hio->indicator;
2795 ct_params.tag = hio->reqtag;
2796 if (sli_gen_request64_wqe(&hw->sli, hio->wqe.wqebuf, hio->sgl,
2797 &ct_params)){
2798 efc_log_err(hw->os, "GEN WQE error\n");
2799 rc = -EIO;
2800 }
2801 break;
2802 }
2803 case EFC_DISC_IO_CT_RESP: {
2804 struct sli_ct_params ct_params;
2805
2806 hio->type = EFCT_HW_FC_CT_RSP;
2807 efct_fill_ct_params(io, &ct_params);
2808 ct_params.xri = hio->indicator;
2809 ct_params.tag = hio->reqtag;
2810 if (sli_xmit_sequence64_wqe(&hw->sli, hio->wqe.wqebuf, hio->sgl,
2811 &ct_params)){
2812 efc_log_err(hw->os, "XMIT SEQ WQE error\n");
2813 rc = -EIO;
2814 }
2815 break;
2816 }
2817 default:
2818 efc_log_err(hw->os, "bad SRRS type %#x\n", io->io_type);
2819 rc = -EIO;
2820 }
2821
2822 if (rc == 0) {
2823 hio->xbusy = true;
2824
2825 /*
2826 * Add IO to active io wqe list before submitting, in case the
2827 * wcqe processing preempts this thread.
2828 */
2829 hio->wq->use_count++;
2830 rc = efct_hw_wq_write(hio->wq, &hio->wqe);
2831 if (rc >= 0) {
2832 /* non-negative return is success */
2833 rc = 0;
2834 } else {
2835 /* failed to write wqe, remove from active wqe list */
2836 efc_log_err(hw->os,
2837 "sli_queue_write failed: %d\n", rc);
2838 hio->xbusy = false;
2839 }
2840 }
2841
2842 return rc;
2843 }
2844
2845 int
efct_hw_io_send(struct efct_hw * hw,enum efct_hw_io_type type,struct efct_hw_io * io,union efct_hw_io_param_u * iparam,void * cb,void * arg)2846 efct_hw_io_send(struct efct_hw *hw, enum efct_hw_io_type type,
2847 struct efct_hw_io *io, union efct_hw_io_param_u *iparam,
2848 void *cb, void *arg)
2849 {
2850 int rc = 0;
2851 bool send_wqe = true;
2852
2853 if (!io) {
2854 pr_err("bad parm hw=%p io=%p\n", hw, io);
2855 return -EIO;
2856 }
2857
2858 if (hw->state != EFCT_HW_STATE_ACTIVE) {
2859 efc_log_err(hw->os, "cannot send IO, HW state=%d\n", hw->state);
2860 return -EIO;
2861 }
2862
2863 /*
2864 * Save state needed during later stages
2865 */
2866 io->type = type;
2867 io->done = cb;
2868 io->arg = arg;
2869
2870 /*
2871 * Format the work queue entry used to send the IO
2872 */
2873 switch (type) {
2874 case EFCT_HW_IO_TARGET_WRITE: {
2875 u16 *flags = &iparam->fcp_tgt.flags;
2876 struct fcp_txrdy *xfer = io->xfer_rdy.virt;
2877
2878 /*
2879 * Fill in the XFER_RDY for IF_TYPE 0 devices
2880 */
2881 xfer->ft_data_ro = cpu_to_be32(iparam->fcp_tgt.offset);
2882 xfer->ft_burst_len = cpu_to_be32(iparam->fcp_tgt.xmit_len);
2883
2884 if (io->xbusy)
2885 *flags |= SLI4_IO_CONTINUATION;
2886 else
2887 *flags &= ~SLI4_IO_CONTINUATION;
2888 iparam->fcp_tgt.xri = io->indicator;
2889 iparam->fcp_tgt.tag = io->reqtag;
2890
2891 if (sli_fcp_treceive64_wqe(&hw->sli, io->wqe.wqebuf,
2892 &io->def_sgl, io->first_data_sge,
2893 SLI4_CQ_DEFAULT,
2894 0, 0, &iparam->fcp_tgt)) {
2895 efc_log_err(hw->os, "TRECEIVE WQE error\n");
2896 rc = -EIO;
2897 }
2898 break;
2899 }
2900 case EFCT_HW_IO_TARGET_READ: {
2901 u16 *flags = &iparam->fcp_tgt.flags;
2902
2903 if (io->xbusy)
2904 *flags |= SLI4_IO_CONTINUATION;
2905 else
2906 *flags &= ~SLI4_IO_CONTINUATION;
2907
2908 iparam->fcp_tgt.xri = io->indicator;
2909 iparam->fcp_tgt.tag = io->reqtag;
2910
2911 if (sli_fcp_tsend64_wqe(&hw->sli, io->wqe.wqebuf,
2912 &io->def_sgl, io->first_data_sge,
2913 SLI4_CQ_DEFAULT,
2914 0, 0, &iparam->fcp_tgt)) {
2915 efc_log_err(hw->os, "TSEND WQE error\n");
2916 rc = -EIO;
2917 }
2918 break;
2919 }
2920 case EFCT_HW_IO_TARGET_RSP: {
2921 u16 *flags = &iparam->fcp_tgt.flags;
2922
2923 if (io->xbusy)
2924 *flags |= SLI4_IO_CONTINUATION;
2925 else
2926 *flags &= ~SLI4_IO_CONTINUATION;
2927
2928 iparam->fcp_tgt.xri = io->indicator;
2929 iparam->fcp_tgt.tag = io->reqtag;
2930
2931 if (sli_fcp_trsp64_wqe(&hw->sli, io->wqe.wqebuf,
2932 &io->def_sgl, SLI4_CQ_DEFAULT,
2933 0, &iparam->fcp_tgt)) {
2934 efc_log_err(hw->os, "TRSP WQE error\n");
2935 rc = -EIO;
2936 }
2937
2938 break;
2939 }
2940 default:
2941 efc_log_err(hw->os, "unsupported IO type %#x\n", type);
2942 rc = -EIO;
2943 }
2944
2945 if (send_wqe && rc == 0) {
2946 io->xbusy = true;
2947
2948 /*
2949 * Add IO to active io wqe list before submitting, in case the
2950 * wcqe processing preempts this thread.
2951 */
2952 hw->tcmd_wq_submit[io->wq->instance]++;
2953 io->wq->use_count++;
2954 rc = efct_hw_wq_write(io->wq, &io->wqe);
2955 if (rc >= 0) {
2956 /* non-negative return is success */
2957 rc = 0;
2958 } else {
2959 /* failed to write wqe, remove from active wqe list */
2960 efc_log_err(hw->os,
2961 "sli_queue_write failed: %d\n", rc);
2962 io->xbusy = false;
2963 }
2964 }
2965
2966 return rc;
2967 }
2968
2969 int
efct_hw_send_frame(struct efct_hw * hw,struct fc_frame_header * hdr,u8 sof,u8 eof,struct efc_dma * payload,struct efct_hw_send_frame_context * ctx,void (* callback)(void * arg,u8 * cqe,int status),void * arg)2970 efct_hw_send_frame(struct efct_hw *hw, struct fc_frame_header *hdr,
2971 u8 sof, u8 eof, struct efc_dma *payload,
2972 struct efct_hw_send_frame_context *ctx,
2973 void (*callback)(void *arg, u8 *cqe, int status),
2974 void *arg)
2975 {
2976 int rc;
2977 struct efct_hw_wqe *wqe;
2978 u32 xri;
2979 struct hw_wq *wq;
2980
2981 wqe = &ctx->wqe;
2982
2983 /* populate the callback object */
2984 ctx->hw = hw;
2985
2986 /* Fetch and populate request tag */
2987 ctx->wqcb = efct_hw_reqtag_alloc(hw, callback, arg);
2988 if (!ctx->wqcb) {
2989 efc_log_err(hw->os, "can't allocate request tag\n");
2990 return -ENOSPC;
2991 }
2992
2993 wq = hw->hw_wq[0];
2994
2995 /* Set XRI and RX_ID in the header based on which WQ, and which
2996 * send_frame_io we are using
2997 */
2998 xri = wq->send_frame_io->indicator;
2999
3000 /* Build the send frame WQE */
3001 rc = sli_send_frame_wqe(&hw->sli, wqe->wqebuf,
3002 sof, eof, (u32 *)hdr, payload, payload->len,
3003 EFCT_HW_SEND_FRAME_TIMEOUT, xri,
3004 ctx->wqcb->instance_index);
3005 if (rc) {
3006 efc_log_err(hw->os, "sli_send_frame_wqe failed: %d\n", rc);
3007 return -EIO;
3008 }
3009
3010 /* Write to WQ */
3011 rc = efct_hw_wq_write(wq, wqe);
3012 if (rc) {
3013 efc_log_err(hw->os, "efct_hw_wq_write failed: %d\n", rc);
3014 return -EIO;
3015 }
3016
3017 wq->use_count++;
3018
3019 return 0;
3020 }
3021
3022 static int
efct_hw_cb_link_stat(struct efct_hw * hw,int status,u8 * mqe,void * arg)3023 efct_hw_cb_link_stat(struct efct_hw *hw, int status,
3024 u8 *mqe, void *arg)
3025 {
3026 struct sli4_cmd_read_link_stats *mbox_rsp;
3027 struct efct_hw_link_stat_cb_arg *cb_arg = arg;
3028 struct efct_hw_link_stat_counts counts[EFCT_HW_LINK_STAT_MAX];
3029 u32 num_counters, i;
3030 u32 mbox_rsp_flags = 0;
3031
3032 mbox_rsp = (struct sli4_cmd_read_link_stats *)mqe;
3033 mbox_rsp_flags = le32_to_cpu(mbox_rsp->dw1_flags);
3034 num_counters = (mbox_rsp_flags & SLI4_READ_LNKSTAT_GEC) ? 20 : 13;
3035 memset(counts, 0, sizeof(struct efct_hw_link_stat_counts) *
3036 EFCT_HW_LINK_STAT_MAX);
3037
3038 /* Fill overflow counts, mask starts from SLI4_READ_LNKSTAT_W02OF*/
3039 for (i = 0; i < EFCT_HW_LINK_STAT_MAX; i++)
3040 counts[i].overflow = (mbox_rsp_flags & (1 << (i + 2)));
3041
3042 counts[EFCT_HW_LINK_STAT_LINK_FAILURE_COUNT].counter =
3043 le32_to_cpu(mbox_rsp->linkfail_errcnt);
3044 counts[EFCT_HW_LINK_STAT_LOSS_OF_SYNC_COUNT].counter =
3045 le32_to_cpu(mbox_rsp->losssync_errcnt);
3046 counts[EFCT_HW_LINK_STAT_LOSS_OF_SIGNAL_COUNT].counter =
3047 le32_to_cpu(mbox_rsp->losssignal_errcnt);
3048 counts[EFCT_HW_LINK_STAT_PRIMITIVE_SEQ_COUNT].counter =
3049 le32_to_cpu(mbox_rsp->primseq_errcnt);
3050 counts[EFCT_HW_LINK_STAT_INVALID_XMIT_WORD_COUNT].counter =
3051 le32_to_cpu(mbox_rsp->inval_txword_errcnt);
3052 counts[EFCT_HW_LINK_STAT_CRC_COUNT].counter =
3053 le32_to_cpu(mbox_rsp->crc_errcnt);
3054 counts[EFCT_HW_LINK_STAT_PRIMITIVE_SEQ_TIMEOUT_COUNT].counter =
3055 le32_to_cpu(mbox_rsp->primseq_eventtimeout_cnt);
3056 counts[EFCT_HW_LINK_STAT_ELASTIC_BUFFER_OVERRUN_COUNT].counter =
3057 le32_to_cpu(mbox_rsp->elastic_bufoverrun_errcnt);
3058 counts[EFCT_HW_LINK_STAT_ARB_TIMEOUT_COUNT].counter =
3059 le32_to_cpu(mbox_rsp->arbit_fc_al_timeout_cnt);
3060 counts[EFCT_HW_LINK_STAT_ADVERTISED_RCV_B2B_CREDIT].counter =
3061 le32_to_cpu(mbox_rsp->adv_rx_buftor_to_buf_credit);
3062 counts[EFCT_HW_LINK_STAT_CURR_RCV_B2B_CREDIT].counter =
3063 le32_to_cpu(mbox_rsp->curr_rx_buf_to_buf_credit);
3064 counts[EFCT_HW_LINK_STAT_ADVERTISED_XMIT_B2B_CREDIT].counter =
3065 le32_to_cpu(mbox_rsp->adv_tx_buf_to_buf_credit);
3066 counts[EFCT_HW_LINK_STAT_CURR_XMIT_B2B_CREDIT].counter =
3067 le32_to_cpu(mbox_rsp->curr_tx_buf_to_buf_credit);
3068 counts[EFCT_HW_LINK_STAT_RCV_EOFA_COUNT].counter =
3069 le32_to_cpu(mbox_rsp->rx_eofa_cnt);
3070 counts[EFCT_HW_LINK_STAT_RCV_EOFDTI_COUNT].counter =
3071 le32_to_cpu(mbox_rsp->rx_eofdti_cnt);
3072 counts[EFCT_HW_LINK_STAT_RCV_EOFNI_COUNT].counter =
3073 le32_to_cpu(mbox_rsp->rx_eofni_cnt);
3074 counts[EFCT_HW_LINK_STAT_RCV_SOFF_COUNT].counter =
3075 le32_to_cpu(mbox_rsp->rx_soff_cnt);
3076 counts[EFCT_HW_LINK_STAT_RCV_DROPPED_NO_AER_COUNT].counter =
3077 le32_to_cpu(mbox_rsp->rx_dropped_no_aer_cnt);
3078 counts[EFCT_HW_LINK_STAT_RCV_DROPPED_NO_RPI_COUNT].counter =
3079 le32_to_cpu(mbox_rsp->rx_dropped_no_avail_rpi_rescnt);
3080 counts[EFCT_HW_LINK_STAT_RCV_DROPPED_NO_XRI_COUNT].counter =
3081 le32_to_cpu(mbox_rsp->rx_dropped_no_avail_xri_rescnt);
3082
3083 if (cb_arg) {
3084 if (cb_arg->cb) {
3085 if (status == 0 && le16_to_cpu(mbox_rsp->hdr.status))
3086 status = le16_to_cpu(mbox_rsp->hdr.status);
3087 cb_arg->cb(status, num_counters, counts, cb_arg->arg);
3088 }
3089
3090 kfree(cb_arg);
3091 }
3092
3093 return 0;
3094 }
3095
3096 int
efct_hw_get_link_stats(struct efct_hw * hw,u8 req_ext_counters,u8 clear_overflow_flags,u8 clear_all_counters,void (* cb)(int status,u32 num_counters,struct efct_hw_link_stat_counts * counters,void * arg),void * arg)3097 efct_hw_get_link_stats(struct efct_hw *hw, u8 req_ext_counters,
3098 u8 clear_overflow_flags, u8 clear_all_counters,
3099 void (*cb)(int status, u32 num_counters,
3100 struct efct_hw_link_stat_counts *counters,
3101 void *arg),
3102 void *arg)
3103 {
3104 int rc = -EIO;
3105 struct efct_hw_link_stat_cb_arg *cb_arg;
3106 u8 mbxdata[SLI4_BMBX_SIZE];
3107
3108 cb_arg = kzalloc(sizeof(*cb_arg), GFP_ATOMIC);
3109 if (!cb_arg)
3110 return -ENOMEM;
3111
3112 cb_arg->cb = cb;
3113 cb_arg->arg = arg;
3114
3115 /* Send the HW command */
3116 if (!sli_cmd_read_link_stats(&hw->sli, mbxdata, req_ext_counters,
3117 clear_overflow_flags, clear_all_counters))
3118 rc = efct_hw_command(hw, mbxdata, EFCT_CMD_NOWAIT,
3119 efct_hw_cb_link_stat, cb_arg);
3120
3121 if (rc)
3122 kfree(cb_arg);
3123
3124 return rc;
3125 }
3126
3127 static int
efct_hw_cb_host_stat(struct efct_hw * hw,int status,u8 * mqe,void * arg)3128 efct_hw_cb_host_stat(struct efct_hw *hw, int status, u8 *mqe, void *arg)
3129 {
3130 struct sli4_cmd_read_status *mbox_rsp =
3131 (struct sli4_cmd_read_status *)mqe;
3132 struct efct_hw_host_stat_cb_arg *cb_arg = arg;
3133 struct efct_hw_host_stat_counts counts[EFCT_HW_HOST_STAT_MAX];
3134 u32 num_counters = EFCT_HW_HOST_STAT_MAX;
3135
3136 memset(counts, 0, sizeof(struct efct_hw_host_stat_counts) *
3137 EFCT_HW_HOST_STAT_MAX);
3138
3139 counts[EFCT_HW_HOST_STAT_TX_KBYTE_COUNT].counter =
3140 le32_to_cpu(mbox_rsp->trans_kbyte_cnt);
3141 counts[EFCT_HW_HOST_STAT_RX_KBYTE_COUNT].counter =
3142 le32_to_cpu(mbox_rsp->recv_kbyte_cnt);
3143 counts[EFCT_HW_HOST_STAT_TX_FRAME_COUNT].counter =
3144 le32_to_cpu(mbox_rsp->trans_frame_cnt);
3145 counts[EFCT_HW_HOST_STAT_RX_FRAME_COUNT].counter =
3146 le32_to_cpu(mbox_rsp->recv_frame_cnt);
3147 counts[EFCT_HW_HOST_STAT_TX_SEQ_COUNT].counter =
3148 le32_to_cpu(mbox_rsp->trans_seq_cnt);
3149 counts[EFCT_HW_HOST_STAT_RX_SEQ_COUNT].counter =
3150 le32_to_cpu(mbox_rsp->recv_seq_cnt);
3151 counts[EFCT_HW_HOST_STAT_TOTAL_EXCH_ORIG].counter =
3152 le32_to_cpu(mbox_rsp->tot_exchanges_orig);
3153 counts[EFCT_HW_HOST_STAT_TOTAL_EXCH_RESP].counter =
3154 le32_to_cpu(mbox_rsp->tot_exchanges_resp);
3155 counts[EFCT_HW_HOSY_STAT_RX_P_BSY_COUNT].counter =
3156 le32_to_cpu(mbox_rsp->recv_p_bsy_cnt);
3157 counts[EFCT_HW_HOST_STAT_RX_F_BSY_COUNT].counter =
3158 le32_to_cpu(mbox_rsp->recv_f_bsy_cnt);
3159 counts[EFCT_HW_HOST_STAT_DROP_FRM_DUE_TO_NO_RQ_BUF_COUNT].counter =
3160 le32_to_cpu(mbox_rsp->no_rq_buf_dropped_frames_cnt);
3161 counts[EFCT_HW_HOST_STAT_EMPTY_RQ_TIMEOUT_COUNT].counter =
3162 le32_to_cpu(mbox_rsp->empty_rq_timeout_cnt);
3163 counts[EFCT_HW_HOST_STAT_DROP_FRM_DUE_TO_NO_XRI_COUNT].counter =
3164 le32_to_cpu(mbox_rsp->no_xri_dropped_frames_cnt);
3165 counts[EFCT_HW_HOST_STAT_EMPTY_XRI_POOL_COUNT].counter =
3166 le32_to_cpu(mbox_rsp->empty_xri_pool_cnt);
3167
3168 if (cb_arg) {
3169 if (cb_arg->cb) {
3170 if (status == 0 && le16_to_cpu(mbox_rsp->hdr.status))
3171 status = le16_to_cpu(mbox_rsp->hdr.status);
3172 cb_arg->cb(status, num_counters, counts, cb_arg->arg);
3173 }
3174
3175 kfree(cb_arg);
3176 }
3177
3178 return 0;
3179 }
3180
3181 int
efct_hw_get_host_stats(struct efct_hw * hw,u8 cc,void (* cb)(int status,u32 num_counters,struct efct_hw_host_stat_counts * counters,void * arg),void * arg)3182 efct_hw_get_host_stats(struct efct_hw *hw, u8 cc,
3183 void (*cb)(int status, u32 num_counters,
3184 struct efct_hw_host_stat_counts *counters,
3185 void *arg),
3186 void *arg)
3187 {
3188 int rc = -EIO;
3189 struct efct_hw_host_stat_cb_arg *cb_arg;
3190 u8 mbxdata[SLI4_BMBX_SIZE];
3191
3192 cb_arg = kmalloc(sizeof(*cb_arg), GFP_ATOMIC);
3193 if (!cb_arg)
3194 return -ENOMEM;
3195
3196 cb_arg->cb = cb;
3197 cb_arg->arg = arg;
3198
3199 /* Send the HW command to get the host stats */
3200 if (!sli_cmd_read_status(&hw->sli, mbxdata, cc))
3201 rc = efct_hw_command(hw, mbxdata, EFCT_CMD_NOWAIT,
3202 efct_hw_cb_host_stat, cb_arg);
3203
3204 if (rc) {
3205 efc_log_debug(hw->os, "READ_HOST_STATS failed\n");
3206 kfree(cb_arg);
3207 }
3208
3209 return rc;
3210 }
3211
3212 struct efct_hw_async_call_ctx {
3213 efct_hw_async_cb_t callback;
3214 void *arg;
3215 u8 cmd[SLI4_BMBX_SIZE];
3216 };
3217
3218 static void
efct_hw_async_cb(struct efct_hw * hw,int status,u8 * mqe,void * arg)3219 efct_hw_async_cb(struct efct_hw *hw, int status, u8 *mqe, void *arg)
3220 {
3221 struct efct_hw_async_call_ctx *ctx = arg;
3222
3223 if (ctx) {
3224 if (ctx->callback)
3225 (*ctx->callback)(hw, status, mqe, ctx->arg);
3226
3227 kfree(ctx);
3228 }
3229 }
3230
3231 int
efct_hw_async_call(struct efct_hw * hw,efct_hw_async_cb_t callback,void * arg)3232 efct_hw_async_call(struct efct_hw *hw, efct_hw_async_cb_t callback, void *arg)
3233 {
3234 struct efct_hw_async_call_ctx *ctx;
3235 int rc;
3236
3237 /*
3238 * Allocate a callback context (which includes the mbox cmd buffer),
3239 * we need this to be persistent as the mbox cmd submission may be
3240 * queued and executed later execution.
3241 */
3242 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
3243 if (!ctx)
3244 return -ENOMEM;
3245
3246 ctx->callback = callback;
3247 ctx->arg = arg;
3248
3249 /* Build and send a NOP mailbox command */
3250 if (sli_cmd_common_nop(&hw->sli, ctx->cmd, 0)) {
3251 efc_log_err(hw->os, "COMMON_NOP format failure\n");
3252 kfree(ctx);
3253 return -EIO;
3254 }
3255
3256 rc = efct_hw_command(hw, ctx->cmd, EFCT_CMD_NOWAIT, efct_hw_async_cb,
3257 ctx);
3258 if (rc) {
3259 efc_log_err(hw->os, "COMMON_NOP command failure, rc=%d\n", rc);
3260 kfree(ctx);
3261 return -EIO;
3262 }
3263 return 0;
3264 }
3265
3266 static int
efct_hw_cb_fw_write(struct efct_hw * hw,int status,u8 * mqe,void * arg)3267 efct_hw_cb_fw_write(struct efct_hw *hw, int status, u8 *mqe, void *arg)
3268 {
3269 struct sli4_cmd_sli_config *mbox_rsp =
3270 (struct sli4_cmd_sli_config *)mqe;
3271 struct sli4_rsp_cmn_write_object *wr_obj_rsp;
3272 struct efct_hw_fw_wr_cb_arg *cb_arg = arg;
3273 u32 bytes_written;
3274 u16 mbox_status;
3275 u32 change_status;
3276
3277 wr_obj_rsp = (struct sli4_rsp_cmn_write_object *)
3278 &mbox_rsp->payload.embed;
3279 bytes_written = le32_to_cpu(wr_obj_rsp->actual_write_length);
3280 mbox_status = le16_to_cpu(mbox_rsp->hdr.status);
3281 change_status = (le32_to_cpu(wr_obj_rsp->change_status_dword) &
3282 RSP_CHANGE_STATUS);
3283
3284 if (cb_arg) {
3285 if (cb_arg->cb) {
3286 if (!status && mbox_status)
3287 status = mbox_status;
3288 cb_arg->cb(status, bytes_written, change_status,
3289 cb_arg->arg);
3290 }
3291
3292 kfree(cb_arg);
3293 }
3294
3295 return 0;
3296 }
3297
3298 int
efct_hw_firmware_write(struct efct_hw * hw,struct efc_dma * dma,u32 size,u32 offset,int last,void (* cb)(int status,u32 bytes_written,u32 change_status,void * arg),void * arg)3299 efct_hw_firmware_write(struct efct_hw *hw, struct efc_dma *dma, u32 size,
3300 u32 offset, int last,
3301 void (*cb)(int status, u32 bytes_written,
3302 u32 change_status, void *arg),
3303 void *arg)
3304 {
3305 int rc = -EIO;
3306 u8 mbxdata[SLI4_BMBX_SIZE];
3307 struct efct_hw_fw_wr_cb_arg *cb_arg;
3308 int noc = 0;
3309
3310 cb_arg = kzalloc(sizeof(*cb_arg), GFP_KERNEL);
3311 if (!cb_arg)
3312 return -ENOMEM;
3313
3314 cb_arg->cb = cb;
3315 cb_arg->arg = arg;
3316
3317 /* Write a portion of a firmware image to the device */
3318 if (!sli_cmd_common_write_object(&hw->sli, mbxdata,
3319 noc, last, size, offset, "/prg/",
3320 dma))
3321 rc = efct_hw_command(hw, mbxdata, EFCT_CMD_NOWAIT,
3322 efct_hw_cb_fw_write, cb_arg);
3323
3324 if (rc != 0) {
3325 efc_log_debug(hw->os, "COMMON_WRITE_OBJECT failed\n");
3326 kfree(cb_arg);
3327 }
3328
3329 return rc;
3330 }
3331
3332 static int
efct_hw_cb_port_control(struct efct_hw * hw,int status,u8 * mqe,void * arg)3333 efct_hw_cb_port_control(struct efct_hw *hw, int status, u8 *mqe,
3334 void *arg)
3335 {
3336 return 0;
3337 }
3338
3339 int
efct_hw_port_control(struct efct_hw * hw,enum efct_hw_port ctrl,uintptr_t value,void (* cb)(int status,uintptr_t value,void * arg),void * arg)3340 efct_hw_port_control(struct efct_hw *hw, enum efct_hw_port ctrl,
3341 uintptr_t value,
3342 void (*cb)(int status, uintptr_t value, void *arg),
3343 void *arg)
3344 {
3345 int rc = -EIO;
3346 u8 link[SLI4_BMBX_SIZE];
3347 u32 speed = 0;
3348 u8 reset_alpa = 0;
3349
3350 switch (ctrl) {
3351 case EFCT_HW_PORT_INIT:
3352 if (!sli_cmd_config_link(&hw->sli, link))
3353 rc = efct_hw_command(hw, link, EFCT_CMD_NOWAIT,
3354 efct_hw_cb_port_control, NULL);
3355
3356 if (rc != 0) {
3357 efc_log_err(hw->os, "CONFIG_LINK failed\n");
3358 break;
3359 }
3360 speed = hw->config.speed;
3361 reset_alpa = (u8)(value & 0xff);
3362
3363 rc = -EIO;
3364 if (!sli_cmd_init_link(&hw->sli, link, speed, reset_alpa))
3365 rc = efct_hw_command(hw, link, EFCT_CMD_NOWAIT,
3366 efct_hw_cb_port_control, NULL);
3367 /* Free buffer on error, since no callback is coming */
3368 if (rc)
3369 efc_log_err(hw->os, "INIT_LINK failed\n");
3370 break;
3371
3372 case EFCT_HW_PORT_SHUTDOWN:
3373 if (!sli_cmd_down_link(&hw->sli, link))
3374 rc = efct_hw_command(hw, link, EFCT_CMD_NOWAIT,
3375 efct_hw_cb_port_control, NULL);
3376 /* Free buffer on error, since no callback is coming */
3377 if (rc)
3378 efc_log_err(hw->os, "DOWN_LINK failed\n");
3379 break;
3380
3381 default:
3382 efc_log_debug(hw->os, "unhandled control %#x\n", ctrl);
3383 break;
3384 }
3385
3386 return rc;
3387 }
3388
3389 void
efct_hw_teardown(struct efct_hw * hw)3390 efct_hw_teardown(struct efct_hw *hw)
3391 {
3392 u32 i = 0;
3393 u32 destroy_queues;
3394 u32 free_memory;
3395 struct efc_dma *dma;
3396 struct efct *efct = hw->os;
3397
3398 destroy_queues = (hw->state == EFCT_HW_STATE_ACTIVE);
3399 free_memory = (hw->state != EFCT_HW_STATE_UNINITIALIZED);
3400
3401 /* Cancel Sliport Healthcheck */
3402 if (hw->sliport_healthcheck) {
3403 hw->sliport_healthcheck = 0;
3404 efct_hw_config_sli_port_health_check(hw, 0, 0);
3405 }
3406
3407 if (hw->state != EFCT_HW_STATE_QUEUES_ALLOCATED) {
3408 hw->state = EFCT_HW_STATE_TEARDOWN_IN_PROGRESS;
3409
3410 efct_hw_flush(hw);
3411
3412 if (list_empty(&hw->cmd_head))
3413 efc_log_debug(hw->os,
3414 "All commands completed on MQ queue\n");
3415 else
3416 efc_log_debug(hw->os,
3417 "Some cmds still pending on MQ queue\n");
3418
3419 /* Cancel any remaining commands */
3420 efct_hw_command_cancel(hw);
3421 } else {
3422 hw->state = EFCT_HW_STATE_TEARDOWN_IN_PROGRESS;
3423 }
3424
3425 dma_free_coherent(&efct->pci->dev,
3426 hw->rnode_mem.size, hw->rnode_mem.virt,
3427 hw->rnode_mem.phys);
3428 memset(&hw->rnode_mem, 0, sizeof(struct efc_dma));
3429
3430 if (hw->io) {
3431 for (i = 0; i < hw->config.n_io; i++) {
3432 if (hw->io[i] && hw->io[i]->sgl &&
3433 hw->io[i]->sgl->virt) {
3434 dma_free_coherent(&efct->pci->dev,
3435 hw->io[i]->sgl->size,
3436 hw->io[i]->sgl->virt,
3437 hw->io[i]->sgl->phys);
3438 }
3439 kfree(hw->io[i]);
3440 hw->io[i] = NULL;
3441 }
3442 kfree(hw->io);
3443 hw->io = NULL;
3444 kfree(hw->wqe_buffs);
3445 hw->wqe_buffs = NULL;
3446 }
3447
3448 dma = &hw->xfer_rdy;
3449 dma_free_coherent(&efct->pci->dev,
3450 dma->size, dma->virt, dma->phys);
3451 memset(dma, 0, sizeof(struct efc_dma));
3452
3453 dma = &hw->loop_map;
3454 dma_free_coherent(&efct->pci->dev,
3455 dma->size, dma->virt, dma->phys);
3456 memset(dma, 0, sizeof(struct efc_dma));
3457
3458 for (i = 0; i < hw->wq_count; i++)
3459 sli_queue_free(&hw->sli, &hw->wq[i], destroy_queues,
3460 free_memory);
3461
3462 for (i = 0; i < hw->rq_count; i++)
3463 sli_queue_free(&hw->sli, &hw->rq[i], destroy_queues,
3464 free_memory);
3465
3466 for (i = 0; i < hw->mq_count; i++)
3467 sli_queue_free(&hw->sli, &hw->mq[i], destroy_queues,
3468 free_memory);
3469
3470 for (i = 0; i < hw->cq_count; i++)
3471 sli_queue_free(&hw->sli, &hw->cq[i], destroy_queues,
3472 free_memory);
3473
3474 for (i = 0; i < hw->eq_count; i++)
3475 sli_queue_free(&hw->sli, &hw->eq[i], destroy_queues,
3476 free_memory);
3477
3478 /* Free rq buffers */
3479 efct_hw_rx_free(hw);
3480
3481 efct_hw_queue_teardown(hw);
3482
3483 kfree(hw->wq_cpu_array);
3484
3485 sli_teardown(&hw->sli);
3486
3487 /* record the fact that the queues are non-functional */
3488 hw->state = EFCT_HW_STATE_UNINITIALIZED;
3489
3490 /* free sequence free pool */
3491 kfree(hw->seq_pool);
3492 hw->seq_pool = NULL;
3493
3494 /* free hw_wq_callback pool */
3495 efct_hw_reqtag_pool_free(hw);
3496
3497 mempool_destroy(hw->cmd_ctx_pool);
3498 mempool_destroy(hw->mbox_rqst_pool);
3499
3500 /* Mark HW setup as not having been called */
3501 hw->hw_setup_called = false;
3502 }
3503
3504 static int
efct_hw_sli_reset(struct efct_hw * hw,enum efct_hw_reset reset,enum efct_hw_state prev_state)3505 efct_hw_sli_reset(struct efct_hw *hw, enum efct_hw_reset reset,
3506 enum efct_hw_state prev_state)
3507 {
3508 int rc = 0;
3509
3510 switch (reset) {
3511 case EFCT_HW_RESET_FUNCTION:
3512 efc_log_debug(hw->os, "issuing function level reset\n");
3513 if (sli_reset(&hw->sli)) {
3514 efc_log_err(hw->os, "sli_reset failed\n");
3515 rc = -EIO;
3516 }
3517 break;
3518 case EFCT_HW_RESET_FIRMWARE:
3519 efc_log_debug(hw->os, "issuing firmware reset\n");
3520 if (sli_fw_reset(&hw->sli)) {
3521 efc_log_err(hw->os, "sli_soft_reset failed\n");
3522 rc = -EIO;
3523 }
3524 /*
3525 * Because the FW reset leaves the FW in a non-running state,
3526 * follow that with a regular reset.
3527 */
3528 efc_log_debug(hw->os, "issuing function level reset\n");
3529 if (sli_reset(&hw->sli)) {
3530 efc_log_err(hw->os, "sli_reset failed\n");
3531 rc = -EIO;
3532 }
3533 break;
3534 default:
3535 efc_log_err(hw->os, "unknown type - no reset performed\n");
3536 hw->state = prev_state;
3537 rc = -EINVAL;
3538 break;
3539 }
3540
3541 return rc;
3542 }
3543
3544 int
efct_hw_reset(struct efct_hw * hw,enum efct_hw_reset reset)3545 efct_hw_reset(struct efct_hw *hw, enum efct_hw_reset reset)
3546 {
3547 int rc = 0;
3548 enum efct_hw_state prev_state = hw->state;
3549
3550 if (hw->state != EFCT_HW_STATE_ACTIVE)
3551 efc_log_debug(hw->os,
3552 "HW state %d is not active\n", hw->state);
3553
3554 hw->state = EFCT_HW_STATE_RESET_IN_PROGRESS;
3555
3556 /*
3557 * If the prev_state is already reset/teardown in progress,
3558 * don't continue further
3559 */
3560 if (prev_state == EFCT_HW_STATE_RESET_IN_PROGRESS ||
3561 prev_state == EFCT_HW_STATE_TEARDOWN_IN_PROGRESS)
3562 return efct_hw_sli_reset(hw, reset, prev_state);
3563
3564 if (prev_state != EFCT_HW_STATE_UNINITIALIZED) {
3565 efct_hw_flush(hw);
3566
3567 if (list_empty(&hw->cmd_head))
3568 efc_log_debug(hw->os,
3569 "All commands completed on MQ queue\n");
3570 else
3571 efc_log_err(hw->os,
3572 "Some commands still pending on MQ queue\n");
3573 }
3574
3575 /* Reset the chip */
3576 rc = efct_hw_sli_reset(hw, reset, prev_state);
3577 if (rc == -EINVAL)
3578 return -EIO;
3579
3580 return rc;
3581 }
3582