1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IBM Accelerator Family 'GenWQE'
4 *
5 * (C) Copyright IBM Corp. 2013
6 *
7 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
9 * Author: Michael Jung <mijung@gmx.net>
10 * Author: Michael Ruettger <michael@ibmra.de>
11 */
12
13 /*
14 * Device Driver Control Block (DDCB) queue support. Definition of
15 * interrupt handlers for queue support as well as triggering the
16 * health monitor code in case of problems. The current hardware uses
17 * an MSI interrupt which is shared between error handling and
18 * functional code.
19 */
20
21 #include <linux/types.h>
22 #include <linux/sched.h>
23 #include <linux/wait.h>
24 #include <linux/pci.h>
25 #include <linux/string.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/delay.h>
28 #include <linux/module.h>
29 #include <linux/interrupt.h>
30 #include <linux/crc-itu-t.h>
31
32 #include "card_base.h"
33 #include "card_ddcb.h"
34
35 /*
36 * N: next DDCB, this is where the next DDCB will be put.
37 * A: active DDCB, this is where the code will look for the next completion.
38 * x: DDCB is enqueued, we are waiting for its completion.
39
40 * Situation (1): Empty queue
41 * +---+---+---+---+---+---+---+---+
42 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
43 * | | | | | | | | |
44 * +---+---+---+---+---+---+---+---+
45 * A/N
46 * enqueued_ddcbs = A - N = 2 - 2 = 0
47 *
48 * Situation (2): Wrapped, N > A
49 * +---+---+---+---+---+---+---+---+
50 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
51 * | | | x | x | | | | |
52 * +---+---+---+---+---+---+---+---+
53 * A N
54 * enqueued_ddcbs = N - A = 4 - 2 = 2
55 *
56 * Situation (3): Queue wrapped, A > N
57 * +---+---+---+---+---+---+---+---+
58 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
59 * | x | x | | | x | x | x | x |
60 * +---+---+---+---+---+---+---+---+
61 * N A
62 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 6
63 *
64 * Situation (4a): Queue full N > A
65 * +---+---+---+---+---+---+---+---+
66 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
67 * | x | x | x | x | x | x | x | |
68 * +---+---+---+---+---+---+---+---+
69 * A N
70 *
71 * enqueued_ddcbs = N - A = 7 - 0 = 7
72 *
73 * Situation (4a): Queue full A > N
74 * +---+---+---+---+---+---+---+---+
75 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
76 * | x | x | x | | x | x | x | x |
77 * +---+---+---+---+---+---+---+---+
78 * N A
79 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
80 */
81
queue_empty(struct ddcb_queue * queue)82 static int queue_empty(struct ddcb_queue *queue)
83 {
84 return queue->ddcb_next == queue->ddcb_act;
85 }
86
queue_enqueued_ddcbs(struct ddcb_queue * queue)87 static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
88 {
89 if (queue->ddcb_next >= queue->ddcb_act)
90 return queue->ddcb_next - queue->ddcb_act;
91
92 return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
93 }
94
queue_free_ddcbs(struct ddcb_queue * queue)95 static int queue_free_ddcbs(struct ddcb_queue *queue)
96 {
97 int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
98
99 if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */
100 return 0;
101 }
102 return free_ddcbs;
103 }
104
105 /*
106 * Use of the PRIV field in the DDCB for queue debugging:
107 *
108 * (1) Trying to get rid of a DDCB which saw a timeout:
109 * pddcb->priv[6] = 0xcc; # cleared
110 *
111 * (2) Append a DDCB via NEXT bit:
112 * pddcb->priv[7] = 0xaa; # appended
113 *
114 * (3) DDCB needed tapping:
115 * pddcb->priv[7] = 0xbb; # tapped
116 *
117 * (4) DDCB marked as correctly finished:
118 * pddcb->priv[6] = 0xff; # finished
119 */
120
ddcb_mark_tapped(struct ddcb * pddcb)121 static inline void ddcb_mark_tapped(struct ddcb *pddcb)
122 {
123 pddcb->priv[7] = 0xbb; /* tapped */
124 }
125
ddcb_mark_appended(struct ddcb * pddcb)126 static inline void ddcb_mark_appended(struct ddcb *pddcb)
127 {
128 pddcb->priv[7] = 0xaa; /* appended */
129 }
130
ddcb_mark_cleared(struct ddcb * pddcb)131 static inline void ddcb_mark_cleared(struct ddcb *pddcb)
132 {
133 pddcb->priv[6] = 0xcc; /* cleared */
134 }
135
ddcb_mark_finished(struct ddcb * pddcb)136 static inline void ddcb_mark_finished(struct ddcb *pddcb)
137 {
138 pddcb->priv[6] = 0xff; /* finished */
139 }
140
ddcb_mark_unused(struct ddcb * pddcb)141 static inline void ddcb_mark_unused(struct ddcb *pddcb)
142 {
143 pddcb->priv_64 = cpu_to_be64(0); /* not tapped */
144 }
145
146 /**
147 * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
148 * @buff: pointer to data buffer
149 * @len: length of data for calculation
150 * @init: initial crc (0xffff at start)
151 *
152 * Polynomial = x^16 + x^12 + x^5 + 1 (0x1021)
153 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff
154 * should result in a crc16 of 0x89c3
155 *
156 * Return: crc16 checksum in big endian format !
157 */
genwqe_crc16(const u8 * buff,size_t len,u16 init)158 static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
159 {
160 return crc_itu_t(init, buff, len);
161 }
162
print_ddcb_info(struct genwqe_dev * cd,struct ddcb_queue * queue)163 static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
164 {
165 int i;
166 struct ddcb *pddcb;
167 unsigned long flags;
168 struct pci_dev *pci_dev = cd->pci_dev;
169
170 spin_lock_irqsave(&cd->print_lock, flags);
171
172 dev_info(&pci_dev->dev,
173 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
174 cd->card_idx, queue->ddcb_act, queue->ddcb_next);
175
176 pddcb = queue->ddcb_vaddr;
177 for (i = 0; i < queue->ddcb_max; i++) {
178 dev_err(&pci_dev->dev,
179 " %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
180 i == queue->ddcb_act ? '>' : ' ',
181 i,
182 be16_to_cpu(pddcb->retc_16),
183 be16_to_cpu(pddcb->seqnum_16),
184 pddcb->hsi,
185 pddcb->shi,
186 be64_to_cpu(pddcb->priv_64),
187 pddcb->cmd);
188 pddcb++;
189 }
190 spin_unlock_irqrestore(&cd->print_lock, flags);
191 }
192
ddcb_requ_alloc(void)193 struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
194 {
195 struct ddcb_requ *req;
196
197 req = kzalloc(sizeof(*req), GFP_KERNEL);
198 if (!req)
199 return NULL;
200
201 return &req->cmd;
202 }
203
ddcb_requ_free(struct genwqe_ddcb_cmd * cmd)204 void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
205 {
206 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
207
208 kfree(req);
209 }
210
ddcb_requ_get_state(struct ddcb_requ * req)211 static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
212 {
213 return req->req_state;
214 }
215
ddcb_requ_set_state(struct ddcb_requ * req,enum genwqe_requ_state new_state)216 static inline void ddcb_requ_set_state(struct ddcb_requ *req,
217 enum genwqe_requ_state new_state)
218 {
219 req->req_state = new_state;
220 }
221
ddcb_requ_collect_debug_data(struct ddcb_requ * req)222 static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
223 {
224 return req->cmd.ddata_addr != 0x0;
225 }
226
227 /**
228 * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
229 * @cd: pointer to genwqe device descriptor
230 * @req: DDCB work request
231 *
232 * Status of ddcb_requ mirrors this hardware state, but is copied in
233 * the ddcb_requ on interrupt/polling function. The lowlevel code
234 * should check the hardware state directly, the higher level code
235 * should check the copy.
236 *
237 * This function will also return true if the state of the queue is
238 * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the
239 * shutdown case.
240 */
ddcb_requ_finished(struct genwqe_dev * cd,struct ddcb_requ * req)241 static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
242 {
243 return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
244 (cd->card_state != GENWQE_CARD_USED);
245 }
246
247 #define RET_DDCB_APPENDED 1
248 #define RET_DDCB_TAPPED 2
249 /**
250 * enqueue_ddcb() - Enqueue a DDCB
251 * @cd: pointer to genwqe device descriptor
252 * @queue: queue this operation should be done on
253 * @pddcb: pointer to ddcb structure
254 * @ddcb_no: pointer to ddcb number being tapped
255 *
256 * Start execution of DDCB by tapping or append to queue via NEXT
257 * bit. This is done by an atomic 'compare and swap' instruction and
258 * checking SHI and HSI of the previous DDCB.
259 *
260 * This function must only be called with ddcb_lock held.
261 *
262 * Return: 1 if new DDCB is appended to previous
263 * 2 if DDCB queue is tapped via register/simulation
264 */
enqueue_ddcb(struct genwqe_dev * cd,struct ddcb_queue * queue,struct ddcb * pddcb,int ddcb_no)265 static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
266 struct ddcb *pddcb, int ddcb_no)
267 {
268 unsigned int try;
269 int prev_no;
270 struct ddcb *prev_ddcb;
271 __be32 old, new, icrc_hsi_shi;
272 u64 num;
273
274 /*
275 * For performance checks a Dispatch Timestamp can be put into
276 * DDCB It is supposed to use the SLU's free running counter,
277 * but this requires PCIe cycles.
278 */
279 ddcb_mark_unused(pddcb);
280
281 /* check previous DDCB if already fetched */
282 prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
283 prev_ddcb = &queue->ddcb_vaddr[prev_no];
284
285 /*
286 * It might have happened that the HSI.FETCHED bit is
287 * set. Retry in this case. Therefore I expect maximum 2 times
288 * trying.
289 */
290 ddcb_mark_appended(pddcb);
291 for (try = 0; try < 2; try++) {
292 old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
293
294 /* try to append via NEXT bit if prev DDCB is not completed */
295 if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
296 break;
297
298 new = (old | DDCB_NEXT_BE32);
299
300 wmb(); /* need to ensure write ordering */
301 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
302
303 if (icrc_hsi_shi == old)
304 return RET_DDCB_APPENDED; /* appended to queue */
305 }
306
307 /* Queue must be re-started by updating QUEUE_OFFSET */
308 ddcb_mark_tapped(pddcb);
309 num = (u64)ddcb_no << 8;
310
311 wmb(); /* need to ensure write ordering */
312 __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */
313
314 return RET_DDCB_TAPPED;
315 }
316
317 /**
318 * copy_ddcb_results() - Copy output state from real DDCB to request
319 * @req: pointer to requested DDCB parameters
320 * @ddcb_no: pointer to ddcb number being tapped
321 *
322 * Copy DDCB ASV to request struct. There is no endian
323 * conversion made, since data structure in ASV is still
324 * unknown here.
325 *
326 * This is needed by:
327 * - genwqe_purge_ddcb()
328 * - genwqe_check_ddcb_queue()
329 */
copy_ddcb_results(struct ddcb_requ * req,int ddcb_no)330 static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
331 {
332 struct ddcb_queue *queue = req->queue;
333 struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
334
335 memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
336
337 /* copy status flags of the variant part */
338 req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16);
339 req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
340 req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
341
342 req->cmd.attn = be16_to_cpu(pddcb->attn_16);
343 req->cmd.progress = be32_to_cpu(pddcb->progress_32);
344 req->cmd.retc = be16_to_cpu(pddcb->retc_16);
345
346 if (ddcb_requ_collect_debug_data(req)) {
347 int prev_no = (ddcb_no == 0) ?
348 queue->ddcb_max - 1 : ddcb_no - 1;
349 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
350
351 memcpy(&req->debug_data.ddcb_finished, pddcb,
352 sizeof(req->debug_data.ddcb_finished));
353 memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
354 sizeof(req->debug_data.ddcb_prev));
355 }
356 }
357
358 /**
359 * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work requests.
360 * @cd: pointer to genwqe device descriptor
361 * @queue: queue to be checked
362 *
363 * Return: Number of DDCBs which were finished
364 */
genwqe_check_ddcb_queue(struct genwqe_dev * cd,struct ddcb_queue * queue)365 static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
366 struct ddcb_queue *queue)
367 {
368 unsigned long flags;
369 int ddcbs_finished = 0;
370 struct pci_dev *pci_dev = cd->pci_dev;
371
372 spin_lock_irqsave(&queue->ddcb_lock, flags);
373
374 /* FIXME avoid soft locking CPU */
375 while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
376
377 struct ddcb *pddcb;
378 struct ddcb_requ *req;
379 u16 vcrc, vcrc_16, retc_16;
380
381 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
382
383 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
384 0x00000000)
385 goto go_home; /* not completed, continue waiting */
386
387 wmb(); /* Add sync to decouple prev. read operations */
388
389 /* Note: DDCB could be purged */
390 req = queue->ddcb_req[queue->ddcb_act];
391 if (req == NULL) {
392 /* this occurs if DDCB is purged, not an error */
393 /* Move active DDCB further; Nothing to do anymore. */
394 goto pick_next_one;
395 }
396
397 /*
398 * HSI=0x44 (fetched and completed), but RETC is
399 * 0x101, or even worse 0x000.
400 *
401 * In case of seeing the queue in inconsistent state
402 * we read the errcnts and the queue status to provide
403 * a trigger for our PCIe analyzer stop capturing.
404 */
405 retc_16 = be16_to_cpu(pddcb->retc_16);
406 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
407 u64 errcnts, status;
408 u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
409
410 errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
411 status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
412
413 dev_err(&pci_dev->dev,
414 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
415 __func__, be16_to_cpu(pddcb->seqnum_16),
416 pddcb->hsi, retc_16, errcnts, status,
417 queue->ddcb_daddr + ddcb_offs);
418 }
419
420 copy_ddcb_results(req, queue->ddcb_act);
421 queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */
422
423 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
424 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
425
426 ddcb_mark_finished(pddcb);
427
428 /* calculate CRC_16 to see if VCRC is correct */
429 vcrc = genwqe_crc16(pddcb->asv,
430 VCRC_LENGTH(req->cmd.asv_length),
431 0xffff);
432 vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
433 if (vcrc != vcrc_16) {
434 printk_ratelimited(KERN_ERR
435 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
436 GENWQE_DEVNAME, dev_name(&pci_dev->dev),
437 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
438 vcrc, vcrc_16);
439 }
440
441 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
442 queue->ddcbs_completed++;
443 queue->ddcbs_in_flight--;
444
445 /* wake up process waiting for this DDCB, and
446 processes on the busy queue */
447 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
448 wake_up_interruptible(&queue->busy_waitq);
449
450 pick_next_one:
451 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
452 ddcbs_finished++;
453 }
454
455 go_home:
456 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
457 return ddcbs_finished;
458 }
459
460 /**
461 * __genwqe_wait_ddcb(): Waits until DDCB is completed
462 * @cd: pointer to genwqe device descriptor
463 * @req: pointer to requsted DDCB parameters
464 *
465 * The Service Layer will update the RETC in DDCB when processing is
466 * pending or done.
467 *
468 * Return: > 0 remaining jiffies, DDCB completed
469 * -ETIMEDOUT when timeout
470 * -ERESTARTSYS when ^C
471 * -EINVAL when unknown error condition
472 *
473 * When an error is returned the called needs to ensure that
474 * purge_ddcb() is being called to get the &req removed from the
475 * queue.
476 */
__genwqe_wait_ddcb(struct genwqe_dev * cd,struct ddcb_requ * req)477 int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
478 {
479 int rc;
480 unsigned int ddcb_no;
481 struct ddcb_queue *queue;
482 struct pci_dev *pci_dev = cd->pci_dev;
483
484 if (req == NULL)
485 return -EINVAL;
486
487 queue = req->queue;
488 if (queue == NULL)
489 return -EINVAL;
490
491 ddcb_no = req->num;
492 if (ddcb_no >= queue->ddcb_max)
493 return -EINVAL;
494
495 rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
496 ddcb_requ_finished(cd, req),
497 GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);
498
499 /*
500 * We need to distinguish 3 cases here:
501 * 1. rc == 0 timeout occurred
502 * 2. rc == -ERESTARTSYS signal received
503 * 3. rc > 0 remaining jiffies condition is true
504 */
505 if (rc == 0) {
506 struct ddcb_queue *queue = req->queue;
507 struct ddcb *pddcb;
508
509 /*
510 * Timeout may be caused by long task switching time.
511 * When timeout happens, check if the request has
512 * meanwhile completed.
513 */
514 genwqe_check_ddcb_queue(cd, req->queue);
515 if (ddcb_requ_finished(cd, req))
516 return rc;
517
518 dev_err(&pci_dev->dev,
519 "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
520 __func__, req->num, rc, ddcb_requ_get_state(req),
521 req);
522 dev_err(&pci_dev->dev,
523 "[%s] IO_QUEUE_STATUS=0x%016llx\n", __func__,
524 __genwqe_readq(cd, queue->IO_QUEUE_STATUS));
525
526 pddcb = &queue->ddcb_vaddr[req->num];
527 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
528
529 print_ddcb_info(cd, req->queue);
530 return -ETIMEDOUT;
531
532 } else if (rc == -ERESTARTSYS) {
533 return rc;
534 /*
535 * EINTR: Stops the application
536 * ERESTARTSYS: Restartable systemcall; called again
537 */
538
539 } else if (rc < 0) {
540 dev_err(&pci_dev->dev,
541 "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
542 __func__, req->num, rc, ddcb_requ_get_state(req));
543 return -EINVAL;
544 }
545
546 /* Severe error occured. Driver is forced to stop operation */
547 if (cd->card_state != GENWQE_CARD_USED) {
548 dev_err(&pci_dev->dev,
549 "[%s] err: DDCB#%d forced to stop (rc=%d)\n",
550 __func__, req->num, rc);
551 return -EIO;
552 }
553 return rc;
554 }
555
556 /**
557 * get_next_ddcb() - Get next available DDCB
558 * @cd: pointer to genwqe device descriptor
559 * @queue: DDCB queue
560 * @num: internal DDCB number
561 *
562 * DDCB's content is completely cleared but presets for PRE and
563 * SEQNUM. This function must only be called when ddcb_lock is held.
564 *
565 * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.
566 */
get_next_ddcb(struct genwqe_dev * cd,struct ddcb_queue * queue,int * num)567 static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
568 struct ddcb_queue *queue,
569 int *num)
570 {
571 u64 *pu64;
572 struct ddcb *pddcb;
573
574 if (queue_free_ddcbs(queue) == 0) /* queue is full */
575 return NULL;
576
577 /* find new ddcb */
578 pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
579
580 /* if it is not completed, we are not allowed to use it */
581 /* barrier(); */
582 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
583 return NULL;
584
585 *num = queue->ddcb_next; /* internal DDCB number */
586 queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
587
588 /* clear important DDCB fields */
589 pu64 = (u64 *)pddcb;
590 pu64[0] = 0ULL; /* offs 0x00 (ICRC,HSI,SHI,...) */
591 pu64[1] = 0ULL; /* offs 0x01 (ACFUNC,CMD...) */
592
593 /* destroy previous results in ASV */
594 pu64[0x80/8] = 0ULL; /* offs 0x80 (ASV + 0) */
595 pu64[0x88/8] = 0ULL; /* offs 0x88 (ASV + 0x08) */
596 pu64[0x90/8] = 0ULL; /* offs 0x90 (ASV + 0x10) */
597 pu64[0x98/8] = 0ULL; /* offs 0x98 (ASV + 0x18) */
598 pu64[0xd0/8] = 0ULL; /* offs 0xd0 (RETC,ATTN...) */
599
600 pddcb->pre = DDCB_PRESET_PRE; /* 128 */
601 pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
602 return pddcb;
603 }
604
605 /**
606 * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
607 * @cd: genwqe device descriptor
608 * @req: DDCB request
609 *
610 * This will fail when the request was already FETCHED. In this case
611 * we need to wait until it is finished. Else the DDCB can be
612 * reused. This function also ensures that the request data structure
613 * is removed from ddcb_req[].
614 *
615 * Do not forget to call this function when genwqe_wait_ddcb() fails,
616 * such that the request gets really removed from ddcb_req[].
617 *
618 * Return: 0 success
619 */
__genwqe_purge_ddcb(struct genwqe_dev * cd,struct ddcb_requ * req)620 int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
621 {
622 struct ddcb *pddcb = NULL;
623 unsigned int t;
624 unsigned long flags;
625 struct ddcb_queue *queue = req->queue;
626 struct pci_dev *pci_dev = cd->pci_dev;
627 u64 queue_status;
628 __be32 icrc_hsi_shi = 0x0000;
629 __be32 old, new;
630
631 /* unsigned long flags; */
632 if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {
633 dev_err(&pci_dev->dev,
634 "[%s] err: software timeout is not set!\n", __func__);
635 return -EFAULT;
636 }
637
638 pddcb = &queue->ddcb_vaddr[req->num];
639
640 for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {
641
642 spin_lock_irqsave(&queue->ddcb_lock, flags);
643
644 /* Check if req was meanwhile finished */
645 if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
646 goto go_home;
647
648 /* try to set PURGE bit if FETCHED/COMPLETED are not set */
649 old = pddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
650 if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
651
652 new = (old | DDCB_PURGE_BE32);
653 icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
654 old, new);
655 if (icrc_hsi_shi == old)
656 goto finish_ddcb;
657 }
658
659 /* normal finish with HSI bit */
660 barrier();
661 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
662 if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
663 goto finish_ddcb;
664
665 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
666
667 /*
668 * Here the check_ddcb() function will most likely
669 * discover this DDCB to be finished some point in
670 * time. It will mark the req finished and free it up
671 * in the list.
672 */
673
674 copy_ddcb_results(req, req->num); /* for the failing case */
675 msleep(100); /* sleep for 1/10 second and try again */
676 continue;
677
678 finish_ddcb:
679 copy_ddcb_results(req, req->num);
680 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
681 queue->ddcbs_in_flight--;
682 queue->ddcb_req[req->num] = NULL; /* delete from array */
683 ddcb_mark_cleared(pddcb);
684
685 /* Move active DDCB further; Nothing to do here anymore. */
686
687 /*
688 * We need to ensure that there is at least one free
689 * DDCB in the queue. To do that, we must update
690 * ddcb_act only if the COMPLETED bit is set for the
691 * DDCB we are working on else we treat that DDCB even
692 * if we PURGED it as occupied (hardware is supposed
693 * to set the COMPLETED bit yet!).
694 */
695 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
696 if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
697 (queue->ddcb_act == req->num)) {
698 queue->ddcb_act = ((queue->ddcb_act + 1) %
699 queue->ddcb_max);
700 }
701 go_home:
702 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
703 return 0;
704 }
705
706 /*
707 * If the card is dead and the queue is forced to stop, we
708 * might see this in the queue status register.
709 */
710 queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
711
712 dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
713 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
714
715 dev_err(&pci_dev->dev,
716 "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
717 __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,
718 queue_status);
719
720 print_ddcb_info(cd, req->queue);
721
722 return -EFAULT;
723 }
724
genwqe_init_debug_data(struct genwqe_dev * cd,struct genwqe_debug_data * d)725 int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
726 {
727 int len;
728 struct pci_dev *pci_dev = cd->pci_dev;
729
730 if (d == NULL) {
731 dev_err(&pci_dev->dev,
732 "[%s] err: invalid memory for debug data!\n",
733 __func__);
734 return -EFAULT;
735 }
736
737 len = sizeof(d->driver_version);
738 snprintf(d->driver_version, len, "%s", DRV_VERSION);
739 d->slu_unitcfg = cd->slu_unitcfg;
740 d->app_unitcfg = cd->app_unitcfg;
741 return 0;
742 }
743
744 /**
745 * __genwqe_enqueue_ddcb() - Enqueue a DDCB
746 * @cd: pointer to genwqe device descriptor
747 * @req: pointer to DDCB execution request
748 * @f_flags: file mode: blocking, non-blocking
749 *
750 * Return: 0 if enqueuing succeeded
751 * -EIO if card is unusable/PCIe problems
752 * -EBUSY if enqueuing failed
753 */
__genwqe_enqueue_ddcb(struct genwqe_dev * cd,struct ddcb_requ * req,unsigned int f_flags)754 int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
755 unsigned int f_flags)
756 {
757 struct ddcb *pddcb;
758 unsigned long flags;
759 struct ddcb_queue *queue;
760 struct pci_dev *pci_dev = cd->pci_dev;
761 u16 icrc;
762
763 retry:
764 if (cd->card_state != GENWQE_CARD_USED) {
765 printk_ratelimited(KERN_ERR
766 "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
767 GENWQE_DEVNAME, dev_name(&pci_dev->dev),
768 __func__, req->num);
769 return -EIO;
770 }
771
772 queue = req->queue = &cd->queue;
773
774 /* FIXME circumvention to improve performance when no irq is
775 * there.
776 */
777 if (GENWQE_POLLING_ENABLED)
778 genwqe_check_ddcb_queue(cd, queue);
779
780 /*
781 * It must be ensured to process all DDCBs in successive
782 * order. Use a lock here in order to prevent nested DDCB
783 * enqueuing.
784 */
785 spin_lock_irqsave(&queue->ddcb_lock, flags);
786
787 pddcb = get_next_ddcb(cd, queue, &req->num); /* get ptr and num */
788 if (pddcb == NULL) {
789 int rc;
790
791 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
792
793 if (f_flags & O_NONBLOCK) {
794 queue->return_on_busy++;
795 return -EBUSY;
796 }
797
798 queue->wait_on_busy++;
799 rc = wait_event_interruptible(queue->busy_waitq,
800 queue_free_ddcbs(queue) != 0);
801 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
802 __func__, rc);
803 if (rc == -ERESTARTSYS)
804 return rc; /* interrupted by a signal */
805
806 goto retry;
807 }
808
809 if (queue->ddcb_req[req->num] != NULL) {
810 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
811
812 dev_err(&pci_dev->dev,
813 "[%s] picked DDCB %d with req=%p still in use!!\n",
814 __func__, req->num, req);
815 return -EFAULT;
816 }
817 ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
818 queue->ddcb_req[req->num] = req;
819
820 pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
821 pddcb->cmd = req->cmd.cmd;
822 pddcb->acfunc = req->cmd.acfunc; /* functional unit */
823
824 /*
825 * We know that we can get retc 0x104 with CRC error, do not
826 * stop the queue in those cases for this command. XDIR = 1
827 * does not work for old SLU versions.
828 *
829 * Last bitstream with the old XDIR behavior had SLU_ID
830 * 0x34199.
831 */
832 if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
833 pddcb->xdir = 0x1;
834 else
835 pddcb->xdir = 0x0;
836
837
838 pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
839 ((req->cmd.asv_length / 8)));
840 pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
841
842 /*
843 * If copying the whole DDCB_ASIV_LENGTH is impacting
844 * performance we need to change it to
845 * req->cmd.asiv_length. But simulation benefits from some
846 * non-architectured bits behind the architectured content.
847 *
848 * How much data is copied depends on the availability of the
849 * ATS field, which was introduced late. If the ATS field is
850 * supported ASIV is 8 bytes shorter than it used to be. Since
851 * the ATS field is copied too, the code should do exactly
852 * what it did before, but I wanted to make copying of the ATS
853 * field very explicit.
854 */
855 if (genwqe_get_slu_id(cd) <= 0x2) {
856 memcpy(&pddcb->__asiv[0], /* destination */
857 &req->cmd.__asiv[0], /* source */
858 DDCB_ASIV_LENGTH); /* req->cmd.asiv_length */
859 } else {
860 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
861 memcpy(&pddcb->n.asiv[0], /* destination */
862 &req->cmd.asiv[0], /* source */
863 DDCB_ASIV_LENGTH_ATS); /* req->cmd.asiv_length */
864 }
865
866 pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */
867
868 /*
869 * Calculate CRC_16 for corresponding range PSP(7:4). Include
870 * empty 4 bytes prior to the data.
871 */
872 icrc = genwqe_crc16((const u8 *)pddcb,
873 ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
874 pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
875
876 /* enable DDCB completion irq */
877 if (!GENWQE_POLLING_ENABLED)
878 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
879
880 dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
881 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
882
883 if (ddcb_requ_collect_debug_data(req)) {
884 /* use the kernel copy of debug data. copying back to
885 user buffer happens later */
886
887 genwqe_init_debug_data(cd, &req->debug_data);
888 memcpy(&req->debug_data.ddcb_before, pddcb,
889 sizeof(req->debug_data.ddcb_before));
890 }
891
892 enqueue_ddcb(cd, queue, pddcb, req->num);
893 queue->ddcbs_in_flight++;
894
895 if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
896 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
897
898 ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
899 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
900 wake_up_interruptible(&cd->queue_waitq);
901
902 return 0;
903 }
904
905 /**
906 * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
907 * @cd: pointer to genwqe device descriptor
908 * @cmd: user provided DDCB command
909 * @f_flags: file mode: blocking, non-blocking
910 */
__genwqe_execute_raw_ddcb(struct genwqe_dev * cd,struct genwqe_ddcb_cmd * cmd,unsigned int f_flags)911 int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
912 struct genwqe_ddcb_cmd *cmd,
913 unsigned int f_flags)
914 {
915 int rc = 0;
916 struct pci_dev *pci_dev = cd->pci_dev;
917 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
918
919 if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
920 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
921 __func__, cmd->asiv_length);
922 return -EINVAL;
923 }
924 if (cmd->asv_length > DDCB_ASV_LENGTH) {
925 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
926 __func__, cmd->asiv_length);
927 return -EINVAL;
928 }
929 rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
930 if (rc != 0)
931 return rc;
932
933 rc = __genwqe_wait_ddcb(cd, req);
934 if (rc < 0) /* error or signal interrupt */
935 goto err_exit;
936
937 if (ddcb_requ_collect_debug_data(req)) {
938 if (copy_to_user((struct genwqe_debug_data __user *)
939 (unsigned long)cmd->ddata_addr,
940 &req->debug_data,
941 sizeof(struct genwqe_debug_data)))
942 return -EFAULT;
943 }
944
945 /*
946 * Higher values than 0x102 indicate completion with faults,
947 * lower values than 0x102 indicate processing faults. Note
948 * that DDCB might have been purged. E.g. Cntl+C.
949 */
950 if (cmd->retc != DDCB_RETC_COMPLETE) {
951 /* This might happen e.g. flash read, and needs to be
952 handled by the upper layer code. */
953 rc = -EBADMSG; /* not processed/error retc */
954 }
955
956 return rc;
957
958 err_exit:
959 __genwqe_purge_ddcb(cd, req);
960
961 if (ddcb_requ_collect_debug_data(req)) {
962 if (copy_to_user((struct genwqe_debug_data __user *)
963 (unsigned long)cmd->ddata_addr,
964 &req->debug_data,
965 sizeof(struct genwqe_debug_data)))
966 return -EFAULT;
967 }
968 return rc;
969 }
970
971 /**
972 * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
973 * @cd: pointer to genwqe device descriptor
974 *
975 * We use this as condition for our wait-queue code.
976 */
genwqe_next_ddcb_ready(struct genwqe_dev * cd)977 static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
978 {
979 unsigned long flags;
980 struct ddcb *pddcb;
981 struct ddcb_queue *queue = &cd->queue;
982
983 spin_lock_irqsave(&queue->ddcb_lock, flags);
984
985 if (queue_empty(queue)) { /* empty queue */
986 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
987 return 0;
988 }
989
990 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
991 if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */
992 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
993 return 1;
994 }
995
996 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
997 return 0;
998 }
999
1000 /**
1001 * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
1002 * @cd: pointer to genwqe device descriptor
1003 *
1004 * Keep track on the number of DDCBs which ware currently in the
1005 * queue. This is needed for statistics as well as condition if we want
1006 * to wait or better do polling in case of no interrupts available.
1007 */
genwqe_ddcbs_in_flight(struct genwqe_dev * cd)1008 int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
1009 {
1010 unsigned long flags;
1011 int ddcbs_in_flight = 0;
1012 struct ddcb_queue *queue = &cd->queue;
1013
1014 spin_lock_irqsave(&queue->ddcb_lock, flags);
1015 ddcbs_in_flight += queue->ddcbs_in_flight;
1016 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1017
1018 return ddcbs_in_flight;
1019 }
1020
setup_ddcb_queue(struct genwqe_dev * cd,struct ddcb_queue * queue)1021 static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1022 {
1023 int rc, i;
1024 struct ddcb *pddcb;
1025 u64 val64;
1026 unsigned int queue_size;
1027 struct pci_dev *pci_dev = cd->pci_dev;
1028
1029 if (GENWQE_DDCB_MAX < 2)
1030 return -EINVAL;
1031
1032 queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);
1033
1034 queue->ddcbs_in_flight = 0; /* statistics */
1035 queue->ddcbs_max_in_flight = 0;
1036 queue->ddcbs_completed = 0;
1037 queue->return_on_busy = 0;
1038 queue->wait_on_busy = 0;
1039
1040 queue->ddcb_seq = 0x100; /* start sequence number */
1041 queue->ddcb_max = GENWQE_DDCB_MAX;
1042 queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
1043 &queue->ddcb_daddr);
1044 if (queue->ddcb_vaddr == NULL) {
1045 dev_err(&pci_dev->dev,
1046 "[%s] **err: could not allocate DDCB **\n", __func__);
1047 return -ENOMEM;
1048 }
1049 queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *),
1050 GFP_KERNEL);
1051 if (!queue->ddcb_req) {
1052 rc = -ENOMEM;
1053 goto free_ddcbs;
1054 }
1055
1056 queue->ddcb_waitqs = kcalloc(queue->ddcb_max,
1057 sizeof(wait_queue_head_t),
1058 GFP_KERNEL);
1059 if (!queue->ddcb_waitqs) {
1060 rc = -ENOMEM;
1061 goto free_requs;
1062 }
1063
1064 for (i = 0; i < queue->ddcb_max; i++) {
1065 pddcb = &queue->ddcb_vaddr[i]; /* DDCBs */
1066 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
1067 pddcb->retc_16 = cpu_to_be16(0xfff);
1068
1069 queue->ddcb_req[i] = NULL; /* requests */
1070 init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */
1071 }
1072
1073 queue->ddcb_act = 0;
1074 queue->ddcb_next = 0; /* queue is empty */
1075
1076 spin_lock_init(&queue->ddcb_lock);
1077 init_waitqueue_head(&queue->busy_waitq);
1078
1079 val64 = ((u64)(queue->ddcb_max - 1) << 8); /* lastptr */
1080 __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG, 0x07); /* iCRC/vCRC */
1081 __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
1082 __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
1083 __genwqe_writeq(cd, queue->IO_QUEUE_WRAP, val64);
1084 return 0;
1085
1086 free_requs:
1087 kfree(queue->ddcb_req);
1088 queue->ddcb_req = NULL;
1089 free_ddcbs:
1090 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1091 queue->ddcb_daddr);
1092 queue->ddcb_vaddr = NULL;
1093 queue->ddcb_daddr = 0ull;
1094 return rc;
1095
1096 }
1097
ddcb_queue_initialized(struct ddcb_queue * queue)1098 static int ddcb_queue_initialized(struct ddcb_queue *queue)
1099 {
1100 return queue->ddcb_vaddr != NULL;
1101 }
1102
free_ddcb_queue(struct genwqe_dev * cd,struct ddcb_queue * queue)1103 static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1104 {
1105 unsigned int queue_size;
1106
1107 queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1108
1109 kfree(queue->ddcb_req);
1110 queue->ddcb_req = NULL;
1111
1112 if (queue->ddcb_vaddr) {
1113 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1114 queue->ddcb_daddr);
1115 queue->ddcb_vaddr = NULL;
1116 queue->ddcb_daddr = 0ull;
1117 }
1118 }
1119
genwqe_pf_isr(int irq,void * dev_id)1120 static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
1121 {
1122 u64 gfir;
1123 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1124 struct pci_dev *pci_dev = cd->pci_dev;
1125
1126 /*
1127 * In case of fatal FIR error the queue is stopped, such that
1128 * we can safely check it without risking anything.
1129 */
1130 cd->irqs_processed++;
1131 wake_up_interruptible(&cd->queue_waitq);
1132
1133 /*
1134 * Checking for errors before kicking the queue might be
1135 * safer, but slower for the good-case ... See above.
1136 */
1137 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
1138 if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
1139 !pci_channel_offline(pci_dev)) {
1140
1141 if (cd->use_platform_recovery) {
1142 /*
1143 * Since we use raw accessors, EEH errors won't be
1144 * detected by the platform until we do a non-raw
1145 * MMIO or config space read
1146 */
1147 readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1148
1149 /* Don't do anything if the PCI channel is frozen */
1150 if (pci_channel_offline(pci_dev))
1151 goto exit;
1152 }
1153
1154 wake_up_interruptible(&cd->health_waitq);
1155
1156 /*
1157 * By default GFIRs causes recovery actions. This
1158 * count is just for debug when recovery is masked.
1159 */
1160 dev_err_ratelimited(&pci_dev->dev,
1161 "[%s] GFIR=%016llx\n",
1162 __func__, gfir);
1163 }
1164
1165 exit:
1166 return IRQ_HANDLED;
1167 }
1168
genwqe_vf_isr(int irq,void * dev_id)1169 static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
1170 {
1171 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1172
1173 cd->irqs_processed++;
1174 wake_up_interruptible(&cd->queue_waitq);
1175
1176 return IRQ_HANDLED;
1177 }
1178
1179 /**
1180 * genwqe_card_thread() - Work thread for the DDCB queue
1181 * @data: pointer to genwqe device descriptor
1182 *
1183 * The idea is to check if there are DDCBs in processing. If there are
1184 * some finished DDCBs, we process them and wakeup the
1185 * requestors. Otherwise we give other processes time using
1186 * cond_resched().
1187 */
genwqe_card_thread(void * data)1188 static int genwqe_card_thread(void *data)
1189 {
1190 int should_stop = 0;
1191 struct genwqe_dev *cd = (struct genwqe_dev *)data;
1192
1193 while (!kthread_should_stop()) {
1194
1195 genwqe_check_ddcb_queue(cd, &cd->queue);
1196
1197 if (GENWQE_POLLING_ENABLED) {
1198 wait_event_interruptible_timeout(
1199 cd->queue_waitq,
1200 genwqe_ddcbs_in_flight(cd) ||
1201 (should_stop = kthread_should_stop()), 1);
1202 } else {
1203 wait_event_interruptible_timeout(
1204 cd->queue_waitq,
1205 genwqe_next_ddcb_ready(cd) ||
1206 (should_stop = kthread_should_stop()), HZ);
1207 }
1208 if (should_stop)
1209 break;
1210
1211 /*
1212 * Avoid soft lockups on heavy loads; we do not want
1213 * to disable our interrupts.
1214 */
1215 cond_resched();
1216 }
1217 return 0;
1218 }
1219
1220 /**
1221 * genwqe_setup_service_layer() - Setup DDCB queue
1222 * @cd: pointer to genwqe device descriptor
1223 *
1224 * Allocate DDCBs. Configure Service Layer Controller (SLC).
1225 *
1226 * Return: 0 success
1227 */
genwqe_setup_service_layer(struct genwqe_dev * cd)1228 int genwqe_setup_service_layer(struct genwqe_dev *cd)
1229 {
1230 int rc;
1231 struct ddcb_queue *queue;
1232 struct pci_dev *pci_dev = cd->pci_dev;
1233
1234 if (genwqe_is_privileged(cd)) {
1235 rc = genwqe_card_reset(cd);
1236 if (rc < 0) {
1237 dev_err(&pci_dev->dev,
1238 "[%s] err: reset failed.\n", __func__);
1239 return rc;
1240 }
1241 genwqe_read_softreset(cd);
1242 }
1243
1244 queue = &cd->queue;
1245 queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG;
1246 queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS;
1247 queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
1248 queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
1249 queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET;
1250 queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP;
1251 queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME;
1252 queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
1253 queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW;
1254
1255 rc = setup_ddcb_queue(cd, queue);
1256 if (rc != 0) {
1257 rc = -ENODEV;
1258 goto err_out;
1259 }
1260
1261 init_waitqueue_head(&cd->queue_waitq);
1262 cd->card_thread = kthread_run(genwqe_card_thread, cd,
1263 GENWQE_DEVNAME "%d_thread",
1264 cd->card_idx);
1265 if (IS_ERR(cd->card_thread)) {
1266 rc = PTR_ERR(cd->card_thread);
1267 cd->card_thread = NULL;
1268 goto stop_free_queue;
1269 }
1270
1271 rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
1272 if (rc)
1273 goto stop_kthread;
1274
1275 /*
1276 * We must have all wait-queues initialized when we enable the
1277 * interrupts. Otherwise we might crash if we get an early
1278 * irq.
1279 */
1280 init_waitqueue_head(&cd->health_waitq);
1281
1282 if (genwqe_is_privileged(cd)) {
1283 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
1284 GENWQE_DEVNAME, cd);
1285 } else {
1286 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
1287 GENWQE_DEVNAME, cd);
1288 }
1289 if (rc < 0) {
1290 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
1291 goto stop_irq_cap;
1292 }
1293
1294 cd->card_state = GENWQE_CARD_USED;
1295 return 0;
1296
1297 stop_irq_cap:
1298 genwqe_reset_interrupt_capability(cd);
1299 stop_kthread:
1300 kthread_stop(cd->card_thread);
1301 cd->card_thread = NULL;
1302 stop_free_queue:
1303 free_ddcb_queue(cd, queue);
1304 err_out:
1305 return rc;
1306 }
1307
1308 /**
1309 * queue_wake_up_all() - Handles fatal error case
1310 * @cd: pointer to genwqe device descriptor
1311 *
1312 * The PCI device got unusable and we have to stop all pending
1313 * requests as fast as we can. The code after this must purge the
1314 * DDCBs in question and ensure that all mappings are freed.
1315 */
queue_wake_up_all(struct genwqe_dev * cd)1316 static int queue_wake_up_all(struct genwqe_dev *cd)
1317 {
1318 unsigned int i;
1319 unsigned long flags;
1320 struct ddcb_queue *queue = &cd->queue;
1321
1322 spin_lock_irqsave(&queue->ddcb_lock, flags);
1323
1324 for (i = 0; i < queue->ddcb_max; i++)
1325 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
1326
1327 wake_up_interruptible(&queue->busy_waitq);
1328 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1329
1330 return 0;
1331 }
1332
1333 /**
1334 * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
1335 * @cd: pointer to genwqe device descriptor
1336 *
1337 * Relies on the pre-condition that there are no users of the card
1338 * device anymore e.g. with open file-descriptors.
1339 *
1340 * This function must be robust enough to be called twice.
1341 */
genwqe_finish_queue(struct genwqe_dev * cd)1342 int genwqe_finish_queue(struct genwqe_dev *cd)
1343 {
1344 int i, rc = 0, in_flight;
1345 int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;
1346 struct pci_dev *pci_dev = cd->pci_dev;
1347 struct ddcb_queue *queue = &cd->queue;
1348
1349 if (!ddcb_queue_initialized(queue))
1350 return 0;
1351
1352 /* Do not wipe out the error state. */
1353 if (cd->card_state == GENWQE_CARD_USED)
1354 cd->card_state = GENWQE_CARD_UNUSED;
1355
1356 /* Wake up all requests in the DDCB queue such that they
1357 should be removed nicely. */
1358 queue_wake_up_all(cd);
1359
1360 /* We must wait to get rid of the DDCBs in flight */
1361 for (i = 0; i < waitmax; i++) {
1362 in_flight = genwqe_ddcbs_in_flight(cd);
1363
1364 if (in_flight == 0)
1365 break;
1366
1367 dev_dbg(&pci_dev->dev,
1368 " DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
1369 i, waitmax, in_flight);
1370
1371 /*
1372 * Severe severe error situation: The card itself has
1373 * 16 DDCB queues, each queue has e.g. 32 entries,
1374 * each DDBC has a hardware timeout of currently 250
1375 * msec but the PFs have a hardware timeout of 8 sec
1376 * ... so I take something large.
1377 */
1378 msleep(1000);
1379 }
1380 if (i == waitmax) {
1381 dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n",
1382 __func__);
1383 rc = -EIO;
1384 }
1385 return rc;
1386 }
1387
1388 /**
1389 * genwqe_release_service_layer() - Shutdown DDCB queue
1390 * @cd: genwqe device descriptor
1391 *
1392 * This function must be robust enough to be called twice.
1393 */
genwqe_release_service_layer(struct genwqe_dev * cd)1394 int genwqe_release_service_layer(struct genwqe_dev *cd)
1395 {
1396 struct pci_dev *pci_dev = cd->pci_dev;
1397
1398 if (!ddcb_queue_initialized(&cd->queue))
1399 return 1;
1400
1401 free_irq(pci_dev->irq, cd);
1402 genwqe_reset_interrupt_capability(cd);
1403
1404 if (cd->card_thread != NULL) {
1405 kthread_stop(cd->card_thread);
1406 cd->card_thread = NULL;
1407 }
1408
1409 free_ddcb_queue(cd, &cd->queue);
1410 return 0;
1411 }
1412