1 // SPDX-License-Identifier: GPL-2.0-only
2 /**
3 * IBM Accelerator Family 'GenWQE'
4 *
5 * (C) Copyright IBM Corp. 2013
6 *
7 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
9 * Author: Michael Jung <mijung@gmx.net>
10 * Author: Michael Ruettger <michael@ibmra.de>
11 */
12
13 /*
14 * Device Driver Control Block (DDCB) queue support. Definition of
15 * interrupt handlers for queue support as well as triggering the
16 * health monitor code in case of problems. The current hardware uses
17 * an MSI interrupt which is shared between error handling and
18 * functional code.
19 */
20
21 #include <linux/types.h>
22 #include <linux/sched.h>
23 #include <linux/wait.h>
24 #include <linux/pci.h>
25 #include <linux/string.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/delay.h>
28 #include <linux/module.h>
29 #include <linux/interrupt.h>
30 #include <linux/crc-itu-t.h>
31
32 #include "card_base.h"
33 #include "card_ddcb.h"
34
35 /*
36 * N: next DDCB, this is where the next DDCB will be put.
37 * A: active DDCB, this is where the code will look for the next completion.
38 * x: DDCB is enqueued, we are waiting for its completion.
39
40 * Situation (1): Empty queue
41 * +---+---+---+---+---+---+---+---+
42 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
43 * | | | | | | | | |
44 * +---+---+---+---+---+---+---+---+
45 * A/N
46 * enqueued_ddcbs = A - N = 2 - 2 = 0
47 *
48 * Situation (2): Wrapped, N > A
49 * +---+---+---+---+---+---+---+---+
50 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
51 * | | | x | x | | | | |
52 * +---+---+---+---+---+---+---+---+
53 * A N
54 * enqueued_ddcbs = N - A = 4 - 2 = 2
55 *
56 * Situation (3): Queue wrapped, A > N
57 * +---+---+---+---+---+---+---+---+
58 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
59 * | x | x | | | x | x | x | x |
60 * +---+---+---+---+---+---+---+---+
61 * N A
62 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 6
63 *
64 * Situation (4a): Queue full N > A
65 * +---+---+---+---+---+---+---+---+
66 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
67 * | x | x | x | x | x | x | x | |
68 * +---+---+---+---+---+---+---+---+
69 * A N
70 *
71 * enqueued_ddcbs = N - A = 7 - 0 = 7
72 *
73 * Situation (4a): Queue full A > N
74 * +---+---+---+---+---+---+---+---+
75 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
76 * | x | x | x | | x | x | x | x |
77 * +---+---+---+---+---+---+---+---+
78 * N A
79 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
80 */
81
queue_empty(struct ddcb_queue * queue)82 static int queue_empty(struct ddcb_queue *queue)
83 {
84 return queue->ddcb_next == queue->ddcb_act;
85 }
86
queue_enqueued_ddcbs(struct ddcb_queue * queue)87 static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
88 {
89 if (queue->ddcb_next >= queue->ddcb_act)
90 return queue->ddcb_next - queue->ddcb_act;
91
92 return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
93 }
94
queue_free_ddcbs(struct ddcb_queue * queue)95 static int queue_free_ddcbs(struct ddcb_queue *queue)
96 {
97 int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
98
99 if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */
100 return 0;
101 }
102 return free_ddcbs;
103 }
104
105 /*
106 * Use of the PRIV field in the DDCB for queue debugging:
107 *
108 * (1) Trying to get rid of a DDCB which saw a timeout:
109 * pddcb->priv[6] = 0xcc; # cleared
110 *
111 * (2) Append a DDCB via NEXT bit:
112 * pddcb->priv[7] = 0xaa; # appended
113 *
114 * (3) DDCB needed tapping:
115 * pddcb->priv[7] = 0xbb; # tapped
116 *
117 * (4) DDCB marked as correctly finished:
118 * pddcb->priv[6] = 0xff; # finished
119 */
120
ddcb_mark_tapped(struct ddcb * pddcb)121 static inline void ddcb_mark_tapped(struct ddcb *pddcb)
122 {
123 pddcb->priv[7] = 0xbb; /* tapped */
124 }
125
ddcb_mark_appended(struct ddcb * pddcb)126 static inline void ddcb_mark_appended(struct ddcb *pddcb)
127 {
128 pddcb->priv[7] = 0xaa; /* appended */
129 }
130
ddcb_mark_cleared(struct ddcb * pddcb)131 static inline void ddcb_mark_cleared(struct ddcb *pddcb)
132 {
133 pddcb->priv[6] = 0xcc; /* cleared */
134 }
135
ddcb_mark_finished(struct ddcb * pddcb)136 static inline void ddcb_mark_finished(struct ddcb *pddcb)
137 {
138 pddcb->priv[6] = 0xff; /* finished */
139 }
140
ddcb_mark_unused(struct ddcb * pddcb)141 static inline void ddcb_mark_unused(struct ddcb *pddcb)
142 {
143 pddcb->priv_64 = cpu_to_be64(0); /* not tapped */
144 }
145
146 /**
147 * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
148 * @buff: pointer to data buffer
149 * @len: length of data for calculation
150 * @init: initial crc (0xffff at start)
151 *
152 * Polynomial = x^16 + x^12 + x^5 + 1 (0x1021)
153 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff
154 * should result in a crc16 of 0x89c3
155 *
156 * Return: crc16 checksum in big endian format !
157 */
genwqe_crc16(const u8 * buff,size_t len,u16 init)158 static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
159 {
160 return crc_itu_t(init, buff, len);
161 }
162
print_ddcb_info(struct genwqe_dev * cd,struct ddcb_queue * queue)163 static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
164 {
165 int i;
166 struct ddcb *pddcb;
167 unsigned long flags;
168 struct pci_dev *pci_dev = cd->pci_dev;
169
170 spin_lock_irqsave(&cd->print_lock, flags);
171
172 dev_info(&pci_dev->dev,
173 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
174 cd->card_idx, queue->ddcb_act, queue->ddcb_next);
175
176 pddcb = queue->ddcb_vaddr;
177 for (i = 0; i < queue->ddcb_max; i++) {
178 dev_err(&pci_dev->dev,
179 " %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
180 i == queue->ddcb_act ? '>' : ' ',
181 i,
182 be16_to_cpu(pddcb->retc_16),
183 be16_to_cpu(pddcb->seqnum_16),
184 pddcb->hsi,
185 pddcb->shi,
186 be64_to_cpu(pddcb->priv_64),
187 pddcb->cmd);
188 pddcb++;
189 }
190 spin_unlock_irqrestore(&cd->print_lock, flags);
191 }
192
ddcb_requ_alloc(void)193 struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
194 {
195 struct ddcb_requ *req;
196
197 req = kzalloc(sizeof(*req), GFP_KERNEL);
198 if (!req)
199 return NULL;
200
201 return &req->cmd;
202 }
203
ddcb_requ_free(struct genwqe_ddcb_cmd * cmd)204 void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
205 {
206 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
207
208 kfree(req);
209 }
210
ddcb_requ_get_state(struct ddcb_requ * req)211 static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
212 {
213 return req->req_state;
214 }
215
ddcb_requ_set_state(struct ddcb_requ * req,enum genwqe_requ_state new_state)216 static inline void ddcb_requ_set_state(struct ddcb_requ *req,
217 enum genwqe_requ_state new_state)
218 {
219 req->req_state = new_state;
220 }
221
ddcb_requ_collect_debug_data(struct ddcb_requ * req)222 static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
223 {
224 return req->cmd.ddata_addr != 0x0;
225 }
226
227 /**
228 * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
229 * @cd: pointer to genwqe device descriptor
230 * @req: DDCB work request
231 *
232 * Status of ddcb_requ mirrors this hardware state, but is copied in
233 * the ddcb_requ on interrupt/polling function. The lowlevel code
234 * should check the hardware state directly, the higher level code
235 * should check the copy.
236 *
237 * This function will also return true if the state of the queue is
238 * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the
239 * shutdown case.
240 */
ddcb_requ_finished(struct genwqe_dev * cd,struct ddcb_requ * req)241 static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
242 {
243 return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
244 (cd->card_state != GENWQE_CARD_USED);
245 }
246
247 /**
248 * enqueue_ddcb() - Enqueue a DDCB
249 * @cd: pointer to genwqe device descriptor
250 * @queue: queue this operation should be done on
251 * @ddcb_no: pointer to ddcb number being tapped
252 *
253 * Start execution of DDCB by tapping or append to queue via NEXT
254 * bit. This is done by an atomic 'compare and swap' instruction and
255 * checking SHI and HSI of the previous DDCB.
256 *
257 * This function must only be called with ddcb_lock held.
258 *
259 * Return: 1 if new DDCB is appended to previous
260 * 2 if DDCB queue is tapped via register/simulation
261 */
262 #define RET_DDCB_APPENDED 1
263 #define RET_DDCB_TAPPED 2
264
enqueue_ddcb(struct genwqe_dev * cd,struct ddcb_queue * queue,struct ddcb * pddcb,int ddcb_no)265 static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
266 struct ddcb *pddcb, int ddcb_no)
267 {
268 unsigned int try;
269 int prev_no;
270 struct ddcb *prev_ddcb;
271 __be32 old, new, icrc_hsi_shi;
272 u64 num;
273
274 /*
275 * For performance checks a Dispatch Timestamp can be put into
276 * DDCB It is supposed to use the SLU's free running counter,
277 * but this requires PCIe cycles.
278 */
279 ddcb_mark_unused(pddcb);
280
281 /* check previous DDCB if already fetched */
282 prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
283 prev_ddcb = &queue->ddcb_vaddr[prev_no];
284
285 /*
286 * It might have happened that the HSI.FETCHED bit is
287 * set. Retry in this case. Therefore I expect maximum 2 times
288 * trying.
289 */
290 ddcb_mark_appended(pddcb);
291 for (try = 0; try < 2; try++) {
292 old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
293
294 /* try to append via NEXT bit if prev DDCB is not completed */
295 if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
296 break;
297
298 new = (old | DDCB_NEXT_BE32);
299
300 wmb(); /* need to ensure write ordering */
301 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
302
303 if (icrc_hsi_shi == old)
304 return RET_DDCB_APPENDED; /* appended to queue */
305 }
306
307 /* Queue must be re-started by updating QUEUE_OFFSET */
308 ddcb_mark_tapped(pddcb);
309 num = (u64)ddcb_no << 8;
310
311 wmb(); /* need to ensure write ordering */
312 __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */
313
314 return RET_DDCB_TAPPED;
315 }
316
317 /**
318 * copy_ddcb_results() - Copy output state from real DDCB to request
319 *
320 * Copy DDCB ASV to request struct. There is no endian
321 * conversion made, since data structure in ASV is still
322 * unknown here.
323 *
324 * This is needed by:
325 * - genwqe_purge_ddcb()
326 * - genwqe_check_ddcb_queue()
327 */
copy_ddcb_results(struct ddcb_requ * req,int ddcb_no)328 static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
329 {
330 struct ddcb_queue *queue = req->queue;
331 struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
332
333 memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
334
335 /* copy status flags of the variant part */
336 req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16);
337 req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
338 req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
339
340 req->cmd.attn = be16_to_cpu(pddcb->attn_16);
341 req->cmd.progress = be32_to_cpu(pddcb->progress_32);
342 req->cmd.retc = be16_to_cpu(pddcb->retc_16);
343
344 if (ddcb_requ_collect_debug_data(req)) {
345 int prev_no = (ddcb_no == 0) ?
346 queue->ddcb_max - 1 : ddcb_no - 1;
347 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
348
349 memcpy(&req->debug_data.ddcb_finished, pddcb,
350 sizeof(req->debug_data.ddcb_finished));
351 memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
352 sizeof(req->debug_data.ddcb_prev));
353 }
354 }
355
356 /**
357 * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work equests.
358 * @cd: pointer to genwqe device descriptor
359 *
360 * Return: Number of DDCBs which were finished
361 */
genwqe_check_ddcb_queue(struct genwqe_dev * cd,struct ddcb_queue * queue)362 static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
363 struct ddcb_queue *queue)
364 {
365 unsigned long flags;
366 int ddcbs_finished = 0;
367 struct pci_dev *pci_dev = cd->pci_dev;
368
369 spin_lock_irqsave(&queue->ddcb_lock, flags);
370
371 /* FIXME avoid soft locking CPU */
372 while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
373
374 struct ddcb *pddcb;
375 struct ddcb_requ *req;
376 u16 vcrc, vcrc_16, retc_16;
377
378 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
379
380 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
381 0x00000000)
382 goto go_home; /* not completed, continue waiting */
383
384 wmb(); /* Add sync to decouple prev. read operations */
385
386 /* Note: DDCB could be purged */
387 req = queue->ddcb_req[queue->ddcb_act];
388 if (req == NULL) {
389 /* this occurs if DDCB is purged, not an error */
390 /* Move active DDCB further; Nothing to do anymore. */
391 goto pick_next_one;
392 }
393
394 /*
395 * HSI=0x44 (fetched and completed), but RETC is
396 * 0x101, or even worse 0x000.
397 *
398 * In case of seeing the queue in inconsistent state
399 * we read the errcnts and the queue status to provide
400 * a trigger for our PCIe analyzer stop capturing.
401 */
402 retc_16 = be16_to_cpu(pddcb->retc_16);
403 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
404 u64 errcnts, status;
405 u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
406
407 errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
408 status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
409
410 dev_err(&pci_dev->dev,
411 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
412 __func__, be16_to_cpu(pddcb->seqnum_16),
413 pddcb->hsi, retc_16, errcnts, status,
414 queue->ddcb_daddr + ddcb_offs);
415 }
416
417 copy_ddcb_results(req, queue->ddcb_act);
418 queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */
419
420 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
421 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
422
423 ddcb_mark_finished(pddcb);
424
425 /* calculate CRC_16 to see if VCRC is correct */
426 vcrc = genwqe_crc16(pddcb->asv,
427 VCRC_LENGTH(req->cmd.asv_length),
428 0xffff);
429 vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
430 if (vcrc != vcrc_16) {
431 printk_ratelimited(KERN_ERR
432 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
433 GENWQE_DEVNAME, dev_name(&pci_dev->dev),
434 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
435 vcrc, vcrc_16);
436 }
437
438 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
439 queue->ddcbs_completed++;
440 queue->ddcbs_in_flight--;
441
442 /* wake up process waiting for this DDCB, and
443 processes on the busy queue */
444 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
445 wake_up_interruptible(&queue->busy_waitq);
446
447 pick_next_one:
448 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
449 ddcbs_finished++;
450 }
451
452 go_home:
453 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
454 return ddcbs_finished;
455 }
456
457 /**
458 * __genwqe_wait_ddcb(): Waits until DDCB is completed
459 * @cd: pointer to genwqe device descriptor
460 * @req: pointer to requsted DDCB parameters
461 *
462 * The Service Layer will update the RETC in DDCB when processing is
463 * pending or done.
464 *
465 * Return: > 0 remaining jiffies, DDCB completed
466 * -ETIMEDOUT when timeout
467 * -ERESTARTSYS when ^C
468 * -EINVAL when unknown error condition
469 *
470 * When an error is returned the called needs to ensure that
471 * purge_ddcb() is being called to get the &req removed from the
472 * queue.
473 */
__genwqe_wait_ddcb(struct genwqe_dev * cd,struct ddcb_requ * req)474 int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
475 {
476 int rc;
477 unsigned int ddcb_no;
478 struct ddcb_queue *queue;
479 struct pci_dev *pci_dev = cd->pci_dev;
480
481 if (req == NULL)
482 return -EINVAL;
483
484 queue = req->queue;
485 if (queue == NULL)
486 return -EINVAL;
487
488 ddcb_no = req->num;
489 if (ddcb_no >= queue->ddcb_max)
490 return -EINVAL;
491
492 rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
493 ddcb_requ_finished(cd, req),
494 GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);
495
496 /*
497 * We need to distinguish 3 cases here:
498 * 1. rc == 0 timeout occured
499 * 2. rc == -ERESTARTSYS signal received
500 * 3. rc > 0 remaining jiffies condition is true
501 */
502 if (rc == 0) {
503 struct ddcb_queue *queue = req->queue;
504 struct ddcb *pddcb;
505
506 /*
507 * Timeout may be caused by long task switching time.
508 * When timeout happens, check if the request has
509 * meanwhile completed.
510 */
511 genwqe_check_ddcb_queue(cd, req->queue);
512 if (ddcb_requ_finished(cd, req))
513 return rc;
514
515 dev_err(&pci_dev->dev,
516 "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
517 __func__, req->num, rc, ddcb_requ_get_state(req),
518 req);
519 dev_err(&pci_dev->dev,
520 "[%s] IO_QUEUE_STATUS=0x%016llx\n", __func__,
521 __genwqe_readq(cd, queue->IO_QUEUE_STATUS));
522
523 pddcb = &queue->ddcb_vaddr[req->num];
524 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
525
526 print_ddcb_info(cd, req->queue);
527 return -ETIMEDOUT;
528
529 } else if (rc == -ERESTARTSYS) {
530 return rc;
531 /*
532 * EINTR: Stops the application
533 * ERESTARTSYS: Restartable systemcall; called again
534 */
535
536 } else if (rc < 0) {
537 dev_err(&pci_dev->dev,
538 "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
539 __func__, req->num, rc, ddcb_requ_get_state(req));
540 return -EINVAL;
541 }
542
543 /* Severe error occured. Driver is forced to stop operation */
544 if (cd->card_state != GENWQE_CARD_USED) {
545 dev_err(&pci_dev->dev,
546 "[%s] err: DDCB#%d forced to stop (rc=%d)\n",
547 __func__, req->num, rc);
548 return -EIO;
549 }
550 return rc;
551 }
552
553 /**
554 * get_next_ddcb() - Get next available DDCB
555 * @cd: pointer to genwqe device descriptor
556 *
557 * DDCB's content is completely cleared but presets for PRE and
558 * SEQNUM. This function must only be called when ddcb_lock is held.
559 *
560 * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.
561 */
get_next_ddcb(struct genwqe_dev * cd,struct ddcb_queue * queue,int * num)562 static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
563 struct ddcb_queue *queue,
564 int *num)
565 {
566 u64 *pu64;
567 struct ddcb *pddcb;
568
569 if (queue_free_ddcbs(queue) == 0) /* queue is full */
570 return NULL;
571
572 /* find new ddcb */
573 pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
574
575 /* if it is not completed, we are not allowed to use it */
576 /* barrier(); */
577 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
578 return NULL;
579
580 *num = queue->ddcb_next; /* internal DDCB number */
581 queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
582
583 /* clear important DDCB fields */
584 pu64 = (u64 *)pddcb;
585 pu64[0] = 0ULL; /* offs 0x00 (ICRC,HSI,SHI,...) */
586 pu64[1] = 0ULL; /* offs 0x01 (ACFUNC,CMD...) */
587
588 /* destroy previous results in ASV */
589 pu64[0x80/8] = 0ULL; /* offs 0x80 (ASV + 0) */
590 pu64[0x88/8] = 0ULL; /* offs 0x88 (ASV + 0x08) */
591 pu64[0x90/8] = 0ULL; /* offs 0x90 (ASV + 0x10) */
592 pu64[0x98/8] = 0ULL; /* offs 0x98 (ASV + 0x18) */
593 pu64[0xd0/8] = 0ULL; /* offs 0xd0 (RETC,ATTN...) */
594
595 pddcb->pre = DDCB_PRESET_PRE; /* 128 */
596 pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
597 return pddcb;
598 }
599
600 /**
601 * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
602 * @cd: genwqe device descriptor
603 * @req: DDCB request
604 *
605 * This will fail when the request was already FETCHED. In this case
606 * we need to wait until it is finished. Else the DDCB can be
607 * reused. This function also ensures that the request data structure
608 * is removed from ddcb_req[].
609 *
610 * Do not forget to call this function when genwqe_wait_ddcb() fails,
611 * such that the request gets really removed from ddcb_req[].
612 *
613 * Return: 0 success
614 */
__genwqe_purge_ddcb(struct genwqe_dev * cd,struct ddcb_requ * req)615 int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
616 {
617 struct ddcb *pddcb = NULL;
618 unsigned int t;
619 unsigned long flags;
620 struct ddcb_queue *queue = req->queue;
621 struct pci_dev *pci_dev = cd->pci_dev;
622 u64 queue_status;
623 __be32 icrc_hsi_shi = 0x0000;
624 __be32 old, new;
625
626 /* unsigned long flags; */
627 if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {
628 dev_err(&pci_dev->dev,
629 "[%s] err: software timeout is not set!\n", __func__);
630 return -EFAULT;
631 }
632
633 pddcb = &queue->ddcb_vaddr[req->num];
634
635 for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {
636
637 spin_lock_irqsave(&queue->ddcb_lock, flags);
638
639 /* Check if req was meanwhile finished */
640 if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
641 goto go_home;
642
643 /* try to set PURGE bit if FETCHED/COMPLETED are not set */
644 old = pddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
645 if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
646
647 new = (old | DDCB_PURGE_BE32);
648 icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
649 old, new);
650 if (icrc_hsi_shi == old)
651 goto finish_ddcb;
652 }
653
654 /* normal finish with HSI bit */
655 barrier();
656 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
657 if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
658 goto finish_ddcb;
659
660 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
661
662 /*
663 * Here the check_ddcb() function will most likely
664 * discover this DDCB to be finished some point in
665 * time. It will mark the req finished and free it up
666 * in the list.
667 */
668
669 copy_ddcb_results(req, req->num); /* for the failing case */
670 msleep(100); /* sleep for 1/10 second and try again */
671 continue;
672
673 finish_ddcb:
674 copy_ddcb_results(req, req->num);
675 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
676 queue->ddcbs_in_flight--;
677 queue->ddcb_req[req->num] = NULL; /* delete from array */
678 ddcb_mark_cleared(pddcb);
679
680 /* Move active DDCB further; Nothing to do here anymore. */
681
682 /*
683 * We need to ensure that there is at least one free
684 * DDCB in the queue. To do that, we must update
685 * ddcb_act only if the COMPLETED bit is set for the
686 * DDCB we are working on else we treat that DDCB even
687 * if we PURGED it as occupied (hardware is supposed
688 * to set the COMPLETED bit yet!).
689 */
690 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
691 if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
692 (queue->ddcb_act == req->num)) {
693 queue->ddcb_act = ((queue->ddcb_act + 1) %
694 queue->ddcb_max);
695 }
696 go_home:
697 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
698 return 0;
699 }
700
701 /*
702 * If the card is dead and the queue is forced to stop, we
703 * might see this in the queue status register.
704 */
705 queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
706
707 dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
708 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
709
710 dev_err(&pci_dev->dev,
711 "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
712 __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,
713 queue_status);
714
715 print_ddcb_info(cd, req->queue);
716
717 return -EFAULT;
718 }
719
genwqe_init_debug_data(struct genwqe_dev * cd,struct genwqe_debug_data * d)720 int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
721 {
722 int len;
723 struct pci_dev *pci_dev = cd->pci_dev;
724
725 if (d == NULL) {
726 dev_err(&pci_dev->dev,
727 "[%s] err: invalid memory for debug data!\n",
728 __func__);
729 return -EFAULT;
730 }
731
732 len = sizeof(d->driver_version);
733 snprintf(d->driver_version, len, "%s", DRV_VERSION);
734 d->slu_unitcfg = cd->slu_unitcfg;
735 d->app_unitcfg = cd->app_unitcfg;
736 return 0;
737 }
738
739 /**
740 * __genwqe_enqueue_ddcb() - Enqueue a DDCB
741 * @cd: pointer to genwqe device descriptor
742 * @req: pointer to DDCB execution request
743 * @f_flags: file mode: blocking, non-blocking
744 *
745 * Return: 0 if enqueuing succeeded
746 * -EIO if card is unusable/PCIe problems
747 * -EBUSY if enqueuing failed
748 */
__genwqe_enqueue_ddcb(struct genwqe_dev * cd,struct ddcb_requ * req,unsigned int f_flags)749 int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
750 unsigned int f_flags)
751 {
752 struct ddcb *pddcb;
753 unsigned long flags;
754 struct ddcb_queue *queue;
755 struct pci_dev *pci_dev = cd->pci_dev;
756 u16 icrc;
757
758 retry:
759 if (cd->card_state != GENWQE_CARD_USED) {
760 printk_ratelimited(KERN_ERR
761 "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
762 GENWQE_DEVNAME, dev_name(&pci_dev->dev),
763 __func__, req->num);
764 return -EIO;
765 }
766
767 queue = req->queue = &cd->queue;
768
769 /* FIXME circumvention to improve performance when no irq is
770 * there.
771 */
772 if (GENWQE_POLLING_ENABLED)
773 genwqe_check_ddcb_queue(cd, queue);
774
775 /*
776 * It must be ensured to process all DDCBs in successive
777 * order. Use a lock here in order to prevent nested DDCB
778 * enqueuing.
779 */
780 spin_lock_irqsave(&queue->ddcb_lock, flags);
781
782 pddcb = get_next_ddcb(cd, queue, &req->num); /* get ptr and num */
783 if (pddcb == NULL) {
784 int rc;
785
786 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
787
788 if (f_flags & O_NONBLOCK) {
789 queue->return_on_busy++;
790 return -EBUSY;
791 }
792
793 queue->wait_on_busy++;
794 rc = wait_event_interruptible(queue->busy_waitq,
795 queue_free_ddcbs(queue) != 0);
796 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
797 __func__, rc);
798 if (rc == -ERESTARTSYS)
799 return rc; /* interrupted by a signal */
800
801 goto retry;
802 }
803
804 if (queue->ddcb_req[req->num] != NULL) {
805 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
806
807 dev_err(&pci_dev->dev,
808 "[%s] picked DDCB %d with req=%p still in use!!\n",
809 __func__, req->num, req);
810 return -EFAULT;
811 }
812 ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
813 queue->ddcb_req[req->num] = req;
814
815 pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
816 pddcb->cmd = req->cmd.cmd;
817 pddcb->acfunc = req->cmd.acfunc; /* functional unit */
818
819 /*
820 * We know that we can get retc 0x104 with CRC error, do not
821 * stop the queue in those cases for this command. XDIR = 1
822 * does not work for old SLU versions.
823 *
824 * Last bitstream with the old XDIR behavior had SLU_ID
825 * 0x34199.
826 */
827 if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
828 pddcb->xdir = 0x1;
829 else
830 pddcb->xdir = 0x0;
831
832
833 pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
834 ((req->cmd.asv_length / 8)));
835 pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
836
837 /*
838 * If copying the whole DDCB_ASIV_LENGTH is impacting
839 * performance we need to change it to
840 * req->cmd.asiv_length. But simulation benefits from some
841 * non-architectured bits behind the architectured content.
842 *
843 * How much data is copied depends on the availability of the
844 * ATS field, which was introduced late. If the ATS field is
845 * supported ASIV is 8 bytes shorter than it used to be. Since
846 * the ATS field is copied too, the code should do exactly
847 * what it did before, but I wanted to make copying of the ATS
848 * field very explicit.
849 */
850 if (genwqe_get_slu_id(cd) <= 0x2) {
851 memcpy(&pddcb->__asiv[0], /* destination */
852 &req->cmd.__asiv[0], /* source */
853 DDCB_ASIV_LENGTH); /* req->cmd.asiv_length */
854 } else {
855 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
856 memcpy(&pddcb->n.asiv[0], /* destination */
857 &req->cmd.asiv[0], /* source */
858 DDCB_ASIV_LENGTH_ATS); /* req->cmd.asiv_length */
859 }
860
861 pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */
862
863 /*
864 * Calculate CRC_16 for corresponding range PSP(7:4). Include
865 * empty 4 bytes prior to the data.
866 */
867 icrc = genwqe_crc16((const u8 *)pddcb,
868 ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
869 pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
870
871 /* enable DDCB completion irq */
872 if (!GENWQE_POLLING_ENABLED)
873 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
874
875 dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
876 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
877
878 if (ddcb_requ_collect_debug_data(req)) {
879 /* use the kernel copy of debug data. copying back to
880 user buffer happens later */
881
882 genwqe_init_debug_data(cd, &req->debug_data);
883 memcpy(&req->debug_data.ddcb_before, pddcb,
884 sizeof(req->debug_data.ddcb_before));
885 }
886
887 enqueue_ddcb(cd, queue, pddcb, req->num);
888 queue->ddcbs_in_flight++;
889
890 if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
891 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
892
893 ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
894 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
895 wake_up_interruptible(&cd->queue_waitq);
896
897 return 0;
898 }
899
900 /**
901 * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
902 * @cd: pointer to genwqe device descriptor
903 * @req: user provided DDCB request
904 * @f_flags: file mode: blocking, non-blocking
905 */
__genwqe_execute_raw_ddcb(struct genwqe_dev * cd,struct genwqe_ddcb_cmd * cmd,unsigned int f_flags)906 int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
907 struct genwqe_ddcb_cmd *cmd,
908 unsigned int f_flags)
909 {
910 int rc = 0;
911 struct pci_dev *pci_dev = cd->pci_dev;
912 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
913
914 if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
915 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
916 __func__, cmd->asiv_length);
917 return -EINVAL;
918 }
919 if (cmd->asv_length > DDCB_ASV_LENGTH) {
920 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
921 __func__, cmd->asiv_length);
922 return -EINVAL;
923 }
924 rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
925 if (rc != 0)
926 return rc;
927
928 rc = __genwqe_wait_ddcb(cd, req);
929 if (rc < 0) /* error or signal interrupt */
930 goto err_exit;
931
932 if (ddcb_requ_collect_debug_data(req)) {
933 if (copy_to_user((struct genwqe_debug_data __user *)
934 (unsigned long)cmd->ddata_addr,
935 &req->debug_data,
936 sizeof(struct genwqe_debug_data)))
937 return -EFAULT;
938 }
939
940 /*
941 * Higher values than 0x102 indicate completion with faults,
942 * lower values than 0x102 indicate processing faults. Note
943 * that DDCB might have been purged. E.g. Cntl+C.
944 */
945 if (cmd->retc != DDCB_RETC_COMPLETE) {
946 /* This might happen e.g. flash read, and needs to be
947 handled by the upper layer code. */
948 rc = -EBADMSG; /* not processed/error retc */
949 }
950
951 return rc;
952
953 err_exit:
954 __genwqe_purge_ddcb(cd, req);
955
956 if (ddcb_requ_collect_debug_data(req)) {
957 if (copy_to_user((struct genwqe_debug_data __user *)
958 (unsigned long)cmd->ddata_addr,
959 &req->debug_data,
960 sizeof(struct genwqe_debug_data)))
961 return -EFAULT;
962 }
963 return rc;
964 }
965
966 /**
967 * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
968 *
969 * We use this as condition for our wait-queue code.
970 */
genwqe_next_ddcb_ready(struct genwqe_dev * cd)971 static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
972 {
973 unsigned long flags;
974 struct ddcb *pddcb;
975 struct ddcb_queue *queue = &cd->queue;
976
977 spin_lock_irqsave(&queue->ddcb_lock, flags);
978
979 if (queue_empty(queue)) { /* emtpy queue */
980 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
981 return 0;
982 }
983
984 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
985 if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */
986 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
987 return 1;
988 }
989
990 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
991 return 0;
992 }
993
994 /**
995 * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
996 *
997 * Keep track on the number of DDCBs which ware currently in the
998 * queue. This is needed for statistics as well as conditon if we want
999 * to wait or better do polling in case of no interrupts available.
1000 */
genwqe_ddcbs_in_flight(struct genwqe_dev * cd)1001 int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
1002 {
1003 unsigned long flags;
1004 int ddcbs_in_flight = 0;
1005 struct ddcb_queue *queue = &cd->queue;
1006
1007 spin_lock_irqsave(&queue->ddcb_lock, flags);
1008 ddcbs_in_flight += queue->ddcbs_in_flight;
1009 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1010
1011 return ddcbs_in_flight;
1012 }
1013
setup_ddcb_queue(struct genwqe_dev * cd,struct ddcb_queue * queue)1014 static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1015 {
1016 int rc, i;
1017 struct ddcb *pddcb;
1018 u64 val64;
1019 unsigned int queue_size;
1020 struct pci_dev *pci_dev = cd->pci_dev;
1021
1022 if (GENWQE_DDCB_MAX < 2)
1023 return -EINVAL;
1024
1025 queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);
1026
1027 queue->ddcbs_in_flight = 0; /* statistics */
1028 queue->ddcbs_max_in_flight = 0;
1029 queue->ddcbs_completed = 0;
1030 queue->return_on_busy = 0;
1031 queue->wait_on_busy = 0;
1032
1033 queue->ddcb_seq = 0x100; /* start sequence number */
1034 queue->ddcb_max = GENWQE_DDCB_MAX;
1035 queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
1036 &queue->ddcb_daddr);
1037 if (queue->ddcb_vaddr == NULL) {
1038 dev_err(&pci_dev->dev,
1039 "[%s] **err: could not allocate DDCB **\n", __func__);
1040 return -ENOMEM;
1041 }
1042 queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *),
1043 GFP_KERNEL);
1044 if (!queue->ddcb_req) {
1045 rc = -ENOMEM;
1046 goto free_ddcbs;
1047 }
1048
1049 queue->ddcb_waitqs = kcalloc(queue->ddcb_max,
1050 sizeof(wait_queue_head_t),
1051 GFP_KERNEL);
1052 if (!queue->ddcb_waitqs) {
1053 rc = -ENOMEM;
1054 goto free_requs;
1055 }
1056
1057 for (i = 0; i < queue->ddcb_max; i++) {
1058 pddcb = &queue->ddcb_vaddr[i]; /* DDCBs */
1059 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
1060 pddcb->retc_16 = cpu_to_be16(0xfff);
1061
1062 queue->ddcb_req[i] = NULL; /* requests */
1063 init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */
1064 }
1065
1066 queue->ddcb_act = 0;
1067 queue->ddcb_next = 0; /* queue is empty */
1068
1069 spin_lock_init(&queue->ddcb_lock);
1070 init_waitqueue_head(&queue->busy_waitq);
1071
1072 val64 = ((u64)(queue->ddcb_max - 1) << 8); /* lastptr */
1073 __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG, 0x07); /* iCRC/vCRC */
1074 __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
1075 __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
1076 __genwqe_writeq(cd, queue->IO_QUEUE_WRAP, val64);
1077 return 0;
1078
1079 free_requs:
1080 kfree(queue->ddcb_req);
1081 queue->ddcb_req = NULL;
1082 free_ddcbs:
1083 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1084 queue->ddcb_daddr);
1085 queue->ddcb_vaddr = NULL;
1086 queue->ddcb_daddr = 0ull;
1087 return -ENODEV;
1088
1089 }
1090
ddcb_queue_initialized(struct ddcb_queue * queue)1091 static int ddcb_queue_initialized(struct ddcb_queue *queue)
1092 {
1093 return queue->ddcb_vaddr != NULL;
1094 }
1095
free_ddcb_queue(struct genwqe_dev * cd,struct ddcb_queue * queue)1096 static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1097 {
1098 unsigned int queue_size;
1099
1100 queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1101
1102 kfree(queue->ddcb_req);
1103 queue->ddcb_req = NULL;
1104
1105 if (queue->ddcb_vaddr) {
1106 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1107 queue->ddcb_daddr);
1108 queue->ddcb_vaddr = NULL;
1109 queue->ddcb_daddr = 0ull;
1110 }
1111 }
1112
genwqe_pf_isr(int irq,void * dev_id)1113 static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
1114 {
1115 u64 gfir;
1116 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1117 struct pci_dev *pci_dev = cd->pci_dev;
1118
1119 /*
1120 * In case of fatal FIR error the queue is stopped, such that
1121 * we can safely check it without risking anything.
1122 */
1123 cd->irqs_processed++;
1124 wake_up_interruptible(&cd->queue_waitq);
1125
1126 /*
1127 * Checking for errors before kicking the queue might be
1128 * safer, but slower for the good-case ... See above.
1129 */
1130 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
1131 if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
1132 !pci_channel_offline(pci_dev)) {
1133
1134 if (cd->use_platform_recovery) {
1135 /*
1136 * Since we use raw accessors, EEH errors won't be
1137 * detected by the platform until we do a non-raw
1138 * MMIO or config space read
1139 */
1140 readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1141
1142 /* Don't do anything if the PCI channel is frozen */
1143 if (pci_channel_offline(pci_dev))
1144 goto exit;
1145 }
1146
1147 wake_up_interruptible(&cd->health_waitq);
1148
1149 /*
1150 * By default GFIRs causes recovery actions. This
1151 * count is just for debug when recovery is masked.
1152 */
1153 dev_err_ratelimited(&pci_dev->dev,
1154 "[%s] GFIR=%016llx\n",
1155 __func__, gfir);
1156 }
1157
1158 exit:
1159 return IRQ_HANDLED;
1160 }
1161
genwqe_vf_isr(int irq,void * dev_id)1162 static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
1163 {
1164 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1165
1166 cd->irqs_processed++;
1167 wake_up_interruptible(&cd->queue_waitq);
1168
1169 return IRQ_HANDLED;
1170 }
1171
1172 /**
1173 * genwqe_card_thread() - Work thread for the DDCB queue
1174 *
1175 * The idea is to check if there are DDCBs in processing. If there are
1176 * some finished DDCBs, we process them and wakeup the
1177 * requestors. Otherwise we give other processes time using
1178 * cond_resched().
1179 */
genwqe_card_thread(void * data)1180 static int genwqe_card_thread(void *data)
1181 {
1182 int should_stop = 0, rc = 0;
1183 struct genwqe_dev *cd = (struct genwqe_dev *)data;
1184
1185 while (!kthread_should_stop()) {
1186
1187 genwqe_check_ddcb_queue(cd, &cd->queue);
1188
1189 if (GENWQE_POLLING_ENABLED) {
1190 rc = wait_event_interruptible_timeout(
1191 cd->queue_waitq,
1192 genwqe_ddcbs_in_flight(cd) ||
1193 (should_stop = kthread_should_stop()), 1);
1194 } else {
1195 rc = wait_event_interruptible_timeout(
1196 cd->queue_waitq,
1197 genwqe_next_ddcb_ready(cd) ||
1198 (should_stop = kthread_should_stop()), HZ);
1199 }
1200 if (should_stop)
1201 break;
1202
1203 /*
1204 * Avoid soft lockups on heavy loads; we do not want
1205 * to disable our interrupts.
1206 */
1207 cond_resched();
1208 }
1209 return 0;
1210 }
1211
1212 /**
1213 * genwqe_setup_service_layer() - Setup DDCB queue
1214 * @cd: pointer to genwqe device descriptor
1215 *
1216 * Allocate DDCBs. Configure Service Layer Controller (SLC).
1217 *
1218 * Return: 0 success
1219 */
genwqe_setup_service_layer(struct genwqe_dev * cd)1220 int genwqe_setup_service_layer(struct genwqe_dev *cd)
1221 {
1222 int rc;
1223 struct ddcb_queue *queue;
1224 struct pci_dev *pci_dev = cd->pci_dev;
1225
1226 if (genwqe_is_privileged(cd)) {
1227 rc = genwqe_card_reset(cd);
1228 if (rc < 0) {
1229 dev_err(&pci_dev->dev,
1230 "[%s] err: reset failed.\n", __func__);
1231 return rc;
1232 }
1233 genwqe_read_softreset(cd);
1234 }
1235
1236 queue = &cd->queue;
1237 queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG;
1238 queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS;
1239 queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
1240 queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
1241 queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET;
1242 queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP;
1243 queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME;
1244 queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
1245 queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW;
1246
1247 rc = setup_ddcb_queue(cd, queue);
1248 if (rc != 0) {
1249 rc = -ENODEV;
1250 goto err_out;
1251 }
1252
1253 init_waitqueue_head(&cd->queue_waitq);
1254 cd->card_thread = kthread_run(genwqe_card_thread, cd,
1255 GENWQE_DEVNAME "%d_thread",
1256 cd->card_idx);
1257 if (IS_ERR(cd->card_thread)) {
1258 rc = PTR_ERR(cd->card_thread);
1259 cd->card_thread = NULL;
1260 goto stop_free_queue;
1261 }
1262
1263 rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
1264 if (rc)
1265 goto stop_kthread;
1266
1267 /*
1268 * We must have all wait-queues initialized when we enable the
1269 * interrupts. Otherwise we might crash if we get an early
1270 * irq.
1271 */
1272 init_waitqueue_head(&cd->health_waitq);
1273
1274 if (genwqe_is_privileged(cd)) {
1275 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
1276 GENWQE_DEVNAME, cd);
1277 } else {
1278 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
1279 GENWQE_DEVNAME, cd);
1280 }
1281 if (rc < 0) {
1282 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
1283 goto stop_irq_cap;
1284 }
1285
1286 cd->card_state = GENWQE_CARD_USED;
1287 return 0;
1288
1289 stop_irq_cap:
1290 genwqe_reset_interrupt_capability(cd);
1291 stop_kthread:
1292 kthread_stop(cd->card_thread);
1293 cd->card_thread = NULL;
1294 stop_free_queue:
1295 free_ddcb_queue(cd, queue);
1296 err_out:
1297 return rc;
1298 }
1299
1300 /**
1301 * queue_wake_up_all() - Handles fatal error case
1302 *
1303 * The PCI device got unusable and we have to stop all pending
1304 * requests as fast as we can. The code after this must purge the
1305 * DDCBs in question and ensure that all mappings are freed.
1306 */
queue_wake_up_all(struct genwqe_dev * cd)1307 static int queue_wake_up_all(struct genwqe_dev *cd)
1308 {
1309 unsigned int i;
1310 unsigned long flags;
1311 struct ddcb_queue *queue = &cd->queue;
1312
1313 spin_lock_irqsave(&queue->ddcb_lock, flags);
1314
1315 for (i = 0; i < queue->ddcb_max; i++)
1316 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
1317
1318 wake_up_interruptible(&queue->busy_waitq);
1319 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1320
1321 return 0;
1322 }
1323
1324 /**
1325 * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
1326 *
1327 * Relies on the pre-condition that there are no users of the card
1328 * device anymore e.g. with open file-descriptors.
1329 *
1330 * This function must be robust enough to be called twice.
1331 */
genwqe_finish_queue(struct genwqe_dev * cd)1332 int genwqe_finish_queue(struct genwqe_dev *cd)
1333 {
1334 int i, rc = 0, in_flight;
1335 int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;
1336 struct pci_dev *pci_dev = cd->pci_dev;
1337 struct ddcb_queue *queue = &cd->queue;
1338
1339 if (!ddcb_queue_initialized(queue))
1340 return 0;
1341
1342 /* Do not wipe out the error state. */
1343 if (cd->card_state == GENWQE_CARD_USED)
1344 cd->card_state = GENWQE_CARD_UNUSED;
1345
1346 /* Wake up all requests in the DDCB queue such that they
1347 should be removed nicely. */
1348 queue_wake_up_all(cd);
1349
1350 /* We must wait to get rid of the DDCBs in flight */
1351 for (i = 0; i < waitmax; i++) {
1352 in_flight = genwqe_ddcbs_in_flight(cd);
1353
1354 if (in_flight == 0)
1355 break;
1356
1357 dev_dbg(&pci_dev->dev,
1358 " DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
1359 i, waitmax, in_flight);
1360
1361 /*
1362 * Severe severe error situation: The card itself has
1363 * 16 DDCB queues, each queue has e.g. 32 entries,
1364 * each DDBC has a hardware timeout of currently 250
1365 * msec but the PFs have a hardware timeout of 8 sec
1366 * ... so I take something large.
1367 */
1368 msleep(1000);
1369 }
1370 if (i == waitmax) {
1371 dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n",
1372 __func__);
1373 rc = -EIO;
1374 }
1375 return rc;
1376 }
1377
1378 /**
1379 * genwqe_release_service_layer() - Shutdown DDCB queue
1380 * @cd: genwqe device descriptor
1381 *
1382 * This function must be robust enough to be called twice.
1383 */
genwqe_release_service_layer(struct genwqe_dev * cd)1384 int genwqe_release_service_layer(struct genwqe_dev *cd)
1385 {
1386 struct pci_dev *pci_dev = cd->pci_dev;
1387
1388 if (!ddcb_queue_initialized(&cd->queue))
1389 return 1;
1390
1391 free_irq(pci_dev->irq, cd);
1392 genwqe_reset_interrupt_capability(cd);
1393
1394 if (cd->card_thread != NULL) {
1395 kthread_stop(cd->card_thread);
1396 cd->card_thread = NULL;
1397 }
1398
1399 free_ddcb_queue(cd, &cd->queue);
1400 return 0;
1401 }
1402