1 /* Copyright 2012 STEC, Inc.
2 *
3 * This file is licensed under the terms of the 3-clause
4 * BSD License (http://opensource.org/licenses/BSD-3-Clause)
5 * or the GNU GPL-2.0 (http://www.gnu.org/licenses/gpl-2.0.html),
6 * at your option. Both licenses are also available in the LICENSE file
7 * distributed with this project. This file may not be copied, modified,
8 * or distributed except in accordance with those terms.
9 * Gordoni Waidhofer <gwaidhofer@stec-inc.com>
10 * Initial Driver Design!
11 * Thomas Swann <tswann@stec-inc.com>
12 * Interrupt handling.
13 * Ramprasad Chinthekindi <rchinthekindi@stec-inc.com>
14 * biomode implementation.
15 * Akhil Bhansali <abhansali@stec-inc.com>
16 * Added support for DISCARD / FLUSH and FUA.
17 */
18
19 #include <linux/kernel.h>
20 #include <linux/module.h>
21 #include <linux/init.h>
22 #include <linux/pci.h>
23 #include <linux/slab.h>
24 #include <linux/spinlock.h>
25 #include <linux/blkdev.h>
26 #include <linux/sched.h>
27 #include <linux/interrupt.h>
28 #include <linux/compiler.h>
29 #include <linux/workqueue.h>
30 #include <linux/bitops.h>
31 #include <linux/delay.h>
32 #include <linux/time.h>
33 #include <linux/hdreg.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/completion.h>
36 #include <linux/scatterlist.h>
37 #include <linux/version.h>
38 #include <linux/err.h>
39 #include <linux/scatterlist.h>
40 #include <linux/aer.h>
41 #include <linux/ctype.h>
42 #include <linux/wait.h>
43 #include <linux/uio.h>
44 #include <scsi/scsi.h>
45 #include <scsi/sg.h>
46 #include <linux/io.h>
47 #include <linux/uaccess.h>
48 #include <asm/unaligned.h>
49
50 #include "skd_s1120.h"
51
52 static int skd_dbg_level;
53 static int skd_isr_comp_limit = 4;
54
55 enum {
56 STEC_LINK_2_5GTS = 0,
57 STEC_LINK_5GTS = 1,
58 STEC_LINK_8GTS = 2,
59 STEC_LINK_UNKNOWN = 0xFF
60 };
61
62 enum {
63 SKD_FLUSH_INITIALIZER,
64 SKD_FLUSH_ZERO_SIZE_FIRST,
65 SKD_FLUSH_DATA_SECOND,
66 };
67
68 #define SKD_ASSERT(expr) \
69 do { \
70 if (unlikely(!(expr))) { \
71 pr_err("Assertion failed! %s,%s,%s,line=%d\n", \
72 # expr, __FILE__, __func__, __LINE__); \
73 } \
74 } while (0)
75
76 #define DRV_NAME "skd"
77 #define DRV_VERSION "2.2.1"
78 #define DRV_BUILD_ID "0260"
79 #define PFX DRV_NAME ": "
80 #define DRV_BIN_VERSION 0x100
81 #define DRV_VER_COMPL "2.2.1." DRV_BUILD_ID
82
83 MODULE_AUTHOR("bug-reports: support@stec-inc.com");
84 MODULE_LICENSE("Dual BSD/GPL");
85
86 MODULE_DESCRIPTION("STEC s1120 PCIe SSD block driver (b" DRV_BUILD_ID ")");
87 MODULE_VERSION(DRV_VERSION "-" DRV_BUILD_ID);
88
89 #define PCI_VENDOR_ID_STEC 0x1B39
90 #define PCI_DEVICE_ID_S1120 0x0001
91
92 #define SKD_FUA_NV (1 << 1)
93 #define SKD_MINORS_PER_DEVICE 16
94
95 #define SKD_MAX_QUEUE_DEPTH 200u
96
97 #define SKD_PAUSE_TIMEOUT (5 * 1000)
98
99 #define SKD_N_FITMSG_BYTES (512u)
100
101 #define SKD_N_SPECIAL_CONTEXT 32u
102 #define SKD_N_SPECIAL_FITMSG_BYTES (128u)
103
104 /* SG elements are 32 bytes, so we can make this 4096 and still be under the
105 * 128KB limit. That allows 4096*4K = 16M xfer size
106 */
107 #define SKD_N_SG_PER_REQ_DEFAULT 256u
108 #define SKD_N_SG_PER_SPECIAL 256u
109
110 #define SKD_N_COMPLETION_ENTRY 256u
111 #define SKD_N_READ_CAP_BYTES (8u)
112
113 #define SKD_N_INTERNAL_BYTES (512u)
114
115 /* 5 bits of uniqifier, 0xF800 */
116 #define SKD_ID_INCR (0x400)
117 #define SKD_ID_TABLE_MASK (3u << 8u)
118 #define SKD_ID_RW_REQUEST (0u << 8u)
119 #define SKD_ID_INTERNAL (1u << 8u)
120 #define SKD_ID_SPECIAL_REQUEST (2u << 8u)
121 #define SKD_ID_FIT_MSG (3u << 8u)
122 #define SKD_ID_SLOT_MASK 0x00FFu
123 #define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
124
125 #define SKD_N_TIMEOUT_SLOT 4u
126 #define SKD_TIMEOUT_SLOT_MASK 3u
127
128 #define SKD_N_MAX_SECTORS 2048u
129
130 #define SKD_MAX_RETRIES 2u
131
132 #define SKD_TIMER_SECONDS(seconds) (seconds)
133 #define SKD_TIMER_MINUTES(minutes) ((minutes) * (60))
134
135 #define INQ_STD_NBYTES 36
136 #define SKD_DISCARD_CDB_LENGTH 24
137
138 enum skd_drvr_state {
139 SKD_DRVR_STATE_LOAD,
140 SKD_DRVR_STATE_IDLE,
141 SKD_DRVR_STATE_BUSY,
142 SKD_DRVR_STATE_STARTING,
143 SKD_DRVR_STATE_ONLINE,
144 SKD_DRVR_STATE_PAUSING,
145 SKD_DRVR_STATE_PAUSED,
146 SKD_DRVR_STATE_DRAINING_TIMEOUT,
147 SKD_DRVR_STATE_RESTARTING,
148 SKD_DRVR_STATE_RESUMING,
149 SKD_DRVR_STATE_STOPPING,
150 SKD_DRVR_STATE_FAULT,
151 SKD_DRVR_STATE_DISAPPEARED,
152 SKD_DRVR_STATE_PROTOCOL_MISMATCH,
153 SKD_DRVR_STATE_BUSY_ERASE,
154 SKD_DRVR_STATE_BUSY_SANITIZE,
155 SKD_DRVR_STATE_BUSY_IMMINENT,
156 SKD_DRVR_STATE_WAIT_BOOT,
157 SKD_DRVR_STATE_SYNCING,
158 };
159
160 #define SKD_WAIT_BOOT_TIMO SKD_TIMER_SECONDS(90u)
161 #define SKD_STARTING_TIMO SKD_TIMER_SECONDS(8u)
162 #define SKD_RESTARTING_TIMO SKD_TIMER_MINUTES(4u)
163 #define SKD_DRAINING_TIMO SKD_TIMER_SECONDS(6u)
164 #define SKD_BUSY_TIMO SKD_TIMER_MINUTES(20u)
165 #define SKD_STARTED_BUSY_TIMO SKD_TIMER_SECONDS(60u)
166 #define SKD_START_WAIT_SECONDS 90u
167
168 enum skd_req_state {
169 SKD_REQ_STATE_IDLE,
170 SKD_REQ_STATE_SETUP,
171 SKD_REQ_STATE_BUSY,
172 SKD_REQ_STATE_COMPLETED,
173 SKD_REQ_STATE_TIMEOUT,
174 SKD_REQ_STATE_ABORTED,
175 };
176
177 enum skd_fit_msg_state {
178 SKD_MSG_STATE_IDLE,
179 SKD_MSG_STATE_BUSY,
180 };
181
182 enum skd_check_status_action {
183 SKD_CHECK_STATUS_REPORT_GOOD,
184 SKD_CHECK_STATUS_REPORT_SMART_ALERT,
185 SKD_CHECK_STATUS_REQUEUE_REQUEST,
186 SKD_CHECK_STATUS_REPORT_ERROR,
187 SKD_CHECK_STATUS_BUSY_IMMINENT,
188 };
189
190 struct skd_fitmsg_context {
191 enum skd_fit_msg_state state;
192
193 struct skd_fitmsg_context *next;
194
195 u32 id;
196 u16 outstanding;
197
198 u32 length;
199 u32 offset;
200
201 u8 *msg_buf;
202 dma_addr_t mb_dma_address;
203 };
204
205 struct skd_request_context {
206 enum skd_req_state state;
207
208 struct skd_request_context *next;
209
210 u16 id;
211 u32 fitmsg_id;
212
213 struct request *req;
214 u8 flush_cmd;
215 u8 discard_page;
216
217 u32 timeout_stamp;
218 u8 sg_data_dir;
219 struct scatterlist *sg;
220 u32 n_sg;
221 u32 sg_byte_count;
222
223 struct fit_sg_descriptor *sksg_list;
224 dma_addr_t sksg_dma_address;
225
226 struct fit_completion_entry_v1 completion;
227
228 struct fit_comp_error_info err_info;
229
230 };
231 #define SKD_DATA_DIR_HOST_TO_CARD 1
232 #define SKD_DATA_DIR_CARD_TO_HOST 2
233 #define SKD_DATA_DIR_NONE 3 /* especially for DISCARD requests. */
234
235 struct skd_special_context {
236 struct skd_request_context req;
237
238 u8 orphaned;
239
240 void *data_buf;
241 dma_addr_t db_dma_address;
242
243 u8 *msg_buf;
244 dma_addr_t mb_dma_address;
245 };
246
247 struct skd_sg_io {
248 fmode_t mode;
249 void __user *argp;
250
251 struct sg_io_hdr sg;
252
253 u8 cdb[16];
254
255 u32 dxfer_len;
256 u32 iovcnt;
257 struct sg_iovec *iov;
258 struct sg_iovec no_iov_iov;
259
260 struct skd_special_context *skspcl;
261 };
262
263 typedef enum skd_irq_type {
264 SKD_IRQ_LEGACY,
265 SKD_IRQ_MSI,
266 SKD_IRQ_MSIX
267 } skd_irq_type_t;
268
269 #define SKD_MAX_BARS 2
270
271 struct skd_device {
272 volatile void __iomem *mem_map[SKD_MAX_BARS];
273 resource_size_t mem_phys[SKD_MAX_BARS];
274 u32 mem_size[SKD_MAX_BARS];
275
276 skd_irq_type_t irq_type;
277 u32 msix_count;
278 struct skd_msix_entry *msix_entries;
279
280 struct pci_dev *pdev;
281 int pcie_error_reporting_is_enabled;
282
283 spinlock_t lock;
284 struct gendisk *disk;
285 struct request_queue *queue;
286 struct device *class_dev;
287 int gendisk_on;
288 int sync_done;
289
290 atomic_t device_count;
291 u32 devno;
292 u32 major;
293 char name[32];
294 char isr_name[30];
295
296 enum skd_drvr_state state;
297 u32 drive_state;
298
299 u32 in_flight;
300 u32 cur_max_queue_depth;
301 u32 queue_low_water_mark;
302 u32 dev_max_queue_depth;
303
304 u32 num_fitmsg_context;
305 u32 num_req_context;
306
307 u32 timeout_slot[SKD_N_TIMEOUT_SLOT];
308 u32 timeout_stamp;
309 struct skd_fitmsg_context *skmsg_free_list;
310 struct skd_fitmsg_context *skmsg_table;
311
312 struct skd_request_context *skreq_free_list;
313 struct skd_request_context *skreq_table;
314
315 struct skd_special_context *skspcl_free_list;
316 struct skd_special_context *skspcl_table;
317
318 struct skd_special_context internal_skspcl;
319 u32 read_cap_blocksize;
320 u32 read_cap_last_lba;
321 int read_cap_is_valid;
322 int inquiry_is_valid;
323 u8 inq_serial_num[13]; /*12 chars plus null term */
324 u8 id_str[80]; /* holds a composite name (pci + sernum) */
325
326 u8 skcomp_cycle;
327 u32 skcomp_ix;
328 struct fit_completion_entry_v1 *skcomp_table;
329 struct fit_comp_error_info *skerr_table;
330 dma_addr_t cq_dma_address;
331
332 wait_queue_head_t waitq;
333
334 struct timer_list timer;
335 u32 timer_countdown;
336 u32 timer_substate;
337
338 int n_special;
339 int sgs_per_request;
340 u32 last_mtd;
341
342 u32 proto_ver;
343
344 int dbg_level;
345 u32 connect_time_stamp;
346 int connect_retries;
347 #define SKD_MAX_CONNECT_RETRIES 16
348 u32 drive_jiffies;
349
350 u32 timo_slot;
351
352
353 struct work_struct completion_worker;
354 };
355
356 #define SKD_WRITEL(DEV, VAL, OFF) skd_reg_write32(DEV, VAL, OFF)
357 #define SKD_READL(DEV, OFF) skd_reg_read32(DEV, OFF)
358 #define SKD_WRITEQ(DEV, VAL, OFF) skd_reg_write64(DEV, VAL, OFF)
359
skd_reg_read32(struct skd_device * skdev,u32 offset)360 static inline u32 skd_reg_read32(struct skd_device *skdev, u32 offset)
361 {
362 u32 val;
363
364 if (likely(skdev->dbg_level < 2))
365 return readl(skdev->mem_map[1] + offset);
366 else {
367 barrier();
368 val = readl(skdev->mem_map[1] + offset);
369 barrier();
370 pr_debug("%s:%s:%d offset %x = %x\n",
371 skdev->name, __func__, __LINE__, offset, val);
372 return val;
373 }
374
375 }
376
skd_reg_write32(struct skd_device * skdev,u32 val,u32 offset)377 static inline void skd_reg_write32(struct skd_device *skdev, u32 val,
378 u32 offset)
379 {
380 if (likely(skdev->dbg_level < 2)) {
381 writel(val, skdev->mem_map[1] + offset);
382 barrier();
383 } else {
384 barrier();
385 writel(val, skdev->mem_map[1] + offset);
386 barrier();
387 pr_debug("%s:%s:%d offset %x = %x\n",
388 skdev->name, __func__, __LINE__, offset, val);
389 }
390 }
391
skd_reg_write64(struct skd_device * skdev,u64 val,u32 offset)392 static inline void skd_reg_write64(struct skd_device *skdev, u64 val,
393 u32 offset)
394 {
395 if (likely(skdev->dbg_level < 2)) {
396 writeq(val, skdev->mem_map[1] + offset);
397 barrier();
398 } else {
399 barrier();
400 writeq(val, skdev->mem_map[1] + offset);
401 barrier();
402 pr_debug("%s:%s:%d offset %x = %016llx\n",
403 skdev->name, __func__, __LINE__, offset, val);
404 }
405 }
406
407
408 #define SKD_IRQ_DEFAULT SKD_IRQ_MSI
409 static int skd_isr_type = SKD_IRQ_DEFAULT;
410
411 module_param(skd_isr_type, int, 0444);
412 MODULE_PARM_DESC(skd_isr_type, "Interrupt type capability."
413 " (0==legacy, 1==MSI, 2==MSI-X, default==1)");
414
415 #define SKD_MAX_REQ_PER_MSG_DEFAULT 1
416 static int skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
417
418 module_param(skd_max_req_per_msg, int, 0444);
419 MODULE_PARM_DESC(skd_max_req_per_msg,
420 "Maximum SCSI requests packed in a single message."
421 " (1-14, default==1)");
422
423 #define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
424 #define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
425 static int skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
426
427 module_param(skd_max_queue_depth, int, 0444);
428 MODULE_PARM_DESC(skd_max_queue_depth,
429 "Maximum SCSI requests issued to s1120."
430 " (1-200, default==" SKD_MAX_QUEUE_DEPTH_DEFAULT_STR ")");
431
432 static int skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
433 module_param(skd_sgs_per_request, int, 0444);
434 MODULE_PARM_DESC(skd_sgs_per_request,
435 "Maximum SG elements per block request."
436 " (1-4096, default==256)");
437
438 static int skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
439 module_param(skd_max_pass_thru, int, 0444);
440 MODULE_PARM_DESC(skd_max_pass_thru,
441 "Maximum SCSI pass-thru at a time." " (1-50, default==32)");
442
443 module_param(skd_dbg_level, int, 0444);
444 MODULE_PARM_DESC(skd_dbg_level, "s1120 debug level (0,1,2)");
445
446 module_param(skd_isr_comp_limit, int, 0444);
447 MODULE_PARM_DESC(skd_isr_comp_limit, "s1120 isr comp limit (0=none) default=4");
448
449 /* Major device number dynamically assigned. */
450 static u32 skd_major;
451
452 static void skd_destruct(struct skd_device *skdev);
453 static const struct block_device_operations skd_blockdev_ops;
454 static void skd_send_fitmsg(struct skd_device *skdev,
455 struct skd_fitmsg_context *skmsg);
456 static void skd_send_special_fitmsg(struct skd_device *skdev,
457 struct skd_special_context *skspcl);
458 static void skd_request_fn(struct request_queue *rq);
459 static void skd_end_request(struct skd_device *skdev,
460 struct skd_request_context *skreq, int error);
461 static int skd_preop_sg_list(struct skd_device *skdev,
462 struct skd_request_context *skreq);
463 static void skd_postop_sg_list(struct skd_device *skdev,
464 struct skd_request_context *skreq);
465
466 static void skd_restart_device(struct skd_device *skdev);
467 static int skd_quiesce_dev(struct skd_device *skdev);
468 static int skd_unquiesce_dev(struct skd_device *skdev);
469 static void skd_release_special(struct skd_device *skdev,
470 struct skd_special_context *skspcl);
471 static void skd_disable_interrupts(struct skd_device *skdev);
472 static void skd_isr_fwstate(struct skd_device *skdev);
473 static void skd_recover_requests(struct skd_device *skdev, int requeue);
474 static void skd_soft_reset(struct skd_device *skdev);
475
476 static const char *skd_name(struct skd_device *skdev);
477 const char *skd_drive_state_to_str(int state);
478 const char *skd_skdev_state_to_str(enum skd_drvr_state state);
479 static void skd_log_skdev(struct skd_device *skdev, const char *event);
480 static void skd_log_skmsg(struct skd_device *skdev,
481 struct skd_fitmsg_context *skmsg, const char *event);
482 static void skd_log_skreq(struct skd_device *skdev,
483 struct skd_request_context *skreq, const char *event);
484
485 /*
486 *****************************************************************************
487 * READ/WRITE REQUESTS
488 *****************************************************************************
489 */
skd_fail_all_pending(struct skd_device * skdev)490 static void skd_fail_all_pending(struct skd_device *skdev)
491 {
492 struct request_queue *q = skdev->queue;
493 struct request *req;
494
495 for (;; ) {
496 req = blk_peek_request(q);
497 if (req == NULL)
498 break;
499 blk_start_request(req);
500 __blk_end_request_all(req, -EIO);
501 }
502 }
503
504 static void
skd_prep_rw_cdb(struct skd_scsi_request * scsi_req,int data_dir,unsigned lba,unsigned count)505 skd_prep_rw_cdb(struct skd_scsi_request *scsi_req,
506 int data_dir, unsigned lba,
507 unsigned count)
508 {
509 if (data_dir == READ)
510 scsi_req->cdb[0] = 0x28;
511 else
512 scsi_req->cdb[0] = 0x2a;
513
514 scsi_req->cdb[1] = 0;
515 scsi_req->cdb[2] = (lba & 0xff000000) >> 24;
516 scsi_req->cdb[3] = (lba & 0xff0000) >> 16;
517 scsi_req->cdb[4] = (lba & 0xff00) >> 8;
518 scsi_req->cdb[5] = (lba & 0xff);
519 scsi_req->cdb[6] = 0;
520 scsi_req->cdb[7] = (count & 0xff00) >> 8;
521 scsi_req->cdb[8] = count & 0xff;
522 scsi_req->cdb[9] = 0;
523 }
524
525 static void
skd_prep_zerosize_flush_cdb(struct skd_scsi_request * scsi_req,struct skd_request_context * skreq)526 skd_prep_zerosize_flush_cdb(struct skd_scsi_request *scsi_req,
527 struct skd_request_context *skreq)
528 {
529 skreq->flush_cmd = 1;
530
531 scsi_req->cdb[0] = 0x35;
532 scsi_req->cdb[1] = 0;
533 scsi_req->cdb[2] = 0;
534 scsi_req->cdb[3] = 0;
535 scsi_req->cdb[4] = 0;
536 scsi_req->cdb[5] = 0;
537 scsi_req->cdb[6] = 0;
538 scsi_req->cdb[7] = 0;
539 scsi_req->cdb[8] = 0;
540 scsi_req->cdb[9] = 0;
541 }
542
543 static void
skd_prep_discard_cdb(struct skd_scsi_request * scsi_req,struct skd_request_context * skreq,struct page * page,u32 lba,u32 count)544 skd_prep_discard_cdb(struct skd_scsi_request *scsi_req,
545 struct skd_request_context *skreq,
546 struct page *page,
547 u32 lba, u32 count)
548 {
549 char *buf;
550 unsigned long len;
551 struct request *req;
552
553 buf = page_address(page);
554 len = SKD_DISCARD_CDB_LENGTH;
555
556 scsi_req->cdb[0] = UNMAP;
557 scsi_req->cdb[8] = len;
558
559 put_unaligned_be16(6 + 16, &buf[0]);
560 put_unaligned_be16(16, &buf[2]);
561 put_unaligned_be64(lba, &buf[8]);
562 put_unaligned_be32(count, &buf[16]);
563
564 req = skreq->req;
565 blk_add_request_payload(req, page, len);
566 }
567
568 static void skd_request_fn_not_online(struct request_queue *q);
569
skd_request_fn(struct request_queue * q)570 static void skd_request_fn(struct request_queue *q)
571 {
572 struct skd_device *skdev = q->queuedata;
573 struct skd_fitmsg_context *skmsg = NULL;
574 struct fit_msg_hdr *fmh = NULL;
575 struct skd_request_context *skreq;
576 struct request *req = NULL;
577 struct skd_scsi_request *scsi_req;
578 struct page *page;
579 unsigned long io_flags;
580 int error;
581 u32 lba;
582 u32 count;
583 int data_dir;
584 u32 be_lba;
585 u32 be_count;
586 u64 be_dmaa;
587 u64 cmdctxt;
588 u32 timo_slot;
589 void *cmd_ptr;
590 int flush, fua;
591
592 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
593 skd_request_fn_not_online(q);
594 return;
595 }
596
597 if (blk_queue_stopped(skdev->queue)) {
598 if (skdev->skmsg_free_list == NULL ||
599 skdev->skreq_free_list == NULL ||
600 skdev->in_flight >= skdev->queue_low_water_mark)
601 /* There is still some kind of shortage */
602 return;
603
604 queue_flag_clear(QUEUE_FLAG_STOPPED, skdev->queue);
605 }
606
607 /*
608 * Stop conditions:
609 * - There are no more native requests
610 * - There are already the maximum number of requests in progress
611 * - There are no more skd_request_context entries
612 * - There are no more FIT msg buffers
613 */
614 for (;; ) {
615
616 flush = fua = 0;
617
618 req = blk_peek_request(q);
619
620 /* Are there any native requests to start? */
621 if (req == NULL)
622 break;
623
624 lba = (u32)blk_rq_pos(req);
625 count = blk_rq_sectors(req);
626 data_dir = rq_data_dir(req);
627 io_flags = req->cmd_flags;
628
629 if (io_flags & REQ_FLUSH)
630 flush++;
631
632 if (io_flags & REQ_FUA)
633 fua++;
634
635 pr_debug("%s:%s:%d new req=%p lba=%u(0x%x) "
636 "count=%u(0x%x) dir=%d\n",
637 skdev->name, __func__, __LINE__,
638 req, lba, lba, count, count, data_dir);
639
640 /* At this point we know there is a request */
641
642 /* Are too many requets already in progress? */
643 if (skdev->in_flight >= skdev->cur_max_queue_depth) {
644 pr_debug("%s:%s:%d qdepth %d, limit %d\n",
645 skdev->name, __func__, __LINE__,
646 skdev->in_flight, skdev->cur_max_queue_depth);
647 break;
648 }
649
650 /* Is a skd_request_context available? */
651 skreq = skdev->skreq_free_list;
652 if (skreq == NULL) {
653 pr_debug("%s:%s:%d Out of req=%p\n",
654 skdev->name, __func__, __LINE__, q);
655 break;
656 }
657 SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);
658 SKD_ASSERT((skreq->id & SKD_ID_INCR) == 0);
659
660 /* Now we check to see if we can get a fit msg */
661 if (skmsg == NULL) {
662 if (skdev->skmsg_free_list == NULL) {
663 pr_debug("%s:%s:%d Out of msg\n",
664 skdev->name, __func__, __LINE__);
665 break;
666 }
667 }
668
669 skreq->flush_cmd = 0;
670 skreq->n_sg = 0;
671 skreq->sg_byte_count = 0;
672 skreq->discard_page = 0;
673
674 /*
675 * OK to now dequeue request from q.
676 *
677 * At this point we are comitted to either start or reject
678 * the native request. Note that skd_request_context is
679 * available but is still at the head of the free list.
680 */
681 blk_start_request(req);
682 skreq->req = req;
683 skreq->fitmsg_id = 0;
684
685 /* Either a FIT msg is in progress or we have to start one. */
686 if (skmsg == NULL) {
687 /* Are there any FIT msg buffers available? */
688 skmsg = skdev->skmsg_free_list;
689 if (skmsg == NULL) {
690 pr_debug("%s:%s:%d Out of msg skdev=%p\n",
691 skdev->name, __func__, __LINE__,
692 skdev);
693 break;
694 }
695 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_IDLE);
696 SKD_ASSERT((skmsg->id & SKD_ID_INCR) == 0);
697
698 skdev->skmsg_free_list = skmsg->next;
699
700 skmsg->state = SKD_MSG_STATE_BUSY;
701 skmsg->id += SKD_ID_INCR;
702
703 /* Initialize the FIT msg header */
704 fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
705 memset(fmh, 0, sizeof(*fmh));
706 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
707 skmsg->length = sizeof(*fmh);
708 }
709
710 skreq->fitmsg_id = skmsg->id;
711
712 /*
713 * Note that a FIT msg may have just been started
714 * but contains no SoFIT requests yet.
715 */
716
717 /*
718 * Transcode the request, checking as we go. The outcome of
719 * the transcoding is represented by the error variable.
720 */
721 cmd_ptr = &skmsg->msg_buf[skmsg->length];
722 memset(cmd_ptr, 0, 32);
723
724 be_lba = cpu_to_be32(lba);
725 be_count = cpu_to_be32(count);
726 be_dmaa = cpu_to_be64((u64)skreq->sksg_dma_address);
727 cmdctxt = skreq->id + SKD_ID_INCR;
728
729 scsi_req = cmd_ptr;
730 scsi_req->hdr.tag = cmdctxt;
731 scsi_req->hdr.sg_list_dma_address = be_dmaa;
732
733 if (data_dir == READ)
734 skreq->sg_data_dir = SKD_DATA_DIR_CARD_TO_HOST;
735 else
736 skreq->sg_data_dir = SKD_DATA_DIR_HOST_TO_CARD;
737
738 if (io_flags & REQ_DISCARD) {
739 page = alloc_page(GFP_ATOMIC | __GFP_ZERO);
740 if (!page) {
741 pr_err("request_fn:Page allocation failed.\n");
742 skd_end_request(skdev, skreq, -ENOMEM);
743 break;
744 }
745 skreq->discard_page = 1;
746 req->completion_data = page;
747 skd_prep_discard_cdb(scsi_req, skreq, page, lba, count);
748
749 } else if (flush == SKD_FLUSH_ZERO_SIZE_FIRST) {
750 skd_prep_zerosize_flush_cdb(scsi_req, skreq);
751 SKD_ASSERT(skreq->flush_cmd == 1);
752
753 } else {
754 skd_prep_rw_cdb(scsi_req, data_dir, lba, count);
755 }
756
757 if (fua)
758 scsi_req->cdb[1] |= SKD_FUA_NV;
759
760 if (!req->bio)
761 goto skip_sg;
762
763 error = skd_preop_sg_list(skdev, skreq);
764
765 if (error != 0) {
766 /*
767 * Complete the native request with error.
768 * Note that the request context is still at the
769 * head of the free list, and that the SoFIT request
770 * was encoded into the FIT msg buffer but the FIT
771 * msg length has not been updated. In short, the
772 * only resource that has been allocated but might
773 * not be used is that the FIT msg could be empty.
774 */
775 pr_debug("%s:%s:%d error Out\n",
776 skdev->name, __func__, __LINE__);
777 skd_end_request(skdev, skreq, error);
778 continue;
779 }
780
781 skip_sg:
782 scsi_req->hdr.sg_list_len_bytes =
783 cpu_to_be32(skreq->sg_byte_count);
784
785 /* Complete resource allocations. */
786 skdev->skreq_free_list = skreq->next;
787 skreq->state = SKD_REQ_STATE_BUSY;
788 skreq->id += SKD_ID_INCR;
789
790 skmsg->length += sizeof(struct skd_scsi_request);
791 fmh->num_protocol_cmds_coalesced++;
792
793 /*
794 * Update the active request counts.
795 * Capture the timeout timestamp.
796 */
797 skreq->timeout_stamp = skdev->timeout_stamp;
798 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
799 skdev->timeout_slot[timo_slot]++;
800 skdev->in_flight++;
801 pr_debug("%s:%s:%d req=0x%x busy=%d\n",
802 skdev->name, __func__, __LINE__,
803 skreq->id, skdev->in_flight);
804
805 /*
806 * If the FIT msg buffer is full send it.
807 */
808 if (skmsg->length >= SKD_N_FITMSG_BYTES ||
809 fmh->num_protocol_cmds_coalesced >= skd_max_req_per_msg) {
810 skd_send_fitmsg(skdev, skmsg);
811 skmsg = NULL;
812 fmh = NULL;
813 }
814 }
815
816 /*
817 * Is a FIT msg in progress? If it is empty put the buffer back
818 * on the free list. If it is non-empty send what we got.
819 * This minimizes latency when there are fewer requests than
820 * what fits in a FIT msg.
821 */
822 if (skmsg != NULL) {
823 /* Bigger than just a FIT msg header? */
824 if (skmsg->length > sizeof(struct fit_msg_hdr)) {
825 pr_debug("%s:%s:%d sending msg=%p, len %d\n",
826 skdev->name, __func__, __LINE__,
827 skmsg, skmsg->length);
828 skd_send_fitmsg(skdev, skmsg);
829 } else {
830 /*
831 * The FIT msg is empty. It means we got started
832 * on the msg, but the requests were rejected.
833 */
834 skmsg->state = SKD_MSG_STATE_IDLE;
835 skmsg->id += SKD_ID_INCR;
836 skmsg->next = skdev->skmsg_free_list;
837 skdev->skmsg_free_list = skmsg;
838 }
839 skmsg = NULL;
840 fmh = NULL;
841 }
842
843 /*
844 * If req is non-NULL it means there is something to do but
845 * we are out of a resource.
846 */
847 if (req)
848 blk_stop_queue(skdev->queue);
849 }
850
skd_end_request(struct skd_device * skdev,struct skd_request_context * skreq,int error)851 static void skd_end_request(struct skd_device *skdev,
852 struct skd_request_context *skreq, int error)
853 {
854 struct request *req = skreq->req;
855 unsigned int io_flags = req->cmd_flags;
856
857 if ((io_flags & REQ_DISCARD) &&
858 (skreq->discard_page == 1)) {
859 pr_debug("%s:%s:%d, free the page!",
860 skdev->name, __func__, __LINE__);
861 __free_page(req->completion_data);
862 }
863
864 if (unlikely(error)) {
865 struct request *req = skreq->req;
866 char *cmd = (rq_data_dir(req) == READ) ? "read" : "write";
867 u32 lba = (u32)blk_rq_pos(req);
868 u32 count = blk_rq_sectors(req);
869
870 pr_err("(%s): Error cmd=%s sect=%u count=%u id=0x%x\n",
871 skd_name(skdev), cmd, lba, count, skreq->id);
872 } else
873 pr_debug("%s:%s:%d id=0x%x error=%d\n",
874 skdev->name, __func__, __LINE__, skreq->id, error);
875
876 __blk_end_request_all(skreq->req, error);
877 }
878
skd_preop_sg_list(struct skd_device * skdev,struct skd_request_context * skreq)879 static int skd_preop_sg_list(struct skd_device *skdev,
880 struct skd_request_context *skreq)
881 {
882 struct request *req = skreq->req;
883 int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
884 int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
885 struct scatterlist *sg = &skreq->sg[0];
886 int n_sg;
887 int i;
888
889 skreq->sg_byte_count = 0;
890
891 /* SKD_ASSERT(skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD ||
892 skreq->sg_data_dir == SKD_DATA_DIR_CARD_TO_HOST); */
893
894 n_sg = blk_rq_map_sg(skdev->queue, req, sg);
895 if (n_sg <= 0)
896 return -EINVAL;
897
898 /*
899 * Map scatterlist to PCI bus addresses.
900 * Note PCI might change the number of entries.
901 */
902 n_sg = pci_map_sg(skdev->pdev, sg, n_sg, pci_dir);
903 if (n_sg <= 0)
904 return -EINVAL;
905
906 SKD_ASSERT(n_sg <= skdev->sgs_per_request);
907
908 skreq->n_sg = n_sg;
909
910 for (i = 0; i < n_sg; i++) {
911 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
912 u32 cnt = sg_dma_len(&sg[i]);
913 uint64_t dma_addr = sg_dma_address(&sg[i]);
914
915 sgd->control = FIT_SGD_CONTROL_NOT_LAST;
916 sgd->byte_count = cnt;
917 skreq->sg_byte_count += cnt;
918 sgd->host_side_addr = dma_addr;
919 sgd->dev_side_addr = 0;
920 }
921
922 skreq->sksg_list[n_sg - 1].next_desc_ptr = 0LL;
923 skreq->sksg_list[n_sg - 1].control = FIT_SGD_CONTROL_LAST;
924
925 if (unlikely(skdev->dbg_level > 1)) {
926 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
927 skdev->name, __func__, __LINE__,
928 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
929 for (i = 0; i < n_sg; i++) {
930 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
931 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
932 "addr=0x%llx next=0x%llx\n",
933 skdev->name, __func__, __LINE__,
934 i, sgd->byte_count, sgd->control,
935 sgd->host_side_addr, sgd->next_desc_ptr);
936 }
937 }
938
939 return 0;
940 }
941
skd_postop_sg_list(struct skd_device * skdev,struct skd_request_context * skreq)942 static void skd_postop_sg_list(struct skd_device *skdev,
943 struct skd_request_context *skreq)
944 {
945 int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
946 int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
947
948 /*
949 * restore the next ptr for next IO request so we
950 * don't have to set it every time.
951 */
952 skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr =
953 skreq->sksg_dma_address +
954 ((skreq->n_sg) * sizeof(struct fit_sg_descriptor));
955 pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, pci_dir);
956 }
957
skd_request_fn_not_online(struct request_queue * q)958 static void skd_request_fn_not_online(struct request_queue *q)
959 {
960 struct skd_device *skdev = q->queuedata;
961 int error;
962
963 SKD_ASSERT(skdev->state != SKD_DRVR_STATE_ONLINE);
964
965 skd_log_skdev(skdev, "req_not_online");
966 switch (skdev->state) {
967 case SKD_DRVR_STATE_PAUSING:
968 case SKD_DRVR_STATE_PAUSED:
969 case SKD_DRVR_STATE_STARTING:
970 case SKD_DRVR_STATE_RESTARTING:
971 case SKD_DRVR_STATE_WAIT_BOOT:
972 /* In case of starting, we haven't started the queue,
973 * so we can't get here... but requests are
974 * possibly hanging out waiting for us because we
975 * reported the dev/skd0 already. They'll wait
976 * forever if connect doesn't complete.
977 * What to do??? delay dev/skd0 ??
978 */
979 case SKD_DRVR_STATE_BUSY:
980 case SKD_DRVR_STATE_BUSY_IMMINENT:
981 case SKD_DRVR_STATE_BUSY_ERASE:
982 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
983 return;
984
985 case SKD_DRVR_STATE_BUSY_SANITIZE:
986 case SKD_DRVR_STATE_STOPPING:
987 case SKD_DRVR_STATE_SYNCING:
988 case SKD_DRVR_STATE_FAULT:
989 case SKD_DRVR_STATE_DISAPPEARED:
990 default:
991 error = -EIO;
992 break;
993 }
994
995 /* If we get here, terminate all pending block requeusts
996 * with EIO and any scsi pass thru with appropriate sense
997 */
998
999 skd_fail_all_pending(skdev);
1000 }
1001
1002 /*
1003 *****************************************************************************
1004 * TIMER
1005 *****************************************************************************
1006 */
1007
1008 static void skd_timer_tick_not_online(struct skd_device *skdev);
1009
skd_timer_tick(ulong arg)1010 static void skd_timer_tick(ulong arg)
1011 {
1012 struct skd_device *skdev = (struct skd_device *)arg;
1013
1014 u32 timo_slot;
1015 u32 overdue_timestamp;
1016 unsigned long reqflags;
1017 u32 state;
1018
1019 if (skdev->state == SKD_DRVR_STATE_FAULT)
1020 /* The driver has declared fault, and we want it to
1021 * stay that way until driver is reloaded.
1022 */
1023 return;
1024
1025 spin_lock_irqsave(&skdev->lock, reqflags);
1026
1027 state = SKD_READL(skdev, FIT_STATUS);
1028 state &= FIT_SR_DRIVE_STATE_MASK;
1029 if (state != skdev->drive_state)
1030 skd_isr_fwstate(skdev);
1031
1032 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
1033 skd_timer_tick_not_online(skdev);
1034 goto timer_func_out;
1035 }
1036 skdev->timeout_stamp++;
1037 timo_slot = skdev->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
1038
1039 /*
1040 * All requests that happened during the previous use of
1041 * this slot should be done by now. The previous use was
1042 * over 7 seconds ago.
1043 */
1044 if (skdev->timeout_slot[timo_slot] == 0)
1045 goto timer_func_out;
1046
1047 /* Something is overdue */
1048 overdue_timestamp = skdev->timeout_stamp - SKD_N_TIMEOUT_SLOT;
1049
1050 pr_debug("%s:%s:%d found %d timeouts, draining busy=%d\n",
1051 skdev->name, __func__, __LINE__,
1052 skdev->timeout_slot[timo_slot], skdev->in_flight);
1053 pr_err("(%s): Overdue IOs (%d), busy %d\n",
1054 skd_name(skdev), skdev->timeout_slot[timo_slot],
1055 skdev->in_flight);
1056
1057 skdev->timer_countdown = SKD_DRAINING_TIMO;
1058 skdev->state = SKD_DRVR_STATE_DRAINING_TIMEOUT;
1059 skdev->timo_slot = timo_slot;
1060 blk_stop_queue(skdev->queue);
1061
1062 timer_func_out:
1063 mod_timer(&skdev->timer, (jiffies + HZ));
1064
1065 spin_unlock_irqrestore(&skdev->lock, reqflags);
1066 }
1067
skd_timer_tick_not_online(struct skd_device * skdev)1068 static void skd_timer_tick_not_online(struct skd_device *skdev)
1069 {
1070 switch (skdev->state) {
1071 case SKD_DRVR_STATE_IDLE:
1072 case SKD_DRVR_STATE_LOAD:
1073 break;
1074 case SKD_DRVR_STATE_BUSY_SANITIZE:
1075 pr_debug("%s:%s:%d drive busy sanitize[%x], driver[%x]\n",
1076 skdev->name, __func__, __LINE__,
1077 skdev->drive_state, skdev->state);
1078 /* If we've been in sanitize for 3 seconds, we figure we're not
1079 * going to get anymore completions, so recover requests now
1080 */
1081 if (skdev->timer_countdown > 0) {
1082 skdev->timer_countdown--;
1083 return;
1084 }
1085 skd_recover_requests(skdev, 0);
1086 break;
1087
1088 case SKD_DRVR_STATE_BUSY:
1089 case SKD_DRVR_STATE_BUSY_IMMINENT:
1090 case SKD_DRVR_STATE_BUSY_ERASE:
1091 pr_debug("%s:%s:%d busy[%x], countdown=%d\n",
1092 skdev->name, __func__, __LINE__,
1093 skdev->state, skdev->timer_countdown);
1094 if (skdev->timer_countdown > 0) {
1095 skdev->timer_countdown--;
1096 return;
1097 }
1098 pr_debug("%s:%s:%d busy[%x], timedout=%d, restarting device.",
1099 skdev->name, __func__, __LINE__,
1100 skdev->state, skdev->timer_countdown);
1101 skd_restart_device(skdev);
1102 break;
1103
1104 case SKD_DRVR_STATE_WAIT_BOOT:
1105 case SKD_DRVR_STATE_STARTING:
1106 if (skdev->timer_countdown > 0) {
1107 skdev->timer_countdown--;
1108 return;
1109 }
1110 /* For now, we fault the drive. Could attempt resets to
1111 * revcover at some point. */
1112 skdev->state = SKD_DRVR_STATE_FAULT;
1113
1114 pr_err("(%s): DriveFault Connect Timeout (%x)\n",
1115 skd_name(skdev), skdev->drive_state);
1116
1117 /*start the queue so we can respond with error to requests */
1118 /* wakeup anyone waiting for startup complete */
1119 blk_start_queue(skdev->queue);
1120 skdev->gendisk_on = -1;
1121 wake_up_interruptible(&skdev->waitq);
1122 break;
1123
1124 case SKD_DRVR_STATE_ONLINE:
1125 /* shouldn't get here. */
1126 break;
1127
1128 case SKD_DRVR_STATE_PAUSING:
1129 case SKD_DRVR_STATE_PAUSED:
1130 break;
1131
1132 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
1133 pr_debug("%s:%s:%d "
1134 "draining busy [%d] tick[%d] qdb[%d] tmls[%d]\n",
1135 skdev->name, __func__, __LINE__,
1136 skdev->timo_slot,
1137 skdev->timer_countdown,
1138 skdev->in_flight,
1139 skdev->timeout_slot[skdev->timo_slot]);
1140 /* if the slot has cleared we can let the I/O continue */
1141 if (skdev->timeout_slot[skdev->timo_slot] == 0) {
1142 pr_debug("%s:%s:%d Slot drained, starting queue.\n",
1143 skdev->name, __func__, __LINE__);
1144 skdev->state = SKD_DRVR_STATE_ONLINE;
1145 blk_start_queue(skdev->queue);
1146 return;
1147 }
1148 if (skdev->timer_countdown > 0) {
1149 skdev->timer_countdown--;
1150 return;
1151 }
1152 skd_restart_device(skdev);
1153 break;
1154
1155 case SKD_DRVR_STATE_RESTARTING:
1156 if (skdev->timer_countdown > 0) {
1157 skdev->timer_countdown--;
1158 return;
1159 }
1160 /* For now, we fault the drive. Could attempt resets to
1161 * revcover at some point. */
1162 skdev->state = SKD_DRVR_STATE_FAULT;
1163 pr_err("(%s): DriveFault Reconnect Timeout (%x)\n",
1164 skd_name(skdev), skdev->drive_state);
1165
1166 /*
1167 * Recovering does two things:
1168 * 1. completes IO with error
1169 * 2. reclaims dma resources
1170 * When is it safe to recover requests?
1171 * - if the drive state is faulted
1172 * - if the state is still soft reset after out timeout
1173 * - if the drive registers are dead (state = FF)
1174 * If it is "unsafe", we still need to recover, so we will
1175 * disable pci bus mastering and disable our interrupts.
1176 */
1177
1178 if ((skdev->drive_state == FIT_SR_DRIVE_SOFT_RESET) ||
1179 (skdev->drive_state == FIT_SR_DRIVE_FAULT) ||
1180 (skdev->drive_state == FIT_SR_DRIVE_STATE_MASK))
1181 /* It never came out of soft reset. Try to
1182 * recover the requests and then let them
1183 * fail. This is to mitigate hung processes. */
1184 skd_recover_requests(skdev, 0);
1185 else {
1186 pr_err("(%s): Disable BusMaster (%x)\n",
1187 skd_name(skdev), skdev->drive_state);
1188 pci_disable_device(skdev->pdev);
1189 skd_disable_interrupts(skdev);
1190 skd_recover_requests(skdev, 0);
1191 }
1192
1193 /*start the queue so we can respond with error to requests */
1194 /* wakeup anyone waiting for startup complete */
1195 blk_start_queue(skdev->queue);
1196 skdev->gendisk_on = -1;
1197 wake_up_interruptible(&skdev->waitq);
1198 break;
1199
1200 case SKD_DRVR_STATE_RESUMING:
1201 case SKD_DRVR_STATE_STOPPING:
1202 case SKD_DRVR_STATE_SYNCING:
1203 case SKD_DRVR_STATE_FAULT:
1204 case SKD_DRVR_STATE_DISAPPEARED:
1205 default:
1206 break;
1207 }
1208 }
1209
skd_start_timer(struct skd_device * skdev)1210 static int skd_start_timer(struct skd_device *skdev)
1211 {
1212 int rc;
1213
1214 init_timer(&skdev->timer);
1215 setup_timer(&skdev->timer, skd_timer_tick, (ulong)skdev);
1216
1217 rc = mod_timer(&skdev->timer, (jiffies + HZ));
1218 if (rc)
1219 pr_err("%s: failed to start timer %d\n",
1220 __func__, rc);
1221 return rc;
1222 }
1223
skd_kill_timer(struct skd_device * skdev)1224 static void skd_kill_timer(struct skd_device *skdev)
1225 {
1226 del_timer_sync(&skdev->timer);
1227 }
1228
1229 /*
1230 *****************************************************************************
1231 * IOCTL
1232 *****************************************************************************
1233 */
1234 static int skd_ioctl_sg_io(struct skd_device *skdev,
1235 fmode_t mode, void __user *argp);
1236 static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1237 struct skd_sg_io *sksgio);
1238 static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1239 struct skd_sg_io *sksgio);
1240 static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1241 struct skd_sg_io *sksgio);
1242 static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1243 struct skd_sg_io *sksgio, int dxfer_dir);
1244 static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1245 struct skd_sg_io *sksgio);
1246 static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio);
1247 static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1248 struct skd_sg_io *sksgio);
1249 static int skd_sg_io_put_status(struct skd_device *skdev,
1250 struct skd_sg_io *sksgio);
1251
1252 static void skd_complete_special(struct skd_device *skdev,
1253 volatile struct fit_completion_entry_v1
1254 *skcomp,
1255 volatile struct fit_comp_error_info *skerr,
1256 struct skd_special_context *skspcl);
1257
skd_bdev_ioctl(struct block_device * bdev,fmode_t mode,uint cmd_in,ulong arg)1258 static int skd_bdev_ioctl(struct block_device *bdev, fmode_t mode,
1259 uint cmd_in, ulong arg)
1260 {
1261 int rc = 0;
1262 struct gendisk *disk = bdev->bd_disk;
1263 struct skd_device *skdev = disk->private_data;
1264 void __user *p = (void *)arg;
1265
1266 pr_debug("%s:%s:%d %s: CMD[%s] ioctl mode 0x%x, cmd 0x%x arg %0lx\n",
1267 skdev->name, __func__, __LINE__,
1268 disk->disk_name, current->comm, mode, cmd_in, arg);
1269
1270 if (!capable(CAP_SYS_ADMIN))
1271 return -EPERM;
1272
1273 switch (cmd_in) {
1274 case SG_SET_TIMEOUT:
1275 case SG_GET_TIMEOUT:
1276 case SG_GET_VERSION_NUM:
1277 rc = scsi_cmd_ioctl(disk->queue, disk, mode, cmd_in, p);
1278 break;
1279 case SG_IO:
1280 rc = skd_ioctl_sg_io(skdev, mode, p);
1281 break;
1282
1283 default:
1284 rc = -ENOTTY;
1285 break;
1286 }
1287
1288 pr_debug("%s:%s:%d %s: completion rc %d\n",
1289 skdev->name, __func__, __LINE__, disk->disk_name, rc);
1290 return rc;
1291 }
1292
skd_ioctl_sg_io(struct skd_device * skdev,fmode_t mode,void __user * argp)1293 static int skd_ioctl_sg_io(struct skd_device *skdev, fmode_t mode,
1294 void __user *argp)
1295 {
1296 int rc;
1297 struct skd_sg_io sksgio;
1298
1299 memset(&sksgio, 0, sizeof(sksgio));
1300 sksgio.mode = mode;
1301 sksgio.argp = argp;
1302 sksgio.iov = &sksgio.no_iov_iov;
1303
1304 switch (skdev->state) {
1305 case SKD_DRVR_STATE_ONLINE:
1306 case SKD_DRVR_STATE_BUSY_IMMINENT:
1307 break;
1308
1309 default:
1310 pr_debug("%s:%s:%d drive not online\n",
1311 skdev->name, __func__, __LINE__);
1312 rc = -ENXIO;
1313 goto out;
1314 }
1315
1316 rc = skd_sg_io_get_and_check_args(skdev, &sksgio);
1317 if (rc)
1318 goto out;
1319
1320 rc = skd_sg_io_obtain_skspcl(skdev, &sksgio);
1321 if (rc)
1322 goto out;
1323
1324 rc = skd_sg_io_prep_buffering(skdev, &sksgio);
1325 if (rc)
1326 goto out;
1327
1328 rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_TO_DEV);
1329 if (rc)
1330 goto out;
1331
1332 rc = skd_sg_io_send_fitmsg(skdev, &sksgio);
1333 if (rc)
1334 goto out;
1335
1336 rc = skd_sg_io_await(skdev, &sksgio);
1337 if (rc)
1338 goto out;
1339
1340 rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_FROM_DEV);
1341 if (rc)
1342 goto out;
1343
1344 rc = skd_sg_io_put_status(skdev, &sksgio);
1345 if (rc)
1346 goto out;
1347
1348 rc = 0;
1349
1350 out:
1351 skd_sg_io_release_skspcl(skdev, &sksgio);
1352
1353 if (sksgio.iov != NULL && sksgio.iov != &sksgio.no_iov_iov)
1354 kfree(sksgio.iov);
1355 return rc;
1356 }
1357
skd_sg_io_get_and_check_args(struct skd_device * skdev,struct skd_sg_io * sksgio)1358 static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1359 struct skd_sg_io *sksgio)
1360 {
1361 struct sg_io_hdr *sgp = &sksgio->sg;
1362 int i, acc;
1363
1364 if (!access_ok(VERIFY_WRITE, sksgio->argp, sizeof(sg_io_hdr_t))) {
1365 pr_debug("%s:%s:%d access sg failed %p\n",
1366 skdev->name, __func__, __LINE__, sksgio->argp);
1367 return -EFAULT;
1368 }
1369
1370 if (__copy_from_user(sgp, sksgio->argp, sizeof(sg_io_hdr_t))) {
1371 pr_debug("%s:%s:%d copy_from_user sg failed %p\n",
1372 skdev->name, __func__, __LINE__, sksgio->argp);
1373 return -EFAULT;
1374 }
1375
1376 if (sgp->interface_id != SG_INTERFACE_ID_ORIG) {
1377 pr_debug("%s:%s:%d interface_id invalid 0x%x\n",
1378 skdev->name, __func__, __LINE__, sgp->interface_id);
1379 return -EINVAL;
1380 }
1381
1382 if (sgp->cmd_len > sizeof(sksgio->cdb)) {
1383 pr_debug("%s:%s:%d cmd_len invalid %d\n",
1384 skdev->name, __func__, __LINE__, sgp->cmd_len);
1385 return -EINVAL;
1386 }
1387
1388 if (sgp->iovec_count > 256) {
1389 pr_debug("%s:%s:%d iovec_count invalid %d\n",
1390 skdev->name, __func__, __LINE__, sgp->iovec_count);
1391 return -EINVAL;
1392 }
1393
1394 if (sgp->dxfer_len > (PAGE_SIZE * SKD_N_SG_PER_SPECIAL)) {
1395 pr_debug("%s:%s:%d dxfer_len invalid %d\n",
1396 skdev->name, __func__, __LINE__, sgp->dxfer_len);
1397 return -EINVAL;
1398 }
1399
1400 switch (sgp->dxfer_direction) {
1401 case SG_DXFER_NONE:
1402 acc = -1;
1403 break;
1404
1405 case SG_DXFER_TO_DEV:
1406 acc = VERIFY_READ;
1407 break;
1408
1409 case SG_DXFER_FROM_DEV:
1410 case SG_DXFER_TO_FROM_DEV:
1411 acc = VERIFY_WRITE;
1412 break;
1413
1414 default:
1415 pr_debug("%s:%s:%d dxfer_dir invalid %d\n",
1416 skdev->name, __func__, __LINE__, sgp->dxfer_direction);
1417 return -EINVAL;
1418 }
1419
1420 if (copy_from_user(sksgio->cdb, sgp->cmdp, sgp->cmd_len)) {
1421 pr_debug("%s:%s:%d copy_from_user cmdp failed %p\n",
1422 skdev->name, __func__, __LINE__, sgp->cmdp);
1423 return -EFAULT;
1424 }
1425
1426 if (sgp->mx_sb_len != 0) {
1427 if (!access_ok(VERIFY_WRITE, sgp->sbp, sgp->mx_sb_len)) {
1428 pr_debug("%s:%s:%d access sbp failed %p\n",
1429 skdev->name, __func__, __LINE__, sgp->sbp);
1430 return -EFAULT;
1431 }
1432 }
1433
1434 if (sgp->iovec_count == 0) {
1435 sksgio->iov[0].iov_base = sgp->dxferp;
1436 sksgio->iov[0].iov_len = sgp->dxfer_len;
1437 sksgio->iovcnt = 1;
1438 sksgio->dxfer_len = sgp->dxfer_len;
1439 } else {
1440 struct sg_iovec *iov;
1441 uint nbytes = sizeof(*iov) * sgp->iovec_count;
1442 size_t iov_data_len;
1443
1444 iov = kmalloc(nbytes, GFP_KERNEL);
1445 if (iov == NULL) {
1446 pr_debug("%s:%s:%d alloc iovec failed %d\n",
1447 skdev->name, __func__, __LINE__,
1448 sgp->iovec_count);
1449 return -ENOMEM;
1450 }
1451 sksgio->iov = iov;
1452 sksgio->iovcnt = sgp->iovec_count;
1453
1454 if (copy_from_user(iov, sgp->dxferp, nbytes)) {
1455 pr_debug("%s:%s:%d copy_from_user iovec failed %p\n",
1456 skdev->name, __func__, __LINE__, sgp->dxferp);
1457 return -EFAULT;
1458 }
1459
1460 /*
1461 * Sum up the vecs, making sure they don't overflow
1462 */
1463 iov_data_len = 0;
1464 for (i = 0; i < sgp->iovec_count; i++) {
1465 if (iov_data_len + iov[i].iov_len < iov_data_len)
1466 return -EINVAL;
1467 iov_data_len += iov[i].iov_len;
1468 }
1469
1470 /* SG_IO howto says that the shorter of the two wins */
1471 if (sgp->dxfer_len < iov_data_len) {
1472 sksgio->iovcnt = iov_shorten((struct iovec *)iov,
1473 sgp->iovec_count,
1474 sgp->dxfer_len);
1475 sksgio->dxfer_len = sgp->dxfer_len;
1476 } else
1477 sksgio->dxfer_len = iov_data_len;
1478 }
1479
1480 if (sgp->dxfer_direction != SG_DXFER_NONE) {
1481 struct sg_iovec *iov = sksgio->iov;
1482 for (i = 0; i < sksgio->iovcnt; i++, iov++) {
1483 if (!access_ok(acc, iov->iov_base, iov->iov_len)) {
1484 pr_debug("%s:%s:%d access data failed %p/%d\n",
1485 skdev->name, __func__, __LINE__,
1486 iov->iov_base, (int)iov->iov_len);
1487 return -EFAULT;
1488 }
1489 }
1490 }
1491
1492 return 0;
1493 }
1494
skd_sg_io_obtain_skspcl(struct skd_device * skdev,struct skd_sg_io * sksgio)1495 static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1496 struct skd_sg_io *sksgio)
1497 {
1498 struct skd_special_context *skspcl = NULL;
1499 int rc;
1500
1501 for (;;) {
1502 ulong flags;
1503
1504 spin_lock_irqsave(&skdev->lock, flags);
1505 skspcl = skdev->skspcl_free_list;
1506 if (skspcl != NULL) {
1507 skdev->skspcl_free_list =
1508 (struct skd_special_context *)skspcl->req.next;
1509 skspcl->req.id += SKD_ID_INCR;
1510 skspcl->req.state = SKD_REQ_STATE_SETUP;
1511 skspcl->orphaned = 0;
1512 skspcl->req.n_sg = 0;
1513 }
1514 spin_unlock_irqrestore(&skdev->lock, flags);
1515
1516 if (skspcl != NULL) {
1517 rc = 0;
1518 break;
1519 }
1520
1521 pr_debug("%s:%s:%d blocking\n",
1522 skdev->name, __func__, __LINE__);
1523
1524 rc = wait_event_interruptible_timeout(
1525 skdev->waitq,
1526 (skdev->skspcl_free_list != NULL),
1527 msecs_to_jiffies(sksgio->sg.timeout));
1528
1529 pr_debug("%s:%s:%d unblocking, rc=%d\n",
1530 skdev->name, __func__, __LINE__, rc);
1531
1532 if (rc <= 0) {
1533 if (rc == 0)
1534 rc = -ETIMEDOUT;
1535 else
1536 rc = -EINTR;
1537 break;
1538 }
1539 /*
1540 * If we get here rc > 0 meaning the timeout to
1541 * wait_event_interruptible_timeout() had time left, hence the
1542 * sought event -- non-empty free list -- happened.
1543 * Retry the allocation.
1544 */
1545 }
1546 sksgio->skspcl = skspcl;
1547
1548 return rc;
1549 }
1550
skd_skreq_prep_buffering(struct skd_device * skdev,struct skd_request_context * skreq,u32 dxfer_len)1551 static int skd_skreq_prep_buffering(struct skd_device *skdev,
1552 struct skd_request_context *skreq,
1553 u32 dxfer_len)
1554 {
1555 u32 resid = dxfer_len;
1556
1557 /*
1558 * The DMA engine must have aligned addresses and byte counts.
1559 */
1560 resid += (-resid) & 3;
1561 skreq->sg_byte_count = resid;
1562
1563 skreq->n_sg = 0;
1564
1565 while (resid > 0) {
1566 u32 nbytes = PAGE_SIZE;
1567 u32 ix = skreq->n_sg;
1568 struct scatterlist *sg = &skreq->sg[ix];
1569 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1570 struct page *page;
1571
1572 if (nbytes > resid)
1573 nbytes = resid;
1574
1575 page = alloc_page(GFP_KERNEL);
1576 if (page == NULL)
1577 return -ENOMEM;
1578
1579 sg_set_page(sg, page, nbytes, 0);
1580
1581 /* TODO: This should be going through a pci_???()
1582 * routine to do proper mapping. */
1583 sksg->control = FIT_SGD_CONTROL_NOT_LAST;
1584 sksg->byte_count = nbytes;
1585
1586 sksg->host_side_addr = sg_phys(sg);
1587
1588 sksg->dev_side_addr = 0;
1589 sksg->next_desc_ptr = skreq->sksg_dma_address +
1590 (ix + 1) * sizeof(*sksg);
1591
1592 skreq->n_sg++;
1593 resid -= nbytes;
1594 }
1595
1596 if (skreq->n_sg > 0) {
1597 u32 ix = skreq->n_sg - 1;
1598 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1599
1600 sksg->control = FIT_SGD_CONTROL_LAST;
1601 sksg->next_desc_ptr = 0;
1602 }
1603
1604 if (unlikely(skdev->dbg_level > 1)) {
1605 u32 i;
1606
1607 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
1608 skdev->name, __func__, __LINE__,
1609 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
1610 for (i = 0; i < skreq->n_sg; i++) {
1611 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
1612
1613 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
1614 "addr=0x%llx next=0x%llx\n",
1615 skdev->name, __func__, __LINE__,
1616 i, sgd->byte_count, sgd->control,
1617 sgd->host_side_addr, sgd->next_desc_ptr);
1618 }
1619 }
1620
1621 return 0;
1622 }
1623
skd_sg_io_prep_buffering(struct skd_device * skdev,struct skd_sg_io * sksgio)1624 static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1625 struct skd_sg_io *sksgio)
1626 {
1627 struct skd_special_context *skspcl = sksgio->skspcl;
1628 struct skd_request_context *skreq = &skspcl->req;
1629 u32 dxfer_len = sksgio->dxfer_len;
1630 int rc;
1631
1632 rc = skd_skreq_prep_buffering(skdev, skreq, dxfer_len);
1633 /*
1634 * Eventually, errors or not, skd_release_special() is called
1635 * to recover allocations including partial allocations.
1636 */
1637 return rc;
1638 }
1639
skd_sg_io_copy_buffer(struct skd_device * skdev,struct skd_sg_io * sksgio,int dxfer_dir)1640 static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1641 struct skd_sg_io *sksgio, int dxfer_dir)
1642 {
1643 struct skd_special_context *skspcl = sksgio->skspcl;
1644 u32 iov_ix = 0;
1645 struct sg_iovec curiov;
1646 u32 sksg_ix = 0;
1647 u8 *bufp = NULL;
1648 u32 buf_len = 0;
1649 u32 resid = sksgio->dxfer_len;
1650 int rc;
1651
1652 curiov.iov_len = 0;
1653 curiov.iov_base = NULL;
1654
1655 if (dxfer_dir != sksgio->sg.dxfer_direction) {
1656 if (dxfer_dir != SG_DXFER_TO_DEV ||
1657 sksgio->sg.dxfer_direction != SG_DXFER_TO_FROM_DEV)
1658 return 0;
1659 }
1660
1661 while (resid > 0) {
1662 u32 nbytes = PAGE_SIZE;
1663
1664 if (curiov.iov_len == 0) {
1665 curiov = sksgio->iov[iov_ix++];
1666 continue;
1667 }
1668
1669 if (buf_len == 0) {
1670 struct page *page;
1671 page = sg_page(&skspcl->req.sg[sksg_ix++]);
1672 bufp = page_address(page);
1673 buf_len = PAGE_SIZE;
1674 }
1675
1676 nbytes = min_t(u32, nbytes, resid);
1677 nbytes = min_t(u32, nbytes, curiov.iov_len);
1678 nbytes = min_t(u32, nbytes, buf_len);
1679
1680 if (dxfer_dir == SG_DXFER_TO_DEV)
1681 rc = __copy_from_user(bufp, curiov.iov_base, nbytes);
1682 else
1683 rc = __copy_to_user(curiov.iov_base, bufp, nbytes);
1684
1685 if (rc)
1686 return -EFAULT;
1687
1688 resid -= nbytes;
1689 curiov.iov_len -= nbytes;
1690 curiov.iov_base += nbytes;
1691 buf_len -= nbytes;
1692 }
1693
1694 return 0;
1695 }
1696
skd_sg_io_send_fitmsg(struct skd_device * skdev,struct skd_sg_io * sksgio)1697 static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1698 struct skd_sg_io *sksgio)
1699 {
1700 struct skd_special_context *skspcl = sksgio->skspcl;
1701 struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
1702 struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
1703
1704 memset(skspcl->msg_buf, 0, SKD_N_SPECIAL_FITMSG_BYTES);
1705
1706 /* Initialize the FIT msg header */
1707 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1708 fmh->num_protocol_cmds_coalesced = 1;
1709
1710 /* Initialize the SCSI request */
1711 if (sksgio->sg.dxfer_direction != SG_DXFER_NONE)
1712 scsi_req->hdr.sg_list_dma_address =
1713 cpu_to_be64(skspcl->req.sksg_dma_address);
1714 scsi_req->hdr.tag = skspcl->req.id;
1715 scsi_req->hdr.sg_list_len_bytes =
1716 cpu_to_be32(skspcl->req.sg_byte_count);
1717 memcpy(scsi_req->cdb, sksgio->cdb, sizeof(scsi_req->cdb));
1718
1719 skspcl->req.state = SKD_REQ_STATE_BUSY;
1720 skd_send_special_fitmsg(skdev, skspcl);
1721
1722 return 0;
1723 }
1724
skd_sg_io_await(struct skd_device * skdev,struct skd_sg_io * sksgio)1725 static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio)
1726 {
1727 unsigned long flags;
1728 int rc;
1729
1730 rc = wait_event_interruptible_timeout(skdev->waitq,
1731 (sksgio->skspcl->req.state !=
1732 SKD_REQ_STATE_BUSY),
1733 msecs_to_jiffies(sksgio->sg.
1734 timeout));
1735
1736 spin_lock_irqsave(&skdev->lock, flags);
1737
1738 if (sksgio->skspcl->req.state == SKD_REQ_STATE_ABORTED) {
1739 pr_debug("%s:%s:%d skspcl %p aborted\n",
1740 skdev->name, __func__, __LINE__, sksgio->skspcl);
1741
1742 /* Build check cond, sense and let command finish. */
1743 /* For a timeout, we must fabricate completion and sense
1744 * data to complete the command */
1745 sksgio->skspcl->req.completion.status =
1746 SAM_STAT_CHECK_CONDITION;
1747
1748 memset(&sksgio->skspcl->req.err_info, 0,
1749 sizeof(sksgio->skspcl->req.err_info));
1750 sksgio->skspcl->req.err_info.type = 0x70;
1751 sksgio->skspcl->req.err_info.key = ABORTED_COMMAND;
1752 sksgio->skspcl->req.err_info.code = 0x44;
1753 sksgio->skspcl->req.err_info.qual = 0;
1754 rc = 0;
1755 } else if (sksgio->skspcl->req.state != SKD_REQ_STATE_BUSY)
1756 /* No longer on the adapter. We finish. */
1757 rc = 0;
1758 else {
1759 /* Something's gone wrong. Still busy. Timeout or
1760 * user interrupted (control-C). Mark as an orphan
1761 * so it will be disposed when completed. */
1762 sksgio->skspcl->orphaned = 1;
1763 sksgio->skspcl = NULL;
1764 if (rc == 0) {
1765 pr_debug("%s:%s:%d timed out %p (%u ms)\n",
1766 skdev->name, __func__, __LINE__,
1767 sksgio, sksgio->sg.timeout);
1768 rc = -ETIMEDOUT;
1769 } else {
1770 pr_debug("%s:%s:%d cntlc %p\n",
1771 skdev->name, __func__, __LINE__, sksgio);
1772 rc = -EINTR;
1773 }
1774 }
1775
1776 spin_unlock_irqrestore(&skdev->lock, flags);
1777
1778 return rc;
1779 }
1780
skd_sg_io_put_status(struct skd_device * skdev,struct skd_sg_io * sksgio)1781 static int skd_sg_io_put_status(struct skd_device *skdev,
1782 struct skd_sg_io *sksgio)
1783 {
1784 struct sg_io_hdr *sgp = &sksgio->sg;
1785 struct skd_special_context *skspcl = sksgio->skspcl;
1786 int resid = 0;
1787
1788 u32 nb = be32_to_cpu(skspcl->req.completion.num_returned_bytes);
1789
1790 sgp->status = skspcl->req.completion.status;
1791 resid = sksgio->dxfer_len - nb;
1792
1793 sgp->masked_status = sgp->status & STATUS_MASK;
1794 sgp->msg_status = 0;
1795 sgp->host_status = 0;
1796 sgp->driver_status = 0;
1797 sgp->resid = resid;
1798 if (sgp->masked_status || sgp->host_status || sgp->driver_status)
1799 sgp->info |= SG_INFO_CHECK;
1800
1801 pr_debug("%s:%s:%d status %x masked %x resid 0x%x\n",
1802 skdev->name, __func__, __LINE__,
1803 sgp->status, sgp->masked_status, sgp->resid);
1804
1805 if (sgp->masked_status == SAM_STAT_CHECK_CONDITION) {
1806 if (sgp->mx_sb_len > 0) {
1807 struct fit_comp_error_info *ei = &skspcl->req.err_info;
1808 u32 nbytes = sizeof(*ei);
1809
1810 nbytes = min_t(u32, nbytes, sgp->mx_sb_len);
1811
1812 sgp->sb_len_wr = nbytes;
1813
1814 if (__copy_to_user(sgp->sbp, ei, nbytes)) {
1815 pr_debug("%s:%s:%d copy_to_user sense failed %p\n",
1816 skdev->name, __func__, __LINE__,
1817 sgp->sbp);
1818 return -EFAULT;
1819 }
1820 }
1821 }
1822
1823 if (__copy_to_user(sksgio->argp, sgp, sizeof(sg_io_hdr_t))) {
1824 pr_debug("%s:%s:%d copy_to_user sg failed %p\n",
1825 skdev->name, __func__, __LINE__, sksgio->argp);
1826 return -EFAULT;
1827 }
1828
1829 return 0;
1830 }
1831
skd_sg_io_release_skspcl(struct skd_device * skdev,struct skd_sg_io * sksgio)1832 static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1833 struct skd_sg_io *sksgio)
1834 {
1835 struct skd_special_context *skspcl = sksgio->skspcl;
1836
1837 if (skspcl != NULL) {
1838 ulong flags;
1839
1840 sksgio->skspcl = NULL;
1841
1842 spin_lock_irqsave(&skdev->lock, flags);
1843 skd_release_special(skdev, skspcl);
1844 spin_unlock_irqrestore(&skdev->lock, flags);
1845 }
1846
1847 return 0;
1848 }
1849
1850 /*
1851 *****************************************************************************
1852 * INTERNAL REQUESTS -- generated by driver itself
1853 *****************************************************************************
1854 */
1855
skd_format_internal_skspcl(struct skd_device * skdev)1856 static int skd_format_internal_skspcl(struct skd_device *skdev)
1857 {
1858 struct skd_special_context *skspcl = &skdev->internal_skspcl;
1859 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1860 struct fit_msg_hdr *fmh;
1861 uint64_t dma_address;
1862 struct skd_scsi_request *scsi;
1863
1864 fmh = (struct fit_msg_hdr *)&skspcl->msg_buf[0];
1865 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1866 fmh->num_protocol_cmds_coalesced = 1;
1867
1868 scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1869 memset(scsi, 0, sizeof(*scsi));
1870 dma_address = skspcl->req.sksg_dma_address;
1871 scsi->hdr.sg_list_dma_address = cpu_to_be64(dma_address);
1872 sgd->control = FIT_SGD_CONTROL_LAST;
1873 sgd->byte_count = 0;
1874 sgd->host_side_addr = skspcl->db_dma_address;
1875 sgd->dev_side_addr = 0;
1876 sgd->next_desc_ptr = 0LL;
1877
1878 return 1;
1879 }
1880
1881 #define WR_BUF_SIZE SKD_N_INTERNAL_BYTES
1882
skd_send_internal_skspcl(struct skd_device * skdev,struct skd_special_context * skspcl,u8 opcode)1883 static void skd_send_internal_skspcl(struct skd_device *skdev,
1884 struct skd_special_context *skspcl,
1885 u8 opcode)
1886 {
1887 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1888 struct skd_scsi_request *scsi;
1889 unsigned char *buf = skspcl->data_buf;
1890 int i;
1891
1892 if (skspcl->req.state != SKD_REQ_STATE_IDLE)
1893 /*
1894 * A refresh is already in progress.
1895 * Just wait for it to finish.
1896 */
1897 return;
1898
1899 SKD_ASSERT((skspcl->req.id & SKD_ID_INCR) == 0);
1900 skspcl->req.state = SKD_REQ_STATE_BUSY;
1901 skspcl->req.id += SKD_ID_INCR;
1902
1903 scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1904 scsi->hdr.tag = skspcl->req.id;
1905
1906 memset(scsi->cdb, 0, sizeof(scsi->cdb));
1907
1908 switch (opcode) {
1909 case TEST_UNIT_READY:
1910 scsi->cdb[0] = TEST_UNIT_READY;
1911 sgd->byte_count = 0;
1912 scsi->hdr.sg_list_len_bytes = 0;
1913 break;
1914
1915 case READ_CAPACITY:
1916 scsi->cdb[0] = READ_CAPACITY;
1917 sgd->byte_count = SKD_N_READ_CAP_BYTES;
1918 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1919 break;
1920
1921 case INQUIRY:
1922 scsi->cdb[0] = INQUIRY;
1923 scsi->cdb[1] = 0x01; /* evpd */
1924 scsi->cdb[2] = 0x80; /* serial number page */
1925 scsi->cdb[4] = 0x10;
1926 sgd->byte_count = 16;
1927 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1928 break;
1929
1930 case SYNCHRONIZE_CACHE:
1931 scsi->cdb[0] = SYNCHRONIZE_CACHE;
1932 sgd->byte_count = 0;
1933 scsi->hdr.sg_list_len_bytes = 0;
1934 break;
1935
1936 case WRITE_BUFFER:
1937 scsi->cdb[0] = WRITE_BUFFER;
1938 scsi->cdb[1] = 0x02;
1939 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1940 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1941 sgd->byte_count = WR_BUF_SIZE;
1942 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1943 /* fill incrementing byte pattern */
1944 for (i = 0; i < sgd->byte_count; i++)
1945 buf[i] = i & 0xFF;
1946 break;
1947
1948 case READ_BUFFER:
1949 scsi->cdb[0] = READ_BUFFER;
1950 scsi->cdb[1] = 0x02;
1951 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1952 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1953 sgd->byte_count = WR_BUF_SIZE;
1954 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1955 memset(skspcl->data_buf, 0, sgd->byte_count);
1956 break;
1957
1958 default:
1959 SKD_ASSERT("Don't know what to send");
1960 return;
1961
1962 }
1963 skd_send_special_fitmsg(skdev, skspcl);
1964 }
1965
skd_refresh_device_data(struct skd_device * skdev)1966 static void skd_refresh_device_data(struct skd_device *skdev)
1967 {
1968 struct skd_special_context *skspcl = &skdev->internal_skspcl;
1969
1970 skd_send_internal_skspcl(skdev, skspcl, TEST_UNIT_READY);
1971 }
1972
skd_chk_read_buf(struct skd_device * skdev,struct skd_special_context * skspcl)1973 static int skd_chk_read_buf(struct skd_device *skdev,
1974 struct skd_special_context *skspcl)
1975 {
1976 unsigned char *buf = skspcl->data_buf;
1977 int i;
1978
1979 /* check for incrementing byte pattern */
1980 for (i = 0; i < WR_BUF_SIZE; i++)
1981 if (buf[i] != (i & 0xFF))
1982 return 1;
1983
1984 return 0;
1985 }
1986
skd_log_check_status(struct skd_device * skdev,u8 status,u8 key,u8 code,u8 qual,u8 fruc)1987 static void skd_log_check_status(struct skd_device *skdev, u8 status, u8 key,
1988 u8 code, u8 qual, u8 fruc)
1989 {
1990 /* If the check condition is of special interest, log a message */
1991 if ((status == SAM_STAT_CHECK_CONDITION) && (key == 0x02)
1992 && (code == 0x04) && (qual == 0x06)) {
1993 pr_err("(%s): *** LOST_WRITE_DATA ERROR *** key/asc/"
1994 "ascq/fruc %02x/%02x/%02x/%02x\n",
1995 skd_name(skdev), key, code, qual, fruc);
1996 }
1997 }
1998
skd_complete_internal(struct skd_device * skdev,volatile struct fit_completion_entry_v1 * skcomp,volatile struct fit_comp_error_info * skerr,struct skd_special_context * skspcl)1999 static void skd_complete_internal(struct skd_device *skdev,
2000 volatile struct fit_completion_entry_v1
2001 *skcomp,
2002 volatile struct fit_comp_error_info *skerr,
2003 struct skd_special_context *skspcl)
2004 {
2005 u8 *buf = skspcl->data_buf;
2006 u8 status;
2007 int i;
2008 struct skd_scsi_request *scsi =
2009 (struct skd_scsi_request *)&skspcl->msg_buf[64];
2010
2011 SKD_ASSERT(skspcl == &skdev->internal_skspcl);
2012
2013 pr_debug("%s:%s:%d complete internal %x\n",
2014 skdev->name, __func__, __LINE__, scsi->cdb[0]);
2015
2016 skspcl->req.completion = *skcomp;
2017 skspcl->req.state = SKD_REQ_STATE_IDLE;
2018 skspcl->req.id += SKD_ID_INCR;
2019
2020 status = skspcl->req.completion.status;
2021
2022 skd_log_check_status(skdev, status, skerr->key, skerr->code,
2023 skerr->qual, skerr->fruc);
2024
2025 switch (scsi->cdb[0]) {
2026 case TEST_UNIT_READY:
2027 if (status == SAM_STAT_GOOD)
2028 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
2029 else if ((status == SAM_STAT_CHECK_CONDITION) &&
2030 (skerr->key == MEDIUM_ERROR))
2031 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
2032 else {
2033 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2034 pr_debug("%s:%s:%d TUR failed, don't send anymore state 0x%x\n",
2035 skdev->name, __func__, __LINE__,
2036 skdev->state);
2037 return;
2038 }
2039 pr_debug("%s:%s:%d **** TUR failed, retry skerr\n",
2040 skdev->name, __func__, __LINE__);
2041 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2042 }
2043 break;
2044
2045 case WRITE_BUFFER:
2046 if (status == SAM_STAT_GOOD)
2047 skd_send_internal_skspcl(skdev, skspcl, READ_BUFFER);
2048 else {
2049 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2050 pr_debug("%s:%s:%d write buffer failed, don't send anymore state 0x%x\n",
2051 skdev->name, __func__, __LINE__,
2052 skdev->state);
2053 return;
2054 }
2055 pr_debug("%s:%s:%d **** write buffer failed, retry skerr\n",
2056 skdev->name, __func__, __LINE__);
2057 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2058 }
2059 break;
2060
2061 case READ_BUFFER:
2062 if (status == SAM_STAT_GOOD) {
2063 if (skd_chk_read_buf(skdev, skspcl) == 0)
2064 skd_send_internal_skspcl(skdev, skspcl,
2065 READ_CAPACITY);
2066 else {
2067 pr_err(
2068 "(%s):*** W/R Buffer mismatch %d ***\n",
2069 skd_name(skdev), skdev->connect_retries);
2070 if (skdev->connect_retries <
2071 SKD_MAX_CONNECT_RETRIES) {
2072 skdev->connect_retries++;
2073 skd_soft_reset(skdev);
2074 } else {
2075 pr_err(
2076 "(%s): W/R Buffer Connect Error\n",
2077 skd_name(skdev));
2078 return;
2079 }
2080 }
2081
2082 } else {
2083 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2084 pr_debug("%s:%s:%d "
2085 "read buffer failed, don't send anymore state 0x%x\n",
2086 skdev->name, __func__, __LINE__,
2087 skdev->state);
2088 return;
2089 }
2090 pr_debug("%s:%s:%d "
2091 "**** read buffer failed, retry skerr\n",
2092 skdev->name, __func__, __LINE__);
2093 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2094 }
2095 break;
2096
2097 case READ_CAPACITY:
2098 skdev->read_cap_is_valid = 0;
2099 if (status == SAM_STAT_GOOD) {
2100 skdev->read_cap_last_lba =
2101 (buf[0] << 24) | (buf[1] << 16) |
2102 (buf[2] << 8) | buf[3];
2103 skdev->read_cap_blocksize =
2104 (buf[4] << 24) | (buf[5] << 16) |
2105 (buf[6] << 8) | buf[7];
2106
2107 pr_debug("%s:%s:%d last lba %d, bs %d\n",
2108 skdev->name, __func__, __LINE__,
2109 skdev->read_cap_last_lba,
2110 skdev->read_cap_blocksize);
2111
2112 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2113
2114 skdev->read_cap_is_valid = 1;
2115
2116 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2117 } else if ((status == SAM_STAT_CHECK_CONDITION) &&
2118 (skerr->key == MEDIUM_ERROR)) {
2119 skdev->read_cap_last_lba = ~0;
2120 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2121 pr_debug("%s:%s:%d "
2122 "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n",
2123 skdev->name, __func__, __LINE__);
2124 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2125 } else {
2126 pr_debug("%s:%s:%d **** READCAP failed, retry TUR\n",
2127 skdev->name, __func__, __LINE__);
2128 skd_send_internal_skspcl(skdev, skspcl,
2129 TEST_UNIT_READY);
2130 }
2131 break;
2132
2133 case INQUIRY:
2134 skdev->inquiry_is_valid = 0;
2135 if (status == SAM_STAT_GOOD) {
2136 skdev->inquiry_is_valid = 1;
2137
2138 for (i = 0; i < 12; i++)
2139 skdev->inq_serial_num[i] = buf[i + 4];
2140 skdev->inq_serial_num[12] = 0;
2141 }
2142
2143 if (skd_unquiesce_dev(skdev) < 0)
2144 pr_debug("%s:%s:%d **** failed, to ONLINE device\n",
2145 skdev->name, __func__, __LINE__);
2146 /* connection is complete */
2147 skdev->connect_retries = 0;
2148 break;
2149
2150 case SYNCHRONIZE_CACHE:
2151 if (status == SAM_STAT_GOOD)
2152 skdev->sync_done = 1;
2153 else
2154 skdev->sync_done = -1;
2155 wake_up_interruptible(&skdev->waitq);
2156 break;
2157
2158 default:
2159 SKD_ASSERT("we didn't send this");
2160 }
2161 }
2162
2163 /*
2164 *****************************************************************************
2165 * FIT MESSAGES
2166 *****************************************************************************
2167 */
2168
skd_send_fitmsg(struct skd_device * skdev,struct skd_fitmsg_context * skmsg)2169 static void skd_send_fitmsg(struct skd_device *skdev,
2170 struct skd_fitmsg_context *skmsg)
2171 {
2172 u64 qcmd;
2173 struct fit_msg_hdr *fmh;
2174
2175 pr_debug("%s:%s:%d dma address 0x%llx, busy=%d\n",
2176 skdev->name, __func__, __LINE__,
2177 skmsg->mb_dma_address, skdev->in_flight);
2178 pr_debug("%s:%s:%d msg_buf 0x%p, offset %x\n",
2179 skdev->name, __func__, __LINE__,
2180 skmsg->msg_buf, skmsg->offset);
2181
2182 qcmd = skmsg->mb_dma_address;
2183 qcmd |= FIT_QCMD_QID_NORMAL;
2184
2185 fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
2186 skmsg->outstanding = fmh->num_protocol_cmds_coalesced;
2187
2188 if (unlikely(skdev->dbg_level > 1)) {
2189 u8 *bp = (u8 *)skmsg->msg_buf;
2190 int i;
2191 for (i = 0; i < skmsg->length; i += 8) {
2192 pr_debug("%s:%s:%d msg[%2d] %02x %02x %02x %02x "
2193 "%02x %02x %02x %02x\n",
2194 skdev->name, __func__, __LINE__,
2195 i, bp[i + 0], bp[i + 1], bp[i + 2],
2196 bp[i + 3], bp[i + 4], bp[i + 5],
2197 bp[i + 6], bp[i + 7]);
2198 if (i == 0)
2199 i = 64 - 8;
2200 }
2201 }
2202
2203 if (skmsg->length > 256)
2204 qcmd |= FIT_QCMD_MSGSIZE_512;
2205 else if (skmsg->length > 128)
2206 qcmd |= FIT_QCMD_MSGSIZE_256;
2207 else if (skmsg->length > 64)
2208 qcmd |= FIT_QCMD_MSGSIZE_128;
2209 else
2210 /*
2211 * This makes no sense because the FIT msg header is
2212 * 64 bytes. If the msg is only 64 bytes long it has
2213 * no payload.
2214 */
2215 qcmd |= FIT_QCMD_MSGSIZE_64;
2216
2217 /* Make sure skd_msg_buf is written before the doorbell is triggered. */
2218 smp_wmb();
2219
2220 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2221
2222 }
2223
skd_send_special_fitmsg(struct skd_device * skdev,struct skd_special_context * skspcl)2224 static void skd_send_special_fitmsg(struct skd_device *skdev,
2225 struct skd_special_context *skspcl)
2226 {
2227 u64 qcmd;
2228
2229 if (unlikely(skdev->dbg_level > 1)) {
2230 u8 *bp = (u8 *)skspcl->msg_buf;
2231 int i;
2232
2233 for (i = 0; i < SKD_N_SPECIAL_FITMSG_BYTES; i += 8) {
2234 pr_debug("%s:%s:%d spcl[%2d] %02x %02x %02x %02x "
2235 "%02x %02x %02x %02x\n",
2236 skdev->name, __func__, __LINE__, i,
2237 bp[i + 0], bp[i + 1], bp[i + 2], bp[i + 3],
2238 bp[i + 4], bp[i + 5], bp[i + 6], bp[i + 7]);
2239 if (i == 0)
2240 i = 64 - 8;
2241 }
2242
2243 pr_debug("%s:%s:%d skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
2244 skdev->name, __func__, __LINE__,
2245 skspcl, skspcl->req.id, skspcl->req.sksg_list,
2246 skspcl->req.sksg_dma_address);
2247 for (i = 0; i < skspcl->req.n_sg; i++) {
2248 struct fit_sg_descriptor *sgd =
2249 &skspcl->req.sksg_list[i];
2250
2251 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
2252 "addr=0x%llx next=0x%llx\n",
2253 skdev->name, __func__, __LINE__,
2254 i, sgd->byte_count, sgd->control,
2255 sgd->host_side_addr, sgd->next_desc_ptr);
2256 }
2257 }
2258
2259 /*
2260 * Special FIT msgs are always 128 bytes: a 64-byte FIT hdr
2261 * and one 64-byte SSDI command.
2262 */
2263 qcmd = skspcl->mb_dma_address;
2264 qcmd |= FIT_QCMD_QID_NORMAL + FIT_QCMD_MSGSIZE_128;
2265
2266 /* Make sure skd_msg_buf is written before the doorbell is triggered. */
2267 smp_wmb();
2268
2269 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2270 }
2271
2272 /*
2273 *****************************************************************************
2274 * COMPLETION QUEUE
2275 *****************************************************************************
2276 */
2277
2278 static void skd_complete_other(struct skd_device *skdev,
2279 volatile struct fit_completion_entry_v1 *skcomp,
2280 volatile struct fit_comp_error_info *skerr);
2281
2282 struct sns_info {
2283 u8 type;
2284 u8 stat;
2285 u8 key;
2286 u8 asc;
2287 u8 ascq;
2288 u8 mask;
2289 enum skd_check_status_action action;
2290 };
2291
2292 static struct sns_info skd_chkstat_table[] = {
2293 /* Good */
2294 { 0x70, 0x02, RECOVERED_ERROR, 0, 0, 0x1c,
2295 SKD_CHECK_STATUS_REPORT_GOOD },
2296
2297 /* Smart alerts */
2298 { 0x70, 0x02, NO_SENSE, 0x0B, 0x00, 0x1E, /* warnings */
2299 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2300 { 0x70, 0x02, NO_SENSE, 0x5D, 0x00, 0x1E, /* thresholds */
2301 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2302 { 0x70, 0x02, RECOVERED_ERROR, 0x0B, 0x01, 0x1F, /* temperature over trigger */
2303 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2304
2305 /* Retry (with limits) */
2306 { 0x70, 0x02, 0x0B, 0, 0, 0x1C, /* This one is for DMA ERROR */
2307 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2308 { 0x70, 0x02, 0x06, 0x0B, 0x00, 0x1E, /* warnings */
2309 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2310 { 0x70, 0x02, 0x06, 0x5D, 0x00, 0x1E, /* thresholds */
2311 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2312 { 0x70, 0x02, 0x06, 0x80, 0x30, 0x1F, /* backup power */
2313 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2314
2315 /* Busy (or about to be) */
2316 { 0x70, 0x02, 0x06, 0x3f, 0x01, 0x1F, /* fw changed */
2317 SKD_CHECK_STATUS_BUSY_IMMINENT },
2318 };
2319
2320 /*
2321 * Look up status and sense data to decide how to handle the error
2322 * from the device.
2323 * mask says which fields must match e.g., mask=0x18 means check
2324 * type and stat, ignore key, asc, ascq.
2325 */
2326
2327 static enum skd_check_status_action
skd_check_status(struct skd_device * skdev,u8 cmp_status,volatile struct fit_comp_error_info * skerr)2328 skd_check_status(struct skd_device *skdev,
2329 u8 cmp_status, volatile struct fit_comp_error_info *skerr)
2330 {
2331 int i, n;
2332
2333 pr_err("(%s): key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
2334 skd_name(skdev), skerr->key, skerr->code, skerr->qual,
2335 skerr->fruc);
2336
2337 pr_debug("%s:%s:%d stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
2338 skdev->name, __func__, __LINE__, skerr->type, cmp_status,
2339 skerr->key, skerr->code, skerr->qual, skerr->fruc);
2340
2341 /* Does the info match an entry in the good category? */
2342 n = sizeof(skd_chkstat_table) / sizeof(skd_chkstat_table[0]);
2343 for (i = 0; i < n; i++) {
2344 struct sns_info *sns = &skd_chkstat_table[i];
2345
2346 if (sns->mask & 0x10)
2347 if (skerr->type != sns->type)
2348 continue;
2349
2350 if (sns->mask & 0x08)
2351 if (cmp_status != sns->stat)
2352 continue;
2353
2354 if (sns->mask & 0x04)
2355 if (skerr->key != sns->key)
2356 continue;
2357
2358 if (sns->mask & 0x02)
2359 if (skerr->code != sns->asc)
2360 continue;
2361
2362 if (sns->mask & 0x01)
2363 if (skerr->qual != sns->ascq)
2364 continue;
2365
2366 if (sns->action == SKD_CHECK_STATUS_REPORT_SMART_ALERT) {
2367 pr_err("(%s): SMART Alert: sense key/asc/ascq "
2368 "%02x/%02x/%02x\n",
2369 skd_name(skdev), skerr->key,
2370 skerr->code, skerr->qual);
2371 }
2372 return sns->action;
2373 }
2374
2375 /* No other match, so nonzero status means error,
2376 * zero status means good
2377 */
2378 if (cmp_status) {
2379 pr_debug("%s:%s:%d status check: error\n",
2380 skdev->name, __func__, __LINE__);
2381 return SKD_CHECK_STATUS_REPORT_ERROR;
2382 }
2383
2384 pr_debug("%s:%s:%d status check good default\n",
2385 skdev->name, __func__, __LINE__);
2386 return SKD_CHECK_STATUS_REPORT_GOOD;
2387 }
2388
skd_resolve_req_exception(struct skd_device * skdev,struct skd_request_context * skreq)2389 static void skd_resolve_req_exception(struct skd_device *skdev,
2390 struct skd_request_context *skreq)
2391 {
2392 u8 cmp_status = skreq->completion.status;
2393
2394 switch (skd_check_status(skdev, cmp_status, &skreq->err_info)) {
2395 case SKD_CHECK_STATUS_REPORT_GOOD:
2396 case SKD_CHECK_STATUS_REPORT_SMART_ALERT:
2397 skd_end_request(skdev, skreq, 0);
2398 break;
2399
2400 case SKD_CHECK_STATUS_BUSY_IMMINENT:
2401 skd_log_skreq(skdev, skreq, "retry(busy)");
2402 blk_requeue_request(skdev->queue, skreq->req);
2403 pr_info("(%s) drive BUSY imminent\n", skd_name(skdev));
2404 skdev->state = SKD_DRVR_STATE_BUSY_IMMINENT;
2405 skdev->timer_countdown = SKD_TIMER_MINUTES(20);
2406 skd_quiesce_dev(skdev);
2407 break;
2408
2409 case SKD_CHECK_STATUS_REQUEUE_REQUEST:
2410 if ((unsigned long) ++skreq->req->special < SKD_MAX_RETRIES) {
2411 skd_log_skreq(skdev, skreq, "retry");
2412 blk_requeue_request(skdev->queue, skreq->req);
2413 break;
2414 }
2415 /* fall through to report error */
2416
2417 case SKD_CHECK_STATUS_REPORT_ERROR:
2418 default:
2419 skd_end_request(skdev, skreq, -EIO);
2420 break;
2421 }
2422 }
2423
2424 /* assume spinlock is already held */
skd_release_skreq(struct skd_device * skdev,struct skd_request_context * skreq)2425 static void skd_release_skreq(struct skd_device *skdev,
2426 struct skd_request_context *skreq)
2427 {
2428 u32 msg_slot;
2429 struct skd_fitmsg_context *skmsg;
2430
2431 u32 timo_slot;
2432
2433 /*
2434 * Reclaim the FIT msg buffer if this is
2435 * the first of the requests it carried to
2436 * be completed. The FIT msg buffer used to
2437 * send this request cannot be reused until
2438 * we are sure the s1120 card has copied
2439 * it to its memory. The FIT msg might have
2440 * contained several requests. As soon as
2441 * any of them are completed we know that
2442 * the entire FIT msg was transferred.
2443 * Only the first completed request will
2444 * match the FIT msg buffer id. The FIT
2445 * msg buffer id is immediately updated.
2446 * When subsequent requests complete the FIT
2447 * msg buffer id won't match, so we know
2448 * quite cheaply that it is already done.
2449 */
2450 msg_slot = skreq->fitmsg_id & SKD_ID_SLOT_MASK;
2451 SKD_ASSERT(msg_slot < skdev->num_fitmsg_context);
2452
2453 skmsg = &skdev->skmsg_table[msg_slot];
2454 if (skmsg->id == skreq->fitmsg_id) {
2455 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_BUSY);
2456 SKD_ASSERT(skmsg->outstanding > 0);
2457 skmsg->outstanding--;
2458 if (skmsg->outstanding == 0) {
2459 skmsg->state = SKD_MSG_STATE_IDLE;
2460 skmsg->id += SKD_ID_INCR;
2461 skmsg->next = skdev->skmsg_free_list;
2462 skdev->skmsg_free_list = skmsg;
2463 }
2464 }
2465
2466 /*
2467 * Decrease the number of active requests.
2468 * Also decrements the count in the timeout slot.
2469 */
2470 SKD_ASSERT(skdev->in_flight > 0);
2471 skdev->in_flight -= 1;
2472
2473 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
2474 SKD_ASSERT(skdev->timeout_slot[timo_slot] > 0);
2475 skdev->timeout_slot[timo_slot] -= 1;
2476
2477 /*
2478 * Reset backpointer
2479 */
2480 skreq->req = NULL;
2481
2482 /*
2483 * Reclaim the skd_request_context
2484 */
2485 skreq->state = SKD_REQ_STATE_IDLE;
2486 skreq->id += SKD_ID_INCR;
2487 skreq->next = skdev->skreq_free_list;
2488 skdev->skreq_free_list = skreq;
2489 }
2490
2491 #define DRIVER_INQ_EVPD_PAGE_CODE 0xDA
2492
skd_do_inq_page_00(struct skd_device * skdev,volatile struct fit_completion_entry_v1 * skcomp,volatile struct fit_comp_error_info * skerr,uint8_t * cdb,uint8_t * buf)2493 static void skd_do_inq_page_00(struct skd_device *skdev,
2494 volatile struct fit_completion_entry_v1 *skcomp,
2495 volatile struct fit_comp_error_info *skerr,
2496 uint8_t *cdb, uint8_t *buf)
2497 {
2498 uint16_t insert_pt, max_bytes, drive_pages, drive_bytes, new_size;
2499
2500 /* Caller requested "supported pages". The driver needs to insert
2501 * its page.
2502 */
2503 pr_debug("%s:%s:%d skd_do_driver_inquiry: modify supported pages.\n",
2504 skdev->name, __func__, __LINE__);
2505
2506 /* If the device rejected the request because the CDB was
2507 * improperly formed, then just leave.
2508 */
2509 if (skcomp->status == SAM_STAT_CHECK_CONDITION &&
2510 skerr->key == ILLEGAL_REQUEST && skerr->code == 0x24)
2511 return;
2512
2513 /* Get the amount of space the caller allocated */
2514 max_bytes = (cdb[3] << 8) | cdb[4];
2515
2516 /* Get the number of pages actually returned by the device */
2517 drive_pages = (buf[2] << 8) | buf[3];
2518 drive_bytes = drive_pages + 4;
2519 new_size = drive_pages + 1;
2520
2521 /* Supported pages must be in numerical order, so find where
2522 * the driver page needs to be inserted into the list of
2523 * pages returned by the device.
2524 */
2525 for (insert_pt = 4; insert_pt < drive_bytes; insert_pt++) {
2526 if (buf[insert_pt] == DRIVER_INQ_EVPD_PAGE_CODE)
2527 return; /* Device using this page code. abort */
2528 else if (buf[insert_pt] > DRIVER_INQ_EVPD_PAGE_CODE)
2529 break;
2530 }
2531
2532 if (insert_pt < max_bytes) {
2533 uint16_t u;
2534
2535 /* Shift everything up one byte to make room. */
2536 for (u = new_size + 3; u > insert_pt; u--)
2537 buf[u] = buf[u - 1];
2538 buf[insert_pt] = DRIVER_INQ_EVPD_PAGE_CODE;
2539
2540 /* SCSI byte order increment of num_returned_bytes by 1 */
2541 skcomp->num_returned_bytes =
2542 be32_to_cpu(skcomp->num_returned_bytes) + 1;
2543 skcomp->num_returned_bytes =
2544 be32_to_cpu(skcomp->num_returned_bytes);
2545 }
2546
2547 /* update page length field to reflect the driver's page too */
2548 buf[2] = (uint8_t)((new_size >> 8) & 0xFF);
2549 buf[3] = (uint8_t)((new_size >> 0) & 0xFF);
2550 }
2551
skd_get_link_info(struct pci_dev * pdev,u8 * speed,u8 * width)2552 static void skd_get_link_info(struct pci_dev *pdev, u8 *speed, u8 *width)
2553 {
2554 int pcie_reg;
2555 u16 pci_bus_speed;
2556 u8 pci_lanes;
2557
2558 pcie_reg = pci_find_capability(pdev, PCI_CAP_ID_EXP);
2559 if (pcie_reg) {
2560 u16 linksta;
2561 pci_read_config_word(pdev, pcie_reg + PCI_EXP_LNKSTA, &linksta);
2562
2563 pci_bus_speed = linksta & 0xF;
2564 pci_lanes = (linksta & 0x3F0) >> 4;
2565 } else {
2566 *speed = STEC_LINK_UNKNOWN;
2567 *width = 0xFF;
2568 return;
2569 }
2570
2571 switch (pci_bus_speed) {
2572 case 1:
2573 *speed = STEC_LINK_2_5GTS;
2574 break;
2575 case 2:
2576 *speed = STEC_LINK_5GTS;
2577 break;
2578 case 3:
2579 *speed = STEC_LINK_8GTS;
2580 break;
2581 default:
2582 *speed = STEC_LINK_UNKNOWN;
2583 break;
2584 }
2585
2586 if (pci_lanes <= 0x20)
2587 *width = pci_lanes;
2588 else
2589 *width = 0xFF;
2590 }
2591
skd_do_inq_page_da(struct skd_device * skdev,volatile struct fit_completion_entry_v1 * skcomp,volatile struct fit_comp_error_info * skerr,uint8_t * cdb,uint8_t * buf)2592 static void skd_do_inq_page_da(struct skd_device *skdev,
2593 volatile struct fit_completion_entry_v1 *skcomp,
2594 volatile struct fit_comp_error_info *skerr,
2595 uint8_t *cdb, uint8_t *buf)
2596 {
2597 struct pci_dev *pdev = skdev->pdev;
2598 unsigned max_bytes;
2599 struct driver_inquiry_data inq;
2600 u16 val;
2601
2602 pr_debug("%s:%s:%d skd_do_driver_inquiry: return driver page\n",
2603 skdev->name, __func__, __LINE__);
2604
2605 memset(&inq, 0, sizeof(inq));
2606
2607 inq.page_code = DRIVER_INQ_EVPD_PAGE_CODE;
2608
2609 skd_get_link_info(pdev, &inq.pcie_link_speed, &inq.pcie_link_lanes);
2610 inq.pcie_bus_number = cpu_to_be16(pdev->bus->number);
2611 inq.pcie_device_number = PCI_SLOT(pdev->devfn);
2612 inq.pcie_function_number = PCI_FUNC(pdev->devfn);
2613
2614 pci_read_config_word(pdev, PCI_VENDOR_ID, &val);
2615 inq.pcie_vendor_id = cpu_to_be16(val);
2616
2617 pci_read_config_word(pdev, PCI_DEVICE_ID, &val);
2618 inq.pcie_device_id = cpu_to_be16(val);
2619
2620 pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, &val);
2621 inq.pcie_subsystem_vendor_id = cpu_to_be16(val);
2622
2623 pci_read_config_word(pdev, PCI_SUBSYSTEM_ID, &val);
2624 inq.pcie_subsystem_device_id = cpu_to_be16(val);
2625
2626 /* Driver version, fixed lenth, padded with spaces on the right */
2627 inq.driver_version_length = sizeof(inq.driver_version);
2628 memset(&inq.driver_version, ' ', sizeof(inq.driver_version));
2629 memcpy(inq.driver_version, DRV_VER_COMPL,
2630 min(sizeof(inq.driver_version), strlen(DRV_VER_COMPL)));
2631
2632 inq.page_length = cpu_to_be16((sizeof(inq) - 4));
2633
2634 /* Clear the error set by the device */
2635 skcomp->status = SAM_STAT_GOOD;
2636 memset((void *)skerr, 0, sizeof(*skerr));
2637
2638 /* copy response into output buffer */
2639 max_bytes = (cdb[3] << 8) | cdb[4];
2640 memcpy(buf, &inq, min_t(unsigned, max_bytes, sizeof(inq)));
2641
2642 skcomp->num_returned_bytes =
2643 be32_to_cpu(min_t(uint16_t, max_bytes, sizeof(inq)));
2644 }
2645
skd_do_driver_inq(struct skd_device * skdev,volatile struct fit_completion_entry_v1 * skcomp,volatile struct fit_comp_error_info * skerr,uint8_t * cdb,uint8_t * buf)2646 static void skd_do_driver_inq(struct skd_device *skdev,
2647 volatile struct fit_completion_entry_v1 *skcomp,
2648 volatile struct fit_comp_error_info *skerr,
2649 uint8_t *cdb, uint8_t *buf)
2650 {
2651 if (!buf)
2652 return;
2653 else if (cdb[0] != INQUIRY)
2654 return; /* Not an INQUIRY */
2655 else if ((cdb[1] & 1) == 0)
2656 return; /* EVPD not set */
2657 else if (cdb[2] == 0)
2658 /* Need to add driver's page to supported pages list */
2659 skd_do_inq_page_00(skdev, skcomp, skerr, cdb, buf);
2660 else if (cdb[2] == DRIVER_INQ_EVPD_PAGE_CODE)
2661 /* Caller requested driver's page */
2662 skd_do_inq_page_da(skdev, skcomp, skerr, cdb, buf);
2663 }
2664
skd_sg_1st_page_ptr(struct scatterlist * sg)2665 static unsigned char *skd_sg_1st_page_ptr(struct scatterlist *sg)
2666 {
2667 if (!sg)
2668 return NULL;
2669 if (!sg_page(sg))
2670 return NULL;
2671 return sg_virt(sg);
2672 }
2673
skd_process_scsi_inq(struct skd_device * skdev,volatile struct fit_completion_entry_v1 * skcomp,volatile struct fit_comp_error_info * skerr,struct skd_special_context * skspcl)2674 static void skd_process_scsi_inq(struct skd_device *skdev,
2675 volatile struct fit_completion_entry_v1
2676 *skcomp,
2677 volatile struct fit_comp_error_info *skerr,
2678 struct skd_special_context *skspcl)
2679 {
2680 uint8_t *buf;
2681 struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
2682 struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
2683
2684 dma_sync_sg_for_cpu(skdev->class_dev, skspcl->req.sg, skspcl->req.n_sg,
2685 skspcl->req.sg_data_dir);
2686 buf = skd_sg_1st_page_ptr(skspcl->req.sg);
2687
2688 if (buf)
2689 skd_do_driver_inq(skdev, skcomp, skerr, scsi_req->cdb, buf);
2690 }
2691
2692
skd_isr_completion_posted(struct skd_device * skdev,int limit,int * enqueued)2693 static int skd_isr_completion_posted(struct skd_device *skdev,
2694 int limit, int *enqueued)
2695 {
2696 volatile struct fit_completion_entry_v1 *skcmp = NULL;
2697 volatile struct fit_comp_error_info *skerr;
2698 u16 req_id;
2699 u32 req_slot;
2700 struct skd_request_context *skreq;
2701 u16 cmp_cntxt = 0;
2702 u8 cmp_status = 0;
2703 u8 cmp_cycle = 0;
2704 u32 cmp_bytes = 0;
2705 int rc = 0;
2706 int processed = 0;
2707
2708 for (;; ) {
2709 SKD_ASSERT(skdev->skcomp_ix < SKD_N_COMPLETION_ENTRY);
2710
2711 skcmp = &skdev->skcomp_table[skdev->skcomp_ix];
2712 cmp_cycle = skcmp->cycle;
2713 cmp_cntxt = skcmp->tag;
2714 cmp_status = skcmp->status;
2715 cmp_bytes = be32_to_cpu(skcmp->num_returned_bytes);
2716
2717 skerr = &skdev->skerr_table[skdev->skcomp_ix];
2718
2719 pr_debug("%s:%s:%d "
2720 "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d "
2721 "busy=%d rbytes=0x%x proto=%d\n",
2722 skdev->name, __func__, __LINE__, skdev->skcomp_cycle,
2723 skdev->skcomp_ix, cmp_cycle, cmp_cntxt, cmp_status,
2724 skdev->in_flight, cmp_bytes, skdev->proto_ver);
2725
2726 if (cmp_cycle != skdev->skcomp_cycle) {
2727 pr_debug("%s:%s:%d end of completions\n",
2728 skdev->name, __func__, __LINE__);
2729 break;
2730 }
2731 /*
2732 * Update the completion queue head index and possibly
2733 * the completion cycle count. 8-bit wrap-around.
2734 */
2735 skdev->skcomp_ix++;
2736 if (skdev->skcomp_ix >= SKD_N_COMPLETION_ENTRY) {
2737 skdev->skcomp_ix = 0;
2738 skdev->skcomp_cycle++;
2739 }
2740
2741 /*
2742 * The command context is a unique 32-bit ID. The low order
2743 * bits help locate the request. The request is usually a
2744 * r/w request (see skd_start() above) or a special request.
2745 */
2746 req_id = cmp_cntxt;
2747 req_slot = req_id & SKD_ID_SLOT_AND_TABLE_MASK;
2748
2749 /* Is this other than a r/w request? */
2750 if (req_slot >= skdev->num_req_context) {
2751 /*
2752 * This is not a completion for a r/w request.
2753 */
2754 skd_complete_other(skdev, skcmp, skerr);
2755 continue;
2756 }
2757
2758 skreq = &skdev->skreq_table[req_slot];
2759
2760 /*
2761 * Make sure the request ID for the slot matches.
2762 */
2763 if (skreq->id != req_id) {
2764 pr_debug("%s:%s:%d mismatch comp_id=0x%x req_id=0x%x\n",
2765 skdev->name, __func__, __LINE__,
2766 req_id, skreq->id);
2767 {
2768 u16 new_id = cmp_cntxt;
2769 pr_err("(%s): Completion mismatch "
2770 "comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
2771 skd_name(skdev), req_id,
2772 skreq->id, new_id);
2773
2774 continue;
2775 }
2776 }
2777
2778 SKD_ASSERT(skreq->state == SKD_REQ_STATE_BUSY);
2779
2780 if (skreq->state == SKD_REQ_STATE_ABORTED) {
2781 pr_debug("%s:%s:%d reclaim req %p id=%04x\n",
2782 skdev->name, __func__, __LINE__,
2783 skreq, skreq->id);
2784 /* a previously timed out command can
2785 * now be cleaned up */
2786 skd_release_skreq(skdev, skreq);
2787 continue;
2788 }
2789
2790 skreq->completion = *skcmp;
2791 if (unlikely(cmp_status == SAM_STAT_CHECK_CONDITION)) {
2792 skreq->err_info = *skerr;
2793 skd_log_check_status(skdev, cmp_status, skerr->key,
2794 skerr->code, skerr->qual,
2795 skerr->fruc);
2796 }
2797 /* Release DMA resources for the request. */
2798 if (skreq->n_sg > 0)
2799 skd_postop_sg_list(skdev, skreq);
2800
2801 if (!skreq->req) {
2802 pr_debug("%s:%s:%d NULL backptr skdreq %p, "
2803 "req=0x%x req_id=0x%x\n",
2804 skdev->name, __func__, __LINE__,
2805 skreq, skreq->id, req_id);
2806 } else {
2807 /*
2808 * Capture the outcome and post it back to the
2809 * native request.
2810 */
2811 if (likely(cmp_status == SAM_STAT_GOOD))
2812 skd_end_request(skdev, skreq, 0);
2813 else
2814 skd_resolve_req_exception(skdev, skreq);
2815 }
2816
2817 /*
2818 * Release the skreq, its FIT msg (if one), timeout slot,
2819 * and queue depth.
2820 */
2821 skd_release_skreq(skdev, skreq);
2822
2823 /* skd_isr_comp_limit equal zero means no limit */
2824 if (limit) {
2825 if (++processed >= limit) {
2826 rc = 1;
2827 break;
2828 }
2829 }
2830 }
2831
2832 if ((skdev->state == SKD_DRVR_STATE_PAUSING)
2833 && (skdev->in_flight) == 0) {
2834 skdev->state = SKD_DRVR_STATE_PAUSED;
2835 wake_up_interruptible(&skdev->waitq);
2836 }
2837
2838 return rc;
2839 }
2840
skd_complete_other(struct skd_device * skdev,volatile struct fit_completion_entry_v1 * skcomp,volatile struct fit_comp_error_info * skerr)2841 static void skd_complete_other(struct skd_device *skdev,
2842 volatile struct fit_completion_entry_v1 *skcomp,
2843 volatile struct fit_comp_error_info *skerr)
2844 {
2845 u32 req_id = 0;
2846 u32 req_table;
2847 u32 req_slot;
2848 struct skd_special_context *skspcl;
2849
2850 req_id = skcomp->tag;
2851 req_table = req_id & SKD_ID_TABLE_MASK;
2852 req_slot = req_id & SKD_ID_SLOT_MASK;
2853
2854 pr_debug("%s:%s:%d table=0x%x id=0x%x slot=%d\n",
2855 skdev->name, __func__, __LINE__,
2856 req_table, req_id, req_slot);
2857
2858 /*
2859 * Based on the request id, determine how to dispatch this completion.
2860 * This swich/case is finding the good cases and forwarding the
2861 * completion entry. Errors are reported below the switch.
2862 */
2863 switch (req_table) {
2864 case SKD_ID_RW_REQUEST:
2865 /*
2866 * The caller, skd_completion_posted_isr() above,
2867 * handles r/w requests. The only way we get here
2868 * is if the req_slot is out of bounds.
2869 */
2870 break;
2871
2872 case SKD_ID_SPECIAL_REQUEST:
2873 /*
2874 * Make sure the req_slot is in bounds and that the id
2875 * matches.
2876 */
2877 if (req_slot < skdev->n_special) {
2878 skspcl = &skdev->skspcl_table[req_slot];
2879 if (skspcl->req.id == req_id &&
2880 skspcl->req.state == SKD_REQ_STATE_BUSY) {
2881 skd_complete_special(skdev,
2882 skcomp, skerr, skspcl);
2883 return;
2884 }
2885 }
2886 break;
2887
2888 case SKD_ID_INTERNAL:
2889 if (req_slot == 0) {
2890 skspcl = &skdev->internal_skspcl;
2891 if (skspcl->req.id == req_id &&
2892 skspcl->req.state == SKD_REQ_STATE_BUSY) {
2893 skd_complete_internal(skdev,
2894 skcomp, skerr, skspcl);
2895 return;
2896 }
2897 }
2898 break;
2899
2900 case SKD_ID_FIT_MSG:
2901 /*
2902 * These id's should never appear in a completion record.
2903 */
2904 break;
2905
2906 default:
2907 /*
2908 * These id's should never appear anywhere;
2909 */
2910 break;
2911 }
2912
2913 /*
2914 * If we get here it is a bad or stale id.
2915 */
2916 }
2917
skd_complete_special(struct skd_device * skdev,volatile struct fit_completion_entry_v1 * skcomp,volatile struct fit_comp_error_info * skerr,struct skd_special_context * skspcl)2918 static void skd_complete_special(struct skd_device *skdev,
2919 volatile struct fit_completion_entry_v1
2920 *skcomp,
2921 volatile struct fit_comp_error_info *skerr,
2922 struct skd_special_context *skspcl)
2923 {
2924 pr_debug("%s:%s:%d completing special request %p\n",
2925 skdev->name, __func__, __LINE__, skspcl);
2926 if (skspcl->orphaned) {
2927 /* Discard orphaned request */
2928 /* ?: Can this release directly or does it need
2929 * to use a worker? */
2930 pr_debug("%s:%s:%d release orphaned %p\n",
2931 skdev->name, __func__, __LINE__, skspcl);
2932 skd_release_special(skdev, skspcl);
2933 return;
2934 }
2935
2936 skd_process_scsi_inq(skdev, skcomp, skerr, skspcl);
2937
2938 skspcl->req.state = SKD_REQ_STATE_COMPLETED;
2939 skspcl->req.completion = *skcomp;
2940 skspcl->req.err_info = *skerr;
2941
2942 skd_log_check_status(skdev, skspcl->req.completion.status, skerr->key,
2943 skerr->code, skerr->qual, skerr->fruc);
2944
2945 wake_up_interruptible(&skdev->waitq);
2946 }
2947
2948 /* assume spinlock is already held */
skd_release_special(struct skd_device * skdev,struct skd_special_context * skspcl)2949 static void skd_release_special(struct skd_device *skdev,
2950 struct skd_special_context *skspcl)
2951 {
2952 int i, was_depleted;
2953
2954 for (i = 0; i < skspcl->req.n_sg; i++) {
2955 struct page *page = sg_page(&skspcl->req.sg[i]);
2956 __free_page(page);
2957 }
2958
2959 was_depleted = (skdev->skspcl_free_list == NULL);
2960
2961 skspcl->req.state = SKD_REQ_STATE_IDLE;
2962 skspcl->req.id += SKD_ID_INCR;
2963 skspcl->req.next =
2964 (struct skd_request_context *)skdev->skspcl_free_list;
2965 skdev->skspcl_free_list = (struct skd_special_context *)skspcl;
2966
2967 if (was_depleted) {
2968 pr_debug("%s:%s:%d skspcl was depleted\n",
2969 skdev->name, __func__, __LINE__);
2970 /* Free list was depleted. Their might be waiters. */
2971 wake_up_interruptible(&skdev->waitq);
2972 }
2973 }
2974
skd_reset_skcomp(struct skd_device * skdev)2975 static void skd_reset_skcomp(struct skd_device *skdev)
2976 {
2977 u32 nbytes;
2978 struct fit_completion_entry_v1 *skcomp;
2979
2980 nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
2981 nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
2982
2983 memset(skdev->skcomp_table, 0, nbytes);
2984
2985 skdev->skcomp_ix = 0;
2986 skdev->skcomp_cycle = 1;
2987 }
2988
2989 /*
2990 *****************************************************************************
2991 * INTERRUPTS
2992 *****************************************************************************
2993 */
skd_completion_worker(struct work_struct * work)2994 static void skd_completion_worker(struct work_struct *work)
2995 {
2996 struct skd_device *skdev =
2997 container_of(work, struct skd_device, completion_worker);
2998 unsigned long flags;
2999 int flush_enqueued = 0;
3000
3001 spin_lock_irqsave(&skdev->lock, flags);
3002
3003 /*
3004 * pass in limit=0, which means no limit..
3005 * process everything in compq
3006 */
3007 skd_isr_completion_posted(skdev, 0, &flush_enqueued);
3008 skd_request_fn(skdev->queue);
3009
3010 spin_unlock_irqrestore(&skdev->lock, flags);
3011 }
3012
3013 static void skd_isr_msg_from_dev(struct skd_device *skdev);
3014
3015 irqreturn_t
skd_isr(int irq,void * ptr)3016 static skd_isr(int irq, void *ptr)
3017 {
3018 struct skd_device *skdev;
3019 u32 intstat;
3020 u32 ack;
3021 int rc = 0;
3022 int deferred = 0;
3023 int flush_enqueued = 0;
3024
3025 skdev = (struct skd_device *)ptr;
3026 spin_lock(&skdev->lock);
3027
3028 for (;; ) {
3029 intstat = SKD_READL(skdev, FIT_INT_STATUS_HOST);
3030
3031 ack = FIT_INT_DEF_MASK;
3032 ack &= intstat;
3033
3034 pr_debug("%s:%s:%d intstat=0x%x ack=0x%x\n",
3035 skdev->name, __func__, __LINE__, intstat, ack);
3036
3037 /* As long as there is an int pending on device, keep
3038 * running loop. When none, get out, but if we've never
3039 * done any processing, call completion handler?
3040 */
3041 if (ack == 0) {
3042 /* No interrupts on device, but run the completion
3043 * processor anyway?
3044 */
3045 if (rc == 0)
3046 if (likely (skdev->state
3047 == SKD_DRVR_STATE_ONLINE))
3048 deferred = 1;
3049 break;
3050 }
3051
3052 rc = IRQ_HANDLED;
3053
3054 SKD_WRITEL(skdev, ack, FIT_INT_STATUS_HOST);
3055
3056 if (likely((skdev->state != SKD_DRVR_STATE_LOAD) &&
3057 (skdev->state != SKD_DRVR_STATE_STOPPING))) {
3058 if (intstat & FIT_ISH_COMPLETION_POSTED) {
3059 /*
3060 * If we have already deferred completion
3061 * processing, don't bother running it again
3062 */
3063 if (deferred == 0)
3064 deferred =
3065 skd_isr_completion_posted(skdev,
3066 skd_isr_comp_limit, &flush_enqueued);
3067 }
3068
3069 if (intstat & FIT_ISH_FW_STATE_CHANGE) {
3070 skd_isr_fwstate(skdev);
3071 if (skdev->state == SKD_DRVR_STATE_FAULT ||
3072 skdev->state ==
3073 SKD_DRVR_STATE_DISAPPEARED) {
3074 spin_unlock(&skdev->lock);
3075 return rc;
3076 }
3077 }
3078
3079 if (intstat & FIT_ISH_MSG_FROM_DEV)
3080 skd_isr_msg_from_dev(skdev);
3081 }
3082 }
3083
3084 if (unlikely(flush_enqueued))
3085 skd_request_fn(skdev->queue);
3086
3087 if (deferred)
3088 schedule_work(&skdev->completion_worker);
3089 else if (!flush_enqueued)
3090 skd_request_fn(skdev->queue);
3091
3092 spin_unlock(&skdev->lock);
3093
3094 return rc;
3095 }
3096
skd_drive_fault(struct skd_device * skdev)3097 static void skd_drive_fault(struct skd_device *skdev)
3098 {
3099 skdev->state = SKD_DRVR_STATE_FAULT;
3100 pr_err("(%s): Drive FAULT\n", skd_name(skdev));
3101 }
3102
skd_drive_disappeared(struct skd_device * skdev)3103 static void skd_drive_disappeared(struct skd_device *skdev)
3104 {
3105 skdev->state = SKD_DRVR_STATE_DISAPPEARED;
3106 pr_err("(%s): Drive DISAPPEARED\n", skd_name(skdev));
3107 }
3108
skd_isr_fwstate(struct skd_device * skdev)3109 static void skd_isr_fwstate(struct skd_device *skdev)
3110 {
3111 u32 sense;
3112 u32 state;
3113 u32 mtd;
3114 int prev_driver_state = skdev->state;
3115
3116 sense = SKD_READL(skdev, FIT_STATUS);
3117 state = sense & FIT_SR_DRIVE_STATE_MASK;
3118
3119 pr_err("(%s): s1120 state %s(%d)=>%s(%d)\n",
3120 skd_name(skdev),
3121 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
3122 skd_drive_state_to_str(state), state);
3123
3124 skdev->drive_state = state;
3125
3126 switch (skdev->drive_state) {
3127 case FIT_SR_DRIVE_INIT:
3128 if (skdev->state == SKD_DRVR_STATE_PROTOCOL_MISMATCH) {
3129 skd_disable_interrupts(skdev);
3130 break;
3131 }
3132 if (skdev->state == SKD_DRVR_STATE_RESTARTING)
3133 skd_recover_requests(skdev, 0);
3134 if (skdev->state == SKD_DRVR_STATE_WAIT_BOOT) {
3135 skdev->timer_countdown = SKD_STARTING_TIMO;
3136 skdev->state = SKD_DRVR_STATE_STARTING;
3137 skd_soft_reset(skdev);
3138 break;
3139 }
3140 mtd = FIT_MXD_CONS(FIT_MTD_FITFW_INIT, 0, 0);
3141 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3142 skdev->last_mtd = mtd;
3143 break;
3144
3145 case FIT_SR_DRIVE_ONLINE:
3146 skdev->cur_max_queue_depth = skd_max_queue_depth;
3147 if (skdev->cur_max_queue_depth > skdev->dev_max_queue_depth)
3148 skdev->cur_max_queue_depth = skdev->dev_max_queue_depth;
3149
3150 skdev->queue_low_water_mark =
3151 skdev->cur_max_queue_depth * 2 / 3 + 1;
3152 if (skdev->queue_low_water_mark < 1)
3153 skdev->queue_low_water_mark = 1;
3154 pr_info(
3155 "(%s): Queue depth limit=%d dev=%d lowat=%d\n",
3156 skd_name(skdev),
3157 skdev->cur_max_queue_depth,
3158 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
3159
3160 skd_refresh_device_data(skdev);
3161 break;
3162
3163 case FIT_SR_DRIVE_BUSY:
3164 skdev->state = SKD_DRVR_STATE_BUSY;
3165 skdev->timer_countdown = SKD_BUSY_TIMO;
3166 skd_quiesce_dev(skdev);
3167 break;
3168 case FIT_SR_DRIVE_BUSY_SANITIZE:
3169 /* set timer for 3 seconds, we'll abort any unfinished
3170 * commands after that expires
3171 */
3172 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3173 skdev->timer_countdown = SKD_TIMER_SECONDS(3);
3174 blk_start_queue(skdev->queue);
3175 break;
3176 case FIT_SR_DRIVE_BUSY_ERASE:
3177 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3178 skdev->timer_countdown = SKD_BUSY_TIMO;
3179 break;
3180 case FIT_SR_DRIVE_OFFLINE:
3181 skdev->state = SKD_DRVR_STATE_IDLE;
3182 break;
3183 case FIT_SR_DRIVE_SOFT_RESET:
3184 switch (skdev->state) {
3185 case SKD_DRVR_STATE_STARTING:
3186 case SKD_DRVR_STATE_RESTARTING:
3187 /* Expected by a caller of skd_soft_reset() */
3188 break;
3189 default:
3190 skdev->state = SKD_DRVR_STATE_RESTARTING;
3191 break;
3192 }
3193 break;
3194 case FIT_SR_DRIVE_FW_BOOTING:
3195 pr_debug("%s:%s:%d ISR FIT_SR_DRIVE_FW_BOOTING %s\n",
3196 skdev->name, __func__, __LINE__, skdev->name);
3197 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3198 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3199 break;
3200
3201 case FIT_SR_DRIVE_DEGRADED:
3202 case FIT_SR_PCIE_LINK_DOWN:
3203 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
3204 break;
3205
3206 case FIT_SR_DRIVE_FAULT:
3207 skd_drive_fault(skdev);
3208 skd_recover_requests(skdev, 0);
3209 blk_start_queue(skdev->queue);
3210 break;
3211
3212 /* PCIe bus returned all Fs? */
3213 case 0xFF:
3214 pr_info("(%s): state=0x%x sense=0x%x\n",
3215 skd_name(skdev), state, sense);
3216 skd_drive_disappeared(skdev);
3217 skd_recover_requests(skdev, 0);
3218 blk_start_queue(skdev->queue);
3219 break;
3220 default:
3221 /*
3222 * Uknown FW State. Wait for a state we recognize.
3223 */
3224 break;
3225 }
3226 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3227 skd_name(skdev),
3228 skd_skdev_state_to_str(prev_driver_state), prev_driver_state,
3229 skd_skdev_state_to_str(skdev->state), skdev->state);
3230 }
3231
skd_recover_requests(struct skd_device * skdev,int requeue)3232 static void skd_recover_requests(struct skd_device *skdev, int requeue)
3233 {
3234 int i;
3235
3236 for (i = 0; i < skdev->num_req_context; i++) {
3237 struct skd_request_context *skreq = &skdev->skreq_table[i];
3238
3239 if (skreq->state == SKD_REQ_STATE_BUSY) {
3240 skd_log_skreq(skdev, skreq, "recover");
3241
3242 SKD_ASSERT((skreq->id & SKD_ID_INCR) != 0);
3243 SKD_ASSERT(skreq->req != NULL);
3244
3245 /* Release DMA resources for the request. */
3246 if (skreq->n_sg > 0)
3247 skd_postop_sg_list(skdev, skreq);
3248
3249 if (requeue &&
3250 (unsigned long) ++skreq->req->special <
3251 SKD_MAX_RETRIES)
3252 blk_requeue_request(skdev->queue, skreq->req);
3253 else
3254 skd_end_request(skdev, skreq, -EIO);
3255
3256 skreq->req = NULL;
3257
3258 skreq->state = SKD_REQ_STATE_IDLE;
3259 skreq->id += SKD_ID_INCR;
3260 }
3261 if (i > 0)
3262 skreq[-1].next = skreq;
3263 skreq->next = NULL;
3264 }
3265 skdev->skreq_free_list = skdev->skreq_table;
3266
3267 for (i = 0; i < skdev->num_fitmsg_context; i++) {
3268 struct skd_fitmsg_context *skmsg = &skdev->skmsg_table[i];
3269
3270 if (skmsg->state == SKD_MSG_STATE_BUSY) {
3271 skd_log_skmsg(skdev, skmsg, "salvaged");
3272 SKD_ASSERT((skmsg->id & SKD_ID_INCR) != 0);
3273 skmsg->state = SKD_MSG_STATE_IDLE;
3274 skmsg->id += SKD_ID_INCR;
3275 }
3276 if (i > 0)
3277 skmsg[-1].next = skmsg;
3278 skmsg->next = NULL;
3279 }
3280 skdev->skmsg_free_list = skdev->skmsg_table;
3281
3282 for (i = 0; i < skdev->n_special; i++) {
3283 struct skd_special_context *skspcl = &skdev->skspcl_table[i];
3284
3285 /* If orphaned, reclaim it because it has already been reported
3286 * to the process as an error (it was just waiting for
3287 * a completion that didn't come, and now it will never come)
3288 * If busy, change to a state that will cause it to error
3289 * out in the wait routine and let it do the normal
3290 * reporting and reclaiming
3291 */
3292 if (skspcl->req.state == SKD_REQ_STATE_BUSY) {
3293 if (skspcl->orphaned) {
3294 pr_debug("%s:%s:%d orphaned %p\n",
3295 skdev->name, __func__, __LINE__,
3296 skspcl);
3297 skd_release_special(skdev, skspcl);
3298 } else {
3299 pr_debug("%s:%s:%d not orphaned %p\n",
3300 skdev->name, __func__, __LINE__,
3301 skspcl);
3302 skspcl->req.state = SKD_REQ_STATE_ABORTED;
3303 }
3304 }
3305 }
3306 skdev->skspcl_free_list = skdev->skspcl_table;
3307
3308 for (i = 0; i < SKD_N_TIMEOUT_SLOT; i++)
3309 skdev->timeout_slot[i] = 0;
3310
3311 skdev->in_flight = 0;
3312 }
3313
skd_isr_msg_from_dev(struct skd_device * skdev)3314 static void skd_isr_msg_from_dev(struct skd_device *skdev)
3315 {
3316 u32 mfd;
3317 u32 mtd;
3318 u32 data;
3319
3320 mfd = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
3321
3322 pr_debug("%s:%s:%d mfd=0x%x last_mtd=0x%x\n",
3323 skdev->name, __func__, __LINE__, mfd, skdev->last_mtd);
3324
3325 /* ignore any mtd that is an ack for something we didn't send */
3326 if (FIT_MXD_TYPE(mfd) != FIT_MXD_TYPE(skdev->last_mtd))
3327 return;
3328
3329 switch (FIT_MXD_TYPE(mfd)) {
3330 case FIT_MTD_FITFW_INIT:
3331 skdev->proto_ver = FIT_PROTOCOL_MAJOR_VER(mfd);
3332
3333 if (skdev->proto_ver != FIT_PROTOCOL_VERSION_1) {
3334 pr_err("(%s): protocol mismatch\n",
3335 skdev->name);
3336 pr_err("(%s): got=%d support=%d\n",
3337 skdev->name, skdev->proto_ver,
3338 FIT_PROTOCOL_VERSION_1);
3339 pr_err("(%s): please upgrade driver\n",
3340 skdev->name);
3341 skdev->state = SKD_DRVR_STATE_PROTOCOL_MISMATCH;
3342 skd_soft_reset(skdev);
3343 break;
3344 }
3345 mtd = FIT_MXD_CONS(FIT_MTD_GET_CMDQ_DEPTH, 0, 0);
3346 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3347 skdev->last_mtd = mtd;
3348 break;
3349
3350 case FIT_MTD_GET_CMDQ_DEPTH:
3351 skdev->dev_max_queue_depth = FIT_MXD_DATA(mfd);
3352 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_DEPTH, 0,
3353 SKD_N_COMPLETION_ENTRY);
3354 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3355 skdev->last_mtd = mtd;
3356 break;
3357
3358 case FIT_MTD_SET_COMPQ_DEPTH:
3359 SKD_WRITEQ(skdev, skdev->cq_dma_address, FIT_MSG_TO_DEVICE_ARG);
3360 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_ADDR, 0, 0);
3361 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3362 skdev->last_mtd = mtd;
3363 break;
3364
3365 case FIT_MTD_SET_COMPQ_ADDR:
3366 skd_reset_skcomp(skdev);
3367 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_HOST_ID, 0, skdev->devno);
3368 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3369 skdev->last_mtd = mtd;
3370 break;
3371
3372 case FIT_MTD_CMD_LOG_HOST_ID:
3373 skdev->connect_time_stamp = get_seconds();
3374 data = skdev->connect_time_stamp & 0xFFFF;
3375 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_LO, 0, data);
3376 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3377 skdev->last_mtd = mtd;
3378 break;
3379
3380 case FIT_MTD_CMD_LOG_TIME_STAMP_LO:
3381 skdev->drive_jiffies = FIT_MXD_DATA(mfd);
3382 data = (skdev->connect_time_stamp >> 16) & 0xFFFF;
3383 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_HI, 0, data);
3384 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3385 skdev->last_mtd = mtd;
3386 break;
3387
3388 case FIT_MTD_CMD_LOG_TIME_STAMP_HI:
3389 skdev->drive_jiffies |= (FIT_MXD_DATA(mfd) << 16);
3390 mtd = FIT_MXD_CONS(FIT_MTD_ARM_QUEUE, 0, 0);
3391 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3392 skdev->last_mtd = mtd;
3393
3394 pr_err("(%s): Time sync driver=0x%x device=0x%x\n",
3395 skd_name(skdev),
3396 skdev->connect_time_stamp, skdev->drive_jiffies);
3397 break;
3398
3399 case FIT_MTD_ARM_QUEUE:
3400 skdev->last_mtd = 0;
3401 /*
3402 * State should be, or soon will be, FIT_SR_DRIVE_ONLINE.
3403 */
3404 break;
3405
3406 default:
3407 break;
3408 }
3409 }
3410
skd_disable_interrupts(struct skd_device * skdev)3411 static void skd_disable_interrupts(struct skd_device *skdev)
3412 {
3413 u32 sense;
3414
3415 sense = SKD_READL(skdev, FIT_CONTROL);
3416 sense &= ~FIT_CR_ENABLE_INTERRUPTS;
3417 SKD_WRITEL(skdev, sense, FIT_CONTROL);
3418 pr_debug("%s:%s:%d sense 0x%x\n",
3419 skdev->name, __func__, __LINE__, sense);
3420
3421 /* Note that the 1s is written. A 1-bit means
3422 * disable, a 0 means enable.
3423 */
3424 SKD_WRITEL(skdev, ~0, FIT_INT_MASK_HOST);
3425 }
3426
skd_enable_interrupts(struct skd_device * skdev)3427 static void skd_enable_interrupts(struct skd_device *skdev)
3428 {
3429 u32 val;
3430
3431 /* unmask interrupts first */
3432 val = FIT_ISH_FW_STATE_CHANGE +
3433 FIT_ISH_COMPLETION_POSTED + FIT_ISH_MSG_FROM_DEV;
3434
3435 /* Note that the compliment of mask is written. A 1-bit means
3436 * disable, a 0 means enable. */
3437 SKD_WRITEL(skdev, ~val, FIT_INT_MASK_HOST);
3438 pr_debug("%s:%s:%d interrupt mask=0x%x\n",
3439 skdev->name, __func__, __LINE__, ~val);
3440
3441 val = SKD_READL(skdev, FIT_CONTROL);
3442 val |= FIT_CR_ENABLE_INTERRUPTS;
3443 pr_debug("%s:%s:%d control=0x%x\n",
3444 skdev->name, __func__, __LINE__, val);
3445 SKD_WRITEL(skdev, val, FIT_CONTROL);
3446 }
3447
3448 /*
3449 *****************************************************************************
3450 * START, STOP, RESTART, QUIESCE, UNQUIESCE
3451 *****************************************************************************
3452 */
3453
skd_soft_reset(struct skd_device * skdev)3454 static void skd_soft_reset(struct skd_device *skdev)
3455 {
3456 u32 val;
3457
3458 val = SKD_READL(skdev, FIT_CONTROL);
3459 val |= (FIT_CR_SOFT_RESET);
3460 pr_debug("%s:%s:%d control=0x%x\n",
3461 skdev->name, __func__, __LINE__, val);
3462 SKD_WRITEL(skdev, val, FIT_CONTROL);
3463 }
3464
skd_start_device(struct skd_device * skdev)3465 static void skd_start_device(struct skd_device *skdev)
3466 {
3467 unsigned long flags;
3468 u32 sense;
3469 u32 state;
3470
3471 spin_lock_irqsave(&skdev->lock, flags);
3472
3473 /* ack all ghost interrupts */
3474 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3475
3476 sense = SKD_READL(skdev, FIT_STATUS);
3477
3478 pr_debug("%s:%s:%d initial status=0x%x\n",
3479 skdev->name, __func__, __LINE__, sense);
3480
3481 state = sense & FIT_SR_DRIVE_STATE_MASK;
3482 skdev->drive_state = state;
3483 skdev->last_mtd = 0;
3484
3485 skdev->state = SKD_DRVR_STATE_STARTING;
3486 skdev->timer_countdown = SKD_STARTING_TIMO;
3487
3488 skd_enable_interrupts(skdev);
3489
3490 switch (skdev->drive_state) {
3491 case FIT_SR_DRIVE_OFFLINE:
3492 pr_err("(%s): Drive offline...\n", skd_name(skdev));
3493 break;
3494
3495 case FIT_SR_DRIVE_FW_BOOTING:
3496 pr_debug("%s:%s:%d FIT_SR_DRIVE_FW_BOOTING %s\n",
3497 skdev->name, __func__, __LINE__, skdev->name);
3498 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3499 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3500 break;
3501
3502 case FIT_SR_DRIVE_BUSY_SANITIZE:
3503 pr_info("(%s): Start: BUSY_SANITIZE\n",
3504 skd_name(skdev));
3505 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3506 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3507 break;
3508
3509 case FIT_SR_DRIVE_BUSY_ERASE:
3510 pr_info("(%s): Start: BUSY_ERASE\n", skd_name(skdev));
3511 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3512 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3513 break;
3514
3515 case FIT_SR_DRIVE_INIT:
3516 case FIT_SR_DRIVE_ONLINE:
3517 skd_soft_reset(skdev);
3518 break;
3519
3520 case FIT_SR_DRIVE_BUSY:
3521 pr_err("(%s): Drive Busy...\n", skd_name(skdev));
3522 skdev->state = SKD_DRVR_STATE_BUSY;
3523 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3524 break;
3525
3526 case FIT_SR_DRIVE_SOFT_RESET:
3527 pr_err("(%s) drive soft reset in prog\n",
3528 skd_name(skdev));
3529 break;
3530
3531 case FIT_SR_DRIVE_FAULT:
3532 /* Fault state is bad...soft reset won't do it...
3533 * Hard reset, maybe, but does it work on device?
3534 * For now, just fault so the system doesn't hang.
3535 */
3536 skd_drive_fault(skdev);
3537 /*start the queue so we can respond with error to requests */
3538 pr_debug("%s:%s:%d starting %s queue\n",
3539 skdev->name, __func__, __LINE__, skdev->name);
3540 blk_start_queue(skdev->queue);
3541 skdev->gendisk_on = -1;
3542 wake_up_interruptible(&skdev->waitq);
3543 break;
3544
3545 case 0xFF:
3546 /* Most likely the device isn't there or isn't responding
3547 * to the BAR1 addresses. */
3548 skd_drive_disappeared(skdev);
3549 /*start the queue so we can respond with error to requests */
3550 pr_debug("%s:%s:%d starting %s queue to error-out reqs\n",
3551 skdev->name, __func__, __LINE__, skdev->name);
3552 blk_start_queue(skdev->queue);
3553 skdev->gendisk_on = -1;
3554 wake_up_interruptible(&skdev->waitq);
3555 break;
3556
3557 default:
3558 pr_err("(%s) Start: unknown state %x\n",
3559 skd_name(skdev), skdev->drive_state);
3560 break;
3561 }
3562
3563 state = SKD_READL(skdev, FIT_CONTROL);
3564 pr_debug("%s:%s:%d FIT Control Status=0x%x\n",
3565 skdev->name, __func__, __LINE__, state);
3566
3567 state = SKD_READL(skdev, FIT_INT_STATUS_HOST);
3568 pr_debug("%s:%s:%d Intr Status=0x%x\n",
3569 skdev->name, __func__, __LINE__, state);
3570
3571 state = SKD_READL(skdev, FIT_INT_MASK_HOST);
3572 pr_debug("%s:%s:%d Intr Mask=0x%x\n",
3573 skdev->name, __func__, __LINE__, state);
3574
3575 state = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
3576 pr_debug("%s:%s:%d Msg from Dev=0x%x\n",
3577 skdev->name, __func__, __LINE__, state);
3578
3579 state = SKD_READL(skdev, FIT_HW_VERSION);
3580 pr_debug("%s:%s:%d HW version=0x%x\n",
3581 skdev->name, __func__, __LINE__, state);
3582
3583 spin_unlock_irqrestore(&skdev->lock, flags);
3584 }
3585
skd_stop_device(struct skd_device * skdev)3586 static void skd_stop_device(struct skd_device *skdev)
3587 {
3588 unsigned long flags;
3589 struct skd_special_context *skspcl = &skdev->internal_skspcl;
3590 u32 dev_state;
3591 int i;
3592
3593 spin_lock_irqsave(&skdev->lock, flags);
3594
3595 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
3596 pr_err("(%s): skd_stop_device not online no sync\n",
3597 skd_name(skdev));
3598 goto stop_out;
3599 }
3600
3601 if (skspcl->req.state != SKD_REQ_STATE_IDLE) {
3602 pr_err("(%s): skd_stop_device no special\n",
3603 skd_name(skdev));
3604 goto stop_out;
3605 }
3606
3607 skdev->state = SKD_DRVR_STATE_SYNCING;
3608 skdev->sync_done = 0;
3609
3610 skd_send_internal_skspcl(skdev, skspcl, SYNCHRONIZE_CACHE);
3611
3612 spin_unlock_irqrestore(&skdev->lock, flags);
3613
3614 wait_event_interruptible_timeout(skdev->waitq,
3615 (skdev->sync_done), (10 * HZ));
3616
3617 spin_lock_irqsave(&skdev->lock, flags);
3618
3619 switch (skdev->sync_done) {
3620 case 0:
3621 pr_err("(%s): skd_stop_device no sync\n",
3622 skd_name(skdev));
3623 break;
3624 case 1:
3625 pr_err("(%s): skd_stop_device sync done\n",
3626 skd_name(skdev));
3627 break;
3628 default:
3629 pr_err("(%s): skd_stop_device sync error\n",
3630 skd_name(skdev));
3631 }
3632
3633 stop_out:
3634 skdev->state = SKD_DRVR_STATE_STOPPING;
3635 spin_unlock_irqrestore(&skdev->lock, flags);
3636
3637 skd_kill_timer(skdev);
3638
3639 spin_lock_irqsave(&skdev->lock, flags);
3640 skd_disable_interrupts(skdev);
3641
3642 /* ensure all ints on device are cleared */
3643 /* soft reset the device to unload with a clean slate */
3644 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3645 SKD_WRITEL(skdev, FIT_CR_SOFT_RESET, FIT_CONTROL);
3646
3647 spin_unlock_irqrestore(&skdev->lock, flags);
3648
3649 /* poll every 100ms, 1 second timeout */
3650 for (i = 0; i < 10; i++) {
3651 dev_state =
3652 SKD_READL(skdev, FIT_STATUS) & FIT_SR_DRIVE_STATE_MASK;
3653 if (dev_state == FIT_SR_DRIVE_INIT)
3654 break;
3655 set_current_state(TASK_INTERRUPTIBLE);
3656 schedule_timeout(msecs_to_jiffies(100));
3657 }
3658
3659 if (dev_state != FIT_SR_DRIVE_INIT)
3660 pr_err("(%s): skd_stop_device state error 0x%02x\n",
3661 skd_name(skdev), dev_state);
3662 }
3663
3664 /* assume spinlock is held */
skd_restart_device(struct skd_device * skdev)3665 static void skd_restart_device(struct skd_device *skdev)
3666 {
3667 u32 state;
3668
3669 /* ack all ghost interrupts */
3670 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3671
3672 state = SKD_READL(skdev, FIT_STATUS);
3673
3674 pr_debug("%s:%s:%d drive status=0x%x\n",
3675 skdev->name, __func__, __LINE__, state);
3676
3677 state &= FIT_SR_DRIVE_STATE_MASK;
3678 skdev->drive_state = state;
3679 skdev->last_mtd = 0;
3680
3681 skdev->state = SKD_DRVR_STATE_RESTARTING;
3682 skdev->timer_countdown = SKD_RESTARTING_TIMO;
3683
3684 skd_soft_reset(skdev);
3685 }
3686
3687 /* assume spinlock is held */
skd_quiesce_dev(struct skd_device * skdev)3688 static int skd_quiesce_dev(struct skd_device *skdev)
3689 {
3690 int rc = 0;
3691
3692 switch (skdev->state) {
3693 case SKD_DRVR_STATE_BUSY:
3694 case SKD_DRVR_STATE_BUSY_IMMINENT:
3695 pr_debug("%s:%s:%d stopping %s queue\n",
3696 skdev->name, __func__, __LINE__, skdev->name);
3697 blk_stop_queue(skdev->queue);
3698 break;
3699 case SKD_DRVR_STATE_ONLINE:
3700 case SKD_DRVR_STATE_STOPPING:
3701 case SKD_DRVR_STATE_SYNCING:
3702 case SKD_DRVR_STATE_PAUSING:
3703 case SKD_DRVR_STATE_PAUSED:
3704 case SKD_DRVR_STATE_STARTING:
3705 case SKD_DRVR_STATE_RESTARTING:
3706 case SKD_DRVR_STATE_RESUMING:
3707 default:
3708 rc = -EINVAL;
3709 pr_debug("%s:%s:%d state [%d] not implemented\n",
3710 skdev->name, __func__, __LINE__, skdev->state);
3711 }
3712 return rc;
3713 }
3714
3715 /* assume spinlock is held */
skd_unquiesce_dev(struct skd_device * skdev)3716 static int skd_unquiesce_dev(struct skd_device *skdev)
3717 {
3718 int prev_driver_state = skdev->state;
3719
3720 skd_log_skdev(skdev, "unquiesce");
3721 if (skdev->state == SKD_DRVR_STATE_ONLINE) {
3722 pr_debug("%s:%s:%d **** device already ONLINE\n",
3723 skdev->name, __func__, __LINE__);
3724 return 0;
3725 }
3726 if (skdev->drive_state != FIT_SR_DRIVE_ONLINE) {
3727 /*
3728 * If there has been an state change to other than
3729 * ONLINE, we will rely on controller state change
3730 * to come back online and restart the queue.
3731 * The BUSY state means that driver is ready to
3732 * continue normal processing but waiting for controller
3733 * to become available.
3734 */
3735 skdev->state = SKD_DRVR_STATE_BUSY;
3736 pr_debug("%s:%s:%d drive BUSY state\n",
3737 skdev->name, __func__, __LINE__);
3738 return 0;
3739 }
3740
3741 /*
3742 * Drive has just come online, driver is either in startup,
3743 * paused performing a task, or bust waiting for hardware.
3744 */
3745 switch (skdev->state) {
3746 case SKD_DRVR_STATE_PAUSED:
3747 case SKD_DRVR_STATE_BUSY:
3748 case SKD_DRVR_STATE_BUSY_IMMINENT:
3749 case SKD_DRVR_STATE_BUSY_ERASE:
3750 case SKD_DRVR_STATE_STARTING:
3751 case SKD_DRVR_STATE_RESTARTING:
3752 case SKD_DRVR_STATE_FAULT:
3753 case SKD_DRVR_STATE_IDLE:
3754 case SKD_DRVR_STATE_LOAD:
3755 skdev->state = SKD_DRVR_STATE_ONLINE;
3756 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3757 skd_name(skdev),
3758 skd_skdev_state_to_str(prev_driver_state),
3759 prev_driver_state, skd_skdev_state_to_str(skdev->state),
3760 skdev->state);
3761 pr_debug("%s:%s:%d **** device ONLINE...starting block queue\n",
3762 skdev->name, __func__, __LINE__);
3763 pr_debug("%s:%s:%d starting %s queue\n",
3764 skdev->name, __func__, __LINE__, skdev->name);
3765 pr_info("(%s): STEC s1120 ONLINE\n", skd_name(skdev));
3766 blk_start_queue(skdev->queue);
3767 skdev->gendisk_on = 1;
3768 wake_up_interruptible(&skdev->waitq);
3769 break;
3770
3771 case SKD_DRVR_STATE_DISAPPEARED:
3772 default:
3773 pr_debug("%s:%s:%d **** driver state %d, not implemented \n",
3774 skdev->name, __func__, __LINE__,
3775 skdev->state);
3776 return -EBUSY;
3777 }
3778 return 0;
3779 }
3780
3781 /*
3782 *****************************************************************************
3783 * PCIe MSI/MSI-X INTERRUPT HANDLERS
3784 *****************************************************************************
3785 */
3786
skd_reserved_isr(int irq,void * skd_host_data)3787 static irqreturn_t skd_reserved_isr(int irq, void *skd_host_data)
3788 {
3789 struct skd_device *skdev = skd_host_data;
3790 unsigned long flags;
3791
3792 spin_lock_irqsave(&skdev->lock, flags);
3793 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3794 skdev->name, __func__, __LINE__,
3795 SKD_READL(skdev, FIT_INT_STATUS_HOST));
3796 pr_err("(%s): MSIX reserved irq %d = 0x%x\n", skd_name(skdev),
3797 irq, SKD_READL(skdev, FIT_INT_STATUS_HOST));
3798 SKD_WRITEL(skdev, FIT_INT_RESERVED_MASK, FIT_INT_STATUS_HOST);
3799 spin_unlock_irqrestore(&skdev->lock, flags);
3800 return IRQ_HANDLED;
3801 }
3802
skd_statec_isr(int irq,void * skd_host_data)3803 static irqreturn_t skd_statec_isr(int irq, void *skd_host_data)
3804 {
3805 struct skd_device *skdev = skd_host_data;
3806 unsigned long flags;
3807
3808 spin_lock_irqsave(&skdev->lock, flags);
3809 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3810 skdev->name, __func__, __LINE__,
3811 SKD_READL(skdev, FIT_INT_STATUS_HOST));
3812 SKD_WRITEL(skdev, FIT_ISH_FW_STATE_CHANGE, FIT_INT_STATUS_HOST);
3813 skd_isr_fwstate(skdev);
3814 spin_unlock_irqrestore(&skdev->lock, flags);
3815 return IRQ_HANDLED;
3816 }
3817
skd_comp_q(int irq,void * skd_host_data)3818 static irqreturn_t skd_comp_q(int irq, void *skd_host_data)
3819 {
3820 struct skd_device *skdev = skd_host_data;
3821 unsigned long flags;
3822 int flush_enqueued = 0;
3823 int deferred;
3824
3825 spin_lock_irqsave(&skdev->lock, flags);
3826 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3827 skdev->name, __func__, __LINE__,
3828 SKD_READL(skdev, FIT_INT_STATUS_HOST));
3829 SKD_WRITEL(skdev, FIT_ISH_COMPLETION_POSTED, FIT_INT_STATUS_HOST);
3830 deferred = skd_isr_completion_posted(skdev, skd_isr_comp_limit,
3831 &flush_enqueued);
3832 if (flush_enqueued)
3833 skd_request_fn(skdev->queue);
3834
3835 if (deferred)
3836 schedule_work(&skdev->completion_worker);
3837 else if (!flush_enqueued)
3838 skd_request_fn(skdev->queue);
3839
3840 spin_unlock_irqrestore(&skdev->lock, flags);
3841
3842 return IRQ_HANDLED;
3843 }
3844
skd_msg_isr(int irq,void * skd_host_data)3845 static irqreturn_t skd_msg_isr(int irq, void *skd_host_data)
3846 {
3847 struct skd_device *skdev = skd_host_data;
3848 unsigned long flags;
3849
3850 spin_lock_irqsave(&skdev->lock, flags);
3851 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3852 skdev->name, __func__, __LINE__,
3853 SKD_READL(skdev, FIT_INT_STATUS_HOST));
3854 SKD_WRITEL(skdev, FIT_ISH_MSG_FROM_DEV, FIT_INT_STATUS_HOST);
3855 skd_isr_msg_from_dev(skdev);
3856 spin_unlock_irqrestore(&skdev->lock, flags);
3857 return IRQ_HANDLED;
3858 }
3859
skd_qfull_isr(int irq,void * skd_host_data)3860 static irqreturn_t skd_qfull_isr(int irq, void *skd_host_data)
3861 {
3862 struct skd_device *skdev = skd_host_data;
3863 unsigned long flags;
3864
3865 spin_lock_irqsave(&skdev->lock, flags);
3866 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3867 skdev->name, __func__, __LINE__,
3868 SKD_READL(skdev, FIT_INT_STATUS_HOST));
3869 SKD_WRITEL(skdev, FIT_INT_QUEUE_FULL, FIT_INT_STATUS_HOST);
3870 spin_unlock_irqrestore(&skdev->lock, flags);
3871 return IRQ_HANDLED;
3872 }
3873
3874 /*
3875 *****************************************************************************
3876 * PCIe MSI/MSI-X SETUP
3877 *****************************************************************************
3878 */
3879
3880 struct skd_msix_entry {
3881 int have_irq;
3882 u32 vector;
3883 u32 entry;
3884 struct skd_device *rsp;
3885 char isr_name[30];
3886 };
3887
3888 struct skd_init_msix_entry {
3889 const char *name;
3890 irq_handler_t handler;
3891 };
3892
3893 #define SKD_MAX_MSIX_COUNT 13
3894 #define SKD_MIN_MSIX_COUNT 7
3895 #define SKD_BASE_MSIX_IRQ 4
3896
3897 static struct skd_init_msix_entry msix_entries[SKD_MAX_MSIX_COUNT] = {
3898 { "(DMA 0)", skd_reserved_isr },
3899 { "(DMA 1)", skd_reserved_isr },
3900 { "(DMA 2)", skd_reserved_isr },
3901 { "(DMA 3)", skd_reserved_isr },
3902 { "(State Change)", skd_statec_isr },
3903 { "(COMPL_Q)", skd_comp_q },
3904 { "(MSG)", skd_msg_isr },
3905 { "(Reserved)", skd_reserved_isr },
3906 { "(Reserved)", skd_reserved_isr },
3907 { "(Queue Full 0)", skd_qfull_isr },
3908 { "(Queue Full 1)", skd_qfull_isr },
3909 { "(Queue Full 2)", skd_qfull_isr },
3910 { "(Queue Full 3)", skd_qfull_isr },
3911 };
3912
skd_release_msix(struct skd_device * skdev)3913 static void skd_release_msix(struct skd_device *skdev)
3914 {
3915 struct skd_msix_entry *qentry;
3916 int i;
3917
3918 if (skdev->msix_entries) {
3919 for (i = 0; i < skdev->msix_count; i++) {
3920 qentry = &skdev->msix_entries[i];
3921 skdev = qentry->rsp;
3922
3923 if (qentry->have_irq)
3924 devm_free_irq(&skdev->pdev->dev,
3925 qentry->vector, qentry->rsp);
3926 }
3927
3928 kfree(skdev->msix_entries);
3929 }
3930
3931 if (skdev->msix_count)
3932 pci_disable_msix(skdev->pdev);
3933
3934 skdev->msix_count = 0;
3935 skdev->msix_entries = NULL;
3936 }
3937
skd_acquire_msix(struct skd_device * skdev)3938 static int skd_acquire_msix(struct skd_device *skdev)
3939 {
3940 int i, rc;
3941 struct pci_dev *pdev = skdev->pdev;
3942 struct msix_entry *entries;
3943 struct skd_msix_entry *qentry;
3944
3945 entries = kzalloc(sizeof(struct msix_entry) * SKD_MAX_MSIX_COUNT,
3946 GFP_KERNEL);
3947 if (!entries)
3948 return -ENOMEM;
3949
3950 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++)
3951 entries[i].entry = i;
3952
3953 rc = pci_enable_msix_exact(pdev, entries, SKD_MAX_MSIX_COUNT);
3954 if (rc) {
3955 pr_err("(%s): failed to enable MSI-X %d\n",
3956 skd_name(skdev), rc);
3957 goto msix_out;
3958 }
3959
3960 skdev->msix_count = SKD_MAX_MSIX_COUNT;
3961 skdev->msix_entries = kzalloc(sizeof(struct skd_msix_entry) *
3962 skdev->msix_count, GFP_KERNEL);
3963 if (!skdev->msix_entries) {
3964 rc = -ENOMEM;
3965 pr_err("(%s): msix table allocation error\n",
3966 skd_name(skdev));
3967 goto msix_out;
3968 }
3969
3970 for (i = 0; i < skdev->msix_count; i++) {
3971 qentry = &skdev->msix_entries[i];
3972 qentry->vector = entries[i].vector;
3973 qentry->entry = entries[i].entry;
3974 qentry->rsp = NULL;
3975 qentry->have_irq = 0;
3976 pr_debug("%s:%s:%d %s: <%s> msix (%d) vec %d, entry %x\n",
3977 skdev->name, __func__, __LINE__,
3978 pci_name(pdev), skdev->name,
3979 i, qentry->vector, qentry->entry);
3980 }
3981
3982 /* Enable MSI-X vectors for the base queue */
3983 for (i = 0; i < skdev->msix_count; i++) {
3984 qentry = &skdev->msix_entries[i];
3985 snprintf(qentry->isr_name, sizeof(qentry->isr_name),
3986 "%s%d-msix %s", DRV_NAME, skdev->devno,
3987 msix_entries[i].name);
3988 rc = devm_request_irq(&skdev->pdev->dev, qentry->vector,
3989 msix_entries[i].handler, 0,
3990 qentry->isr_name, skdev);
3991 if (rc) {
3992 pr_err("(%s): Unable to register(%d) MSI-X "
3993 "handler %d: %s\n",
3994 skd_name(skdev), rc, i, qentry->isr_name);
3995 goto msix_out;
3996 } else {
3997 qentry->have_irq = 1;
3998 qentry->rsp = skdev;
3999 }
4000 }
4001 pr_debug("%s:%s:%d %s: <%s> msix %d irq(s) enabled\n",
4002 skdev->name, __func__, __LINE__,
4003 pci_name(pdev), skdev->name, skdev->msix_count);
4004 return 0;
4005
4006 msix_out:
4007 if (entries)
4008 kfree(entries);
4009 skd_release_msix(skdev);
4010 return rc;
4011 }
4012
skd_acquire_irq(struct skd_device * skdev)4013 static int skd_acquire_irq(struct skd_device *skdev)
4014 {
4015 int rc;
4016 struct pci_dev *pdev;
4017
4018 pdev = skdev->pdev;
4019 skdev->msix_count = 0;
4020
4021 RETRY_IRQ_TYPE:
4022 switch (skdev->irq_type) {
4023 case SKD_IRQ_MSIX:
4024 rc = skd_acquire_msix(skdev);
4025 if (!rc)
4026 pr_info("(%s): MSI-X %d irqs enabled\n",
4027 skd_name(skdev), skdev->msix_count);
4028 else {
4029 pr_err(
4030 "(%s): failed to enable MSI-X, re-trying with MSI %d\n",
4031 skd_name(skdev), rc);
4032 skdev->irq_type = SKD_IRQ_MSI;
4033 goto RETRY_IRQ_TYPE;
4034 }
4035 break;
4036 case SKD_IRQ_MSI:
4037 snprintf(skdev->isr_name, sizeof(skdev->isr_name), "%s%d-msi",
4038 DRV_NAME, skdev->devno);
4039 rc = pci_enable_msi_range(pdev, 1, 1);
4040 if (rc > 0) {
4041 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr, 0,
4042 skdev->isr_name, skdev);
4043 if (rc) {
4044 pci_disable_msi(pdev);
4045 pr_err(
4046 "(%s): failed to allocate the MSI interrupt %d\n",
4047 skd_name(skdev), rc);
4048 goto RETRY_IRQ_LEGACY;
4049 }
4050 pr_info("(%s): MSI irq %d enabled\n",
4051 skd_name(skdev), pdev->irq);
4052 } else {
4053 RETRY_IRQ_LEGACY:
4054 pr_err(
4055 "(%s): failed to enable MSI, re-trying with LEGACY %d\n",
4056 skd_name(skdev), rc);
4057 skdev->irq_type = SKD_IRQ_LEGACY;
4058 goto RETRY_IRQ_TYPE;
4059 }
4060 break;
4061 case SKD_IRQ_LEGACY:
4062 snprintf(skdev->isr_name, sizeof(skdev->isr_name),
4063 "%s%d-legacy", DRV_NAME, skdev->devno);
4064 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr,
4065 IRQF_SHARED, skdev->isr_name, skdev);
4066 if (!rc)
4067 pr_info("(%s): LEGACY irq %d enabled\n",
4068 skd_name(skdev), pdev->irq);
4069 else
4070 pr_err("(%s): request LEGACY irq error %d\n",
4071 skd_name(skdev), rc);
4072 break;
4073 default:
4074 pr_info("(%s): irq_type %d invalid, re-set to %d\n",
4075 skd_name(skdev), skdev->irq_type, SKD_IRQ_DEFAULT);
4076 skdev->irq_type = SKD_IRQ_LEGACY;
4077 goto RETRY_IRQ_TYPE;
4078 }
4079 return rc;
4080 }
4081
skd_release_irq(struct skd_device * skdev)4082 static void skd_release_irq(struct skd_device *skdev)
4083 {
4084 switch (skdev->irq_type) {
4085 case SKD_IRQ_MSIX:
4086 skd_release_msix(skdev);
4087 break;
4088 case SKD_IRQ_MSI:
4089 devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
4090 pci_disable_msi(skdev->pdev);
4091 break;
4092 case SKD_IRQ_LEGACY:
4093 devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
4094 break;
4095 default:
4096 pr_err("(%s): wrong irq type %d!",
4097 skd_name(skdev), skdev->irq_type);
4098 break;
4099 }
4100 }
4101
4102 /*
4103 *****************************************************************************
4104 * CONSTRUCT
4105 *****************************************************************************
4106 */
4107
skd_cons_skcomp(struct skd_device * skdev)4108 static int skd_cons_skcomp(struct skd_device *skdev)
4109 {
4110 int rc = 0;
4111 struct fit_completion_entry_v1 *skcomp;
4112 u32 nbytes;
4113
4114 nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
4115 nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
4116
4117 pr_debug("%s:%s:%d comp pci_alloc, total bytes %d entries %d\n",
4118 skdev->name, __func__, __LINE__,
4119 nbytes, SKD_N_COMPLETION_ENTRY);
4120
4121 skcomp = pci_zalloc_consistent(skdev->pdev, nbytes,
4122 &skdev->cq_dma_address);
4123
4124 if (skcomp == NULL) {
4125 rc = -ENOMEM;
4126 goto err_out;
4127 }
4128
4129 skdev->skcomp_table = skcomp;
4130 skdev->skerr_table = (struct fit_comp_error_info *)((char *)skcomp +
4131 sizeof(*skcomp) *
4132 SKD_N_COMPLETION_ENTRY);
4133
4134 err_out:
4135 return rc;
4136 }
4137
skd_cons_skmsg(struct skd_device * skdev)4138 static int skd_cons_skmsg(struct skd_device *skdev)
4139 {
4140 int rc = 0;
4141 u32 i;
4142
4143 pr_debug("%s:%s:%d skmsg_table kzalloc, struct %lu, count %u total %lu\n",
4144 skdev->name, __func__, __LINE__,
4145 sizeof(struct skd_fitmsg_context),
4146 skdev->num_fitmsg_context,
4147 sizeof(struct skd_fitmsg_context) * skdev->num_fitmsg_context);
4148
4149 skdev->skmsg_table = kzalloc(sizeof(struct skd_fitmsg_context)
4150 *skdev->num_fitmsg_context, GFP_KERNEL);
4151 if (skdev->skmsg_table == NULL) {
4152 rc = -ENOMEM;
4153 goto err_out;
4154 }
4155
4156 for (i = 0; i < skdev->num_fitmsg_context; i++) {
4157 struct skd_fitmsg_context *skmsg;
4158
4159 skmsg = &skdev->skmsg_table[i];
4160
4161 skmsg->id = i + SKD_ID_FIT_MSG;
4162
4163 skmsg->state = SKD_MSG_STATE_IDLE;
4164 skmsg->msg_buf = pci_alloc_consistent(skdev->pdev,
4165 SKD_N_FITMSG_BYTES + 64,
4166 &skmsg->mb_dma_address);
4167
4168 if (skmsg->msg_buf == NULL) {
4169 rc = -ENOMEM;
4170 goto err_out;
4171 }
4172
4173 skmsg->offset = (u32)((u64)skmsg->msg_buf &
4174 (~FIT_QCMD_BASE_ADDRESS_MASK));
4175 skmsg->msg_buf += ~FIT_QCMD_BASE_ADDRESS_MASK;
4176 skmsg->msg_buf = (u8 *)((u64)skmsg->msg_buf &
4177 FIT_QCMD_BASE_ADDRESS_MASK);
4178 skmsg->mb_dma_address += ~FIT_QCMD_BASE_ADDRESS_MASK;
4179 skmsg->mb_dma_address &= FIT_QCMD_BASE_ADDRESS_MASK;
4180 memset(skmsg->msg_buf, 0, SKD_N_FITMSG_BYTES);
4181
4182 skmsg->next = &skmsg[1];
4183 }
4184
4185 /* Free list is in order starting with the 0th entry. */
4186 skdev->skmsg_table[i - 1].next = NULL;
4187 skdev->skmsg_free_list = skdev->skmsg_table;
4188
4189 err_out:
4190 return rc;
4191 }
4192
skd_cons_sg_list(struct skd_device * skdev,u32 n_sg,dma_addr_t * ret_dma_addr)4193 static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
4194 u32 n_sg,
4195 dma_addr_t *ret_dma_addr)
4196 {
4197 struct fit_sg_descriptor *sg_list;
4198 u32 nbytes;
4199
4200 nbytes = sizeof(*sg_list) * n_sg;
4201
4202 sg_list = pci_alloc_consistent(skdev->pdev, nbytes, ret_dma_addr);
4203
4204 if (sg_list != NULL) {
4205 uint64_t dma_address = *ret_dma_addr;
4206 u32 i;
4207
4208 memset(sg_list, 0, nbytes);
4209
4210 for (i = 0; i < n_sg - 1; i++) {
4211 uint64_t ndp_off;
4212 ndp_off = (i + 1) * sizeof(struct fit_sg_descriptor);
4213
4214 sg_list[i].next_desc_ptr = dma_address + ndp_off;
4215 }
4216 sg_list[i].next_desc_ptr = 0LL;
4217 }
4218
4219 return sg_list;
4220 }
4221
skd_cons_skreq(struct skd_device * skdev)4222 static int skd_cons_skreq(struct skd_device *skdev)
4223 {
4224 int rc = 0;
4225 u32 i;
4226
4227 pr_debug("%s:%s:%d skreq_table kzalloc, struct %lu, count %u total %lu\n",
4228 skdev->name, __func__, __LINE__,
4229 sizeof(struct skd_request_context),
4230 skdev->num_req_context,
4231 sizeof(struct skd_request_context) * skdev->num_req_context);
4232
4233 skdev->skreq_table = kzalloc(sizeof(struct skd_request_context)
4234 * skdev->num_req_context, GFP_KERNEL);
4235 if (skdev->skreq_table == NULL) {
4236 rc = -ENOMEM;
4237 goto err_out;
4238 }
4239
4240 pr_debug("%s:%s:%d alloc sg_table sg_per_req %u scatlist %lu total %lu\n",
4241 skdev->name, __func__, __LINE__,
4242 skdev->sgs_per_request, sizeof(struct scatterlist),
4243 skdev->sgs_per_request * sizeof(struct scatterlist));
4244
4245 for (i = 0; i < skdev->num_req_context; i++) {
4246 struct skd_request_context *skreq;
4247
4248 skreq = &skdev->skreq_table[i];
4249
4250 skreq->id = i + SKD_ID_RW_REQUEST;
4251 skreq->state = SKD_REQ_STATE_IDLE;
4252
4253 skreq->sg = kzalloc(sizeof(struct scatterlist) *
4254 skdev->sgs_per_request, GFP_KERNEL);
4255 if (skreq->sg == NULL) {
4256 rc = -ENOMEM;
4257 goto err_out;
4258 }
4259 sg_init_table(skreq->sg, skdev->sgs_per_request);
4260
4261 skreq->sksg_list = skd_cons_sg_list(skdev,
4262 skdev->sgs_per_request,
4263 &skreq->sksg_dma_address);
4264
4265 if (skreq->sksg_list == NULL) {
4266 rc = -ENOMEM;
4267 goto err_out;
4268 }
4269
4270 skreq->next = &skreq[1];
4271 }
4272
4273 /* Free list is in order starting with the 0th entry. */
4274 skdev->skreq_table[i - 1].next = NULL;
4275 skdev->skreq_free_list = skdev->skreq_table;
4276
4277 err_out:
4278 return rc;
4279 }
4280
skd_cons_skspcl(struct skd_device * skdev)4281 static int skd_cons_skspcl(struct skd_device *skdev)
4282 {
4283 int rc = 0;
4284 u32 i, nbytes;
4285
4286 pr_debug("%s:%s:%d skspcl_table kzalloc, struct %lu, count %u total %lu\n",
4287 skdev->name, __func__, __LINE__,
4288 sizeof(struct skd_special_context),
4289 skdev->n_special,
4290 sizeof(struct skd_special_context) * skdev->n_special);
4291
4292 skdev->skspcl_table = kzalloc(sizeof(struct skd_special_context)
4293 * skdev->n_special, GFP_KERNEL);
4294 if (skdev->skspcl_table == NULL) {
4295 rc = -ENOMEM;
4296 goto err_out;
4297 }
4298
4299 for (i = 0; i < skdev->n_special; i++) {
4300 struct skd_special_context *skspcl;
4301
4302 skspcl = &skdev->skspcl_table[i];
4303
4304 skspcl->req.id = i + SKD_ID_SPECIAL_REQUEST;
4305 skspcl->req.state = SKD_REQ_STATE_IDLE;
4306
4307 skspcl->req.next = &skspcl[1].req;
4308
4309 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4310
4311 skspcl->msg_buf =
4312 pci_zalloc_consistent(skdev->pdev, nbytes,
4313 &skspcl->mb_dma_address);
4314 if (skspcl->msg_buf == NULL) {
4315 rc = -ENOMEM;
4316 goto err_out;
4317 }
4318
4319 skspcl->req.sg = kzalloc(sizeof(struct scatterlist) *
4320 SKD_N_SG_PER_SPECIAL, GFP_KERNEL);
4321 if (skspcl->req.sg == NULL) {
4322 rc = -ENOMEM;
4323 goto err_out;
4324 }
4325
4326 skspcl->req.sksg_list = skd_cons_sg_list(skdev,
4327 SKD_N_SG_PER_SPECIAL,
4328 &skspcl->req.
4329 sksg_dma_address);
4330 if (skspcl->req.sksg_list == NULL) {
4331 rc = -ENOMEM;
4332 goto err_out;
4333 }
4334 }
4335
4336 /* Free list is in order starting with the 0th entry. */
4337 skdev->skspcl_table[i - 1].req.next = NULL;
4338 skdev->skspcl_free_list = skdev->skspcl_table;
4339
4340 return rc;
4341
4342 err_out:
4343 return rc;
4344 }
4345
skd_cons_sksb(struct skd_device * skdev)4346 static int skd_cons_sksb(struct skd_device *skdev)
4347 {
4348 int rc = 0;
4349 struct skd_special_context *skspcl;
4350 u32 nbytes;
4351
4352 skspcl = &skdev->internal_skspcl;
4353
4354 skspcl->req.id = 0 + SKD_ID_INTERNAL;
4355 skspcl->req.state = SKD_REQ_STATE_IDLE;
4356
4357 nbytes = SKD_N_INTERNAL_BYTES;
4358
4359 skspcl->data_buf = pci_zalloc_consistent(skdev->pdev, nbytes,
4360 &skspcl->db_dma_address);
4361 if (skspcl->data_buf == NULL) {
4362 rc = -ENOMEM;
4363 goto err_out;
4364 }
4365
4366 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4367 skspcl->msg_buf = pci_zalloc_consistent(skdev->pdev, nbytes,
4368 &skspcl->mb_dma_address);
4369 if (skspcl->msg_buf == NULL) {
4370 rc = -ENOMEM;
4371 goto err_out;
4372 }
4373
4374 skspcl->req.sksg_list = skd_cons_sg_list(skdev, 1,
4375 &skspcl->req.sksg_dma_address);
4376 if (skspcl->req.sksg_list == NULL) {
4377 rc = -ENOMEM;
4378 goto err_out;
4379 }
4380
4381 if (!skd_format_internal_skspcl(skdev)) {
4382 rc = -EINVAL;
4383 goto err_out;
4384 }
4385
4386 err_out:
4387 return rc;
4388 }
4389
skd_cons_disk(struct skd_device * skdev)4390 static int skd_cons_disk(struct skd_device *skdev)
4391 {
4392 int rc = 0;
4393 struct gendisk *disk;
4394 struct request_queue *q;
4395 unsigned long flags;
4396
4397 disk = alloc_disk(SKD_MINORS_PER_DEVICE);
4398 if (!disk) {
4399 rc = -ENOMEM;
4400 goto err_out;
4401 }
4402
4403 skdev->disk = disk;
4404 sprintf(disk->disk_name, DRV_NAME "%u", skdev->devno);
4405
4406 disk->major = skdev->major;
4407 disk->first_minor = skdev->devno * SKD_MINORS_PER_DEVICE;
4408 disk->fops = &skd_blockdev_ops;
4409 disk->private_data = skdev;
4410
4411 q = blk_init_queue(skd_request_fn, &skdev->lock);
4412 if (!q) {
4413 rc = -ENOMEM;
4414 goto err_out;
4415 }
4416
4417 skdev->queue = q;
4418 disk->queue = q;
4419 q->queuedata = skdev;
4420
4421 blk_queue_flush(q, REQ_FLUSH | REQ_FUA);
4422 blk_queue_max_segments(q, skdev->sgs_per_request);
4423 blk_queue_max_hw_sectors(q, SKD_N_MAX_SECTORS);
4424
4425 /* set sysfs ptimal_io_size to 8K */
4426 blk_queue_io_opt(q, 8192);
4427
4428 /* DISCARD Flag initialization. */
4429 q->limits.discard_granularity = 8192;
4430 q->limits.discard_alignment = 0;
4431 q->limits.max_discard_sectors = UINT_MAX >> 9;
4432 q->limits.discard_zeroes_data = 1;
4433 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
4434 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
4435 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, q);
4436
4437 spin_lock_irqsave(&skdev->lock, flags);
4438 pr_debug("%s:%s:%d stopping %s queue\n",
4439 skdev->name, __func__, __LINE__, skdev->name);
4440 blk_stop_queue(skdev->queue);
4441 spin_unlock_irqrestore(&skdev->lock, flags);
4442
4443 err_out:
4444 return rc;
4445 }
4446
4447 #define SKD_N_DEV_TABLE 16u
4448 static u32 skd_next_devno;
4449
skd_construct(struct pci_dev * pdev)4450 static struct skd_device *skd_construct(struct pci_dev *pdev)
4451 {
4452 struct skd_device *skdev;
4453 int blk_major = skd_major;
4454 int rc;
4455
4456 skdev = kzalloc(sizeof(*skdev), GFP_KERNEL);
4457
4458 if (!skdev) {
4459 pr_err(PFX "(%s): memory alloc failure\n",
4460 pci_name(pdev));
4461 return NULL;
4462 }
4463
4464 skdev->state = SKD_DRVR_STATE_LOAD;
4465 skdev->pdev = pdev;
4466 skdev->devno = skd_next_devno++;
4467 skdev->major = blk_major;
4468 skdev->irq_type = skd_isr_type;
4469 sprintf(skdev->name, DRV_NAME "%d", skdev->devno);
4470 skdev->dev_max_queue_depth = 0;
4471
4472 skdev->num_req_context = skd_max_queue_depth;
4473 skdev->num_fitmsg_context = skd_max_queue_depth;
4474 skdev->n_special = skd_max_pass_thru;
4475 skdev->cur_max_queue_depth = 1;
4476 skdev->queue_low_water_mark = 1;
4477 skdev->proto_ver = 99;
4478 skdev->sgs_per_request = skd_sgs_per_request;
4479 skdev->dbg_level = skd_dbg_level;
4480
4481 atomic_set(&skdev->device_count, 0);
4482
4483 spin_lock_init(&skdev->lock);
4484
4485 INIT_WORK(&skdev->completion_worker, skd_completion_worker);
4486
4487 pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
4488 rc = skd_cons_skcomp(skdev);
4489 if (rc < 0)
4490 goto err_out;
4491
4492 pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
4493 rc = skd_cons_skmsg(skdev);
4494 if (rc < 0)
4495 goto err_out;
4496
4497 pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
4498 rc = skd_cons_skreq(skdev);
4499 if (rc < 0)
4500 goto err_out;
4501
4502 pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
4503 rc = skd_cons_skspcl(skdev);
4504 if (rc < 0)
4505 goto err_out;
4506
4507 pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
4508 rc = skd_cons_sksb(skdev);
4509 if (rc < 0)
4510 goto err_out;
4511
4512 pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
4513 rc = skd_cons_disk(skdev);
4514 if (rc < 0)
4515 goto err_out;
4516
4517 pr_debug("%s:%s:%d VICTORY\n", skdev->name, __func__, __LINE__);
4518 return skdev;
4519
4520 err_out:
4521 pr_debug("%s:%s:%d construct failed\n",
4522 skdev->name, __func__, __LINE__);
4523 skd_destruct(skdev);
4524 return NULL;
4525 }
4526
4527 /*
4528 *****************************************************************************
4529 * DESTRUCT (FREE)
4530 *****************************************************************************
4531 */
4532
skd_free_skcomp(struct skd_device * skdev)4533 static void skd_free_skcomp(struct skd_device *skdev)
4534 {
4535 if (skdev->skcomp_table != NULL) {
4536 u32 nbytes;
4537
4538 nbytes = sizeof(skdev->skcomp_table[0]) *
4539 SKD_N_COMPLETION_ENTRY;
4540 pci_free_consistent(skdev->pdev, nbytes,
4541 skdev->skcomp_table, skdev->cq_dma_address);
4542 }
4543
4544 skdev->skcomp_table = NULL;
4545 skdev->cq_dma_address = 0;
4546 }
4547
skd_free_skmsg(struct skd_device * skdev)4548 static void skd_free_skmsg(struct skd_device *skdev)
4549 {
4550 u32 i;
4551
4552 if (skdev->skmsg_table == NULL)
4553 return;
4554
4555 for (i = 0; i < skdev->num_fitmsg_context; i++) {
4556 struct skd_fitmsg_context *skmsg;
4557
4558 skmsg = &skdev->skmsg_table[i];
4559
4560 if (skmsg->msg_buf != NULL) {
4561 skmsg->msg_buf += skmsg->offset;
4562 skmsg->mb_dma_address += skmsg->offset;
4563 pci_free_consistent(skdev->pdev, SKD_N_FITMSG_BYTES,
4564 skmsg->msg_buf,
4565 skmsg->mb_dma_address);
4566 }
4567 skmsg->msg_buf = NULL;
4568 skmsg->mb_dma_address = 0;
4569 }
4570
4571 kfree(skdev->skmsg_table);
4572 skdev->skmsg_table = NULL;
4573 }
4574
skd_free_sg_list(struct skd_device * skdev,struct fit_sg_descriptor * sg_list,u32 n_sg,dma_addr_t dma_addr)4575 static void skd_free_sg_list(struct skd_device *skdev,
4576 struct fit_sg_descriptor *sg_list,
4577 u32 n_sg, dma_addr_t dma_addr)
4578 {
4579 if (sg_list != NULL) {
4580 u32 nbytes;
4581
4582 nbytes = sizeof(*sg_list) * n_sg;
4583
4584 pci_free_consistent(skdev->pdev, nbytes, sg_list, dma_addr);
4585 }
4586 }
4587
skd_free_skreq(struct skd_device * skdev)4588 static void skd_free_skreq(struct skd_device *skdev)
4589 {
4590 u32 i;
4591
4592 if (skdev->skreq_table == NULL)
4593 return;
4594
4595 for (i = 0; i < skdev->num_req_context; i++) {
4596 struct skd_request_context *skreq;
4597
4598 skreq = &skdev->skreq_table[i];
4599
4600 skd_free_sg_list(skdev, skreq->sksg_list,
4601 skdev->sgs_per_request,
4602 skreq->sksg_dma_address);
4603
4604 skreq->sksg_list = NULL;
4605 skreq->sksg_dma_address = 0;
4606
4607 kfree(skreq->sg);
4608 }
4609
4610 kfree(skdev->skreq_table);
4611 skdev->skreq_table = NULL;
4612 }
4613
skd_free_skspcl(struct skd_device * skdev)4614 static void skd_free_skspcl(struct skd_device *skdev)
4615 {
4616 u32 i;
4617 u32 nbytes;
4618
4619 if (skdev->skspcl_table == NULL)
4620 return;
4621
4622 for (i = 0; i < skdev->n_special; i++) {
4623 struct skd_special_context *skspcl;
4624
4625 skspcl = &skdev->skspcl_table[i];
4626
4627 if (skspcl->msg_buf != NULL) {
4628 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4629 pci_free_consistent(skdev->pdev, nbytes,
4630 skspcl->msg_buf,
4631 skspcl->mb_dma_address);
4632 }
4633
4634 skspcl->msg_buf = NULL;
4635 skspcl->mb_dma_address = 0;
4636
4637 skd_free_sg_list(skdev, skspcl->req.sksg_list,
4638 SKD_N_SG_PER_SPECIAL,
4639 skspcl->req.sksg_dma_address);
4640
4641 skspcl->req.sksg_list = NULL;
4642 skspcl->req.sksg_dma_address = 0;
4643
4644 kfree(skspcl->req.sg);
4645 }
4646
4647 kfree(skdev->skspcl_table);
4648 skdev->skspcl_table = NULL;
4649 }
4650
skd_free_sksb(struct skd_device * skdev)4651 static void skd_free_sksb(struct skd_device *skdev)
4652 {
4653 struct skd_special_context *skspcl;
4654 u32 nbytes;
4655
4656 skspcl = &skdev->internal_skspcl;
4657
4658 if (skspcl->data_buf != NULL) {
4659 nbytes = SKD_N_INTERNAL_BYTES;
4660
4661 pci_free_consistent(skdev->pdev, nbytes,
4662 skspcl->data_buf, skspcl->db_dma_address);
4663 }
4664
4665 skspcl->data_buf = NULL;
4666 skspcl->db_dma_address = 0;
4667
4668 if (skspcl->msg_buf != NULL) {
4669 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4670 pci_free_consistent(skdev->pdev, nbytes,
4671 skspcl->msg_buf, skspcl->mb_dma_address);
4672 }
4673
4674 skspcl->msg_buf = NULL;
4675 skspcl->mb_dma_address = 0;
4676
4677 skd_free_sg_list(skdev, skspcl->req.sksg_list, 1,
4678 skspcl->req.sksg_dma_address);
4679
4680 skspcl->req.sksg_list = NULL;
4681 skspcl->req.sksg_dma_address = 0;
4682 }
4683
skd_free_disk(struct skd_device * skdev)4684 static void skd_free_disk(struct skd_device *skdev)
4685 {
4686 struct gendisk *disk = skdev->disk;
4687
4688 if (disk && (disk->flags & GENHD_FL_UP))
4689 del_gendisk(disk);
4690
4691 if (skdev->queue) {
4692 blk_cleanup_queue(skdev->queue);
4693 skdev->queue = NULL;
4694 disk->queue = NULL;
4695 }
4696
4697 put_disk(disk);
4698 skdev->disk = NULL;
4699 }
4700
skd_destruct(struct skd_device * skdev)4701 static void skd_destruct(struct skd_device *skdev)
4702 {
4703 if (skdev == NULL)
4704 return;
4705
4706
4707 pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
4708 skd_free_disk(skdev);
4709
4710 pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
4711 skd_free_sksb(skdev);
4712
4713 pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
4714 skd_free_skspcl(skdev);
4715
4716 pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
4717 skd_free_skreq(skdev);
4718
4719 pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
4720 skd_free_skmsg(skdev);
4721
4722 pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
4723 skd_free_skcomp(skdev);
4724
4725 pr_debug("%s:%s:%d skdev\n", skdev->name, __func__, __LINE__);
4726 kfree(skdev);
4727 }
4728
4729 /*
4730 *****************************************************************************
4731 * BLOCK DEVICE (BDEV) GLUE
4732 *****************************************************************************
4733 */
4734
skd_bdev_getgeo(struct block_device * bdev,struct hd_geometry * geo)4735 static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
4736 {
4737 struct skd_device *skdev;
4738 u64 capacity;
4739
4740 skdev = bdev->bd_disk->private_data;
4741
4742 pr_debug("%s:%s:%d %s: CMD[%s] getgeo device\n",
4743 skdev->name, __func__, __LINE__,
4744 bdev->bd_disk->disk_name, current->comm);
4745
4746 if (skdev->read_cap_is_valid) {
4747 capacity = get_capacity(skdev->disk);
4748 geo->heads = 64;
4749 geo->sectors = 255;
4750 geo->cylinders = (capacity) / (255 * 64);
4751
4752 return 0;
4753 }
4754 return -EIO;
4755 }
4756
skd_bdev_attach(struct skd_device * skdev)4757 static int skd_bdev_attach(struct skd_device *skdev)
4758 {
4759 pr_debug("%s:%s:%d add_disk\n", skdev->name, __func__, __LINE__);
4760 add_disk(skdev->disk);
4761 return 0;
4762 }
4763
4764 static const struct block_device_operations skd_blockdev_ops = {
4765 .owner = THIS_MODULE,
4766 .ioctl = skd_bdev_ioctl,
4767 .getgeo = skd_bdev_getgeo,
4768 };
4769
4770
4771 /*
4772 *****************************************************************************
4773 * PCIe DRIVER GLUE
4774 *****************************************************************************
4775 */
4776
4777 static const struct pci_device_id skd_pci_tbl[] = {
4778 { PCI_VENDOR_ID_STEC, PCI_DEVICE_ID_S1120,
4779 PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
4780 { 0 } /* terminate list */
4781 };
4782
4783 MODULE_DEVICE_TABLE(pci, skd_pci_tbl);
4784
skd_pci_info(struct skd_device * skdev,char * str)4785 static char *skd_pci_info(struct skd_device *skdev, char *str)
4786 {
4787 int pcie_reg;
4788
4789 strcpy(str, "PCIe (");
4790 pcie_reg = pci_find_capability(skdev->pdev, PCI_CAP_ID_EXP);
4791
4792 if (pcie_reg) {
4793
4794 char lwstr[6];
4795 uint16_t pcie_lstat, lspeed, lwidth;
4796
4797 pcie_reg += 0x12;
4798 pci_read_config_word(skdev->pdev, pcie_reg, &pcie_lstat);
4799 lspeed = pcie_lstat & (0xF);
4800 lwidth = (pcie_lstat & 0x3F0) >> 4;
4801
4802 if (lspeed == 1)
4803 strcat(str, "2.5GT/s ");
4804 else if (lspeed == 2)
4805 strcat(str, "5.0GT/s ");
4806 else
4807 strcat(str, "<unknown> ");
4808 snprintf(lwstr, sizeof(lwstr), "%dX)", lwidth);
4809 strcat(str, lwstr);
4810 }
4811 return str;
4812 }
4813
skd_pci_probe(struct pci_dev * pdev,const struct pci_device_id * ent)4814 static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4815 {
4816 int i;
4817 int rc = 0;
4818 char pci_str[32];
4819 struct skd_device *skdev;
4820
4821 pr_info("STEC s1120 Driver(%s) version %s-b%s\n",
4822 DRV_NAME, DRV_VERSION, DRV_BUILD_ID);
4823 pr_info("(skd?:??:[%s]): vendor=%04X device=%04x\n",
4824 pci_name(pdev), pdev->vendor, pdev->device);
4825
4826 rc = pci_enable_device(pdev);
4827 if (rc)
4828 return rc;
4829 rc = pci_request_regions(pdev, DRV_NAME);
4830 if (rc)
4831 goto err_out;
4832 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
4833 if (!rc) {
4834 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
4835
4836 pr_err("(%s): consistent DMA mask error %d\n",
4837 pci_name(pdev), rc);
4838 }
4839 } else {
4840 (rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)));
4841 if (rc) {
4842
4843 pr_err("(%s): DMA mask error %d\n",
4844 pci_name(pdev), rc);
4845 goto err_out_regions;
4846 }
4847 }
4848
4849 if (!skd_major) {
4850 rc = register_blkdev(0, DRV_NAME);
4851 if (rc < 0)
4852 goto err_out_regions;
4853 BUG_ON(!rc);
4854 skd_major = rc;
4855 }
4856
4857 skdev = skd_construct(pdev);
4858 if (skdev == NULL) {
4859 rc = -ENOMEM;
4860 goto err_out_regions;
4861 }
4862
4863 skd_pci_info(skdev, pci_str);
4864 pr_info("(%s): %s 64bit\n", skd_name(skdev), pci_str);
4865
4866 pci_set_master(pdev);
4867 rc = pci_enable_pcie_error_reporting(pdev);
4868 if (rc) {
4869 pr_err(
4870 "(%s): bad enable of PCIe error reporting rc=%d\n",
4871 skd_name(skdev), rc);
4872 skdev->pcie_error_reporting_is_enabled = 0;
4873 } else
4874 skdev->pcie_error_reporting_is_enabled = 1;
4875
4876
4877 pci_set_drvdata(pdev, skdev);
4878
4879 skdev->disk->driverfs_dev = &pdev->dev;
4880
4881 for (i = 0; i < SKD_MAX_BARS; i++) {
4882 skdev->mem_phys[i] = pci_resource_start(pdev, i);
4883 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
4884 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
4885 skdev->mem_size[i]);
4886 if (!skdev->mem_map[i]) {
4887 pr_err("(%s): Unable to map adapter memory!\n",
4888 skd_name(skdev));
4889 rc = -ENODEV;
4890 goto err_out_iounmap;
4891 }
4892 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
4893 skdev->name, __func__, __LINE__,
4894 skdev->mem_map[i],
4895 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
4896 }
4897
4898 rc = skd_acquire_irq(skdev);
4899 if (rc) {
4900 pr_err("(%s): interrupt resource error %d\n",
4901 skd_name(skdev), rc);
4902 goto err_out_iounmap;
4903 }
4904
4905 rc = skd_start_timer(skdev);
4906 if (rc)
4907 goto err_out_timer;
4908
4909 init_waitqueue_head(&skdev->waitq);
4910
4911 skd_start_device(skdev);
4912
4913 rc = wait_event_interruptible_timeout(skdev->waitq,
4914 (skdev->gendisk_on),
4915 (SKD_START_WAIT_SECONDS * HZ));
4916 if (skdev->gendisk_on > 0) {
4917 /* device came on-line after reset */
4918 skd_bdev_attach(skdev);
4919 rc = 0;
4920 } else {
4921 /* we timed out, something is wrong with the device,
4922 don't add the disk structure */
4923 pr_err(
4924 "(%s): error: waiting for s1120 timed out %d!\n",
4925 skd_name(skdev), rc);
4926 /* in case of no error; we timeout with ENXIO */
4927 if (!rc)
4928 rc = -ENXIO;
4929 goto err_out_timer;
4930 }
4931
4932
4933 #ifdef SKD_VMK_POLL_HANDLER
4934 if (skdev->irq_type == SKD_IRQ_MSIX) {
4935 /* MSIX completion handler is being used for coredump */
4936 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
4937 skdev->msix_entries[5].vector,
4938 skd_comp_q, skdev);
4939 } else {
4940 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
4941 skdev->pdev->irq, skd_isr,
4942 skdev);
4943 }
4944 #endif /* SKD_VMK_POLL_HANDLER */
4945
4946 return rc;
4947
4948 err_out_timer:
4949 skd_stop_device(skdev);
4950 skd_release_irq(skdev);
4951
4952 err_out_iounmap:
4953 for (i = 0; i < SKD_MAX_BARS; i++)
4954 if (skdev->mem_map[i])
4955 iounmap(skdev->mem_map[i]);
4956
4957 if (skdev->pcie_error_reporting_is_enabled)
4958 pci_disable_pcie_error_reporting(pdev);
4959
4960 skd_destruct(skdev);
4961
4962 err_out_regions:
4963 pci_release_regions(pdev);
4964
4965 err_out:
4966 pci_disable_device(pdev);
4967 pci_set_drvdata(pdev, NULL);
4968 return rc;
4969 }
4970
skd_pci_remove(struct pci_dev * pdev)4971 static void skd_pci_remove(struct pci_dev *pdev)
4972 {
4973 int i;
4974 struct skd_device *skdev;
4975
4976 skdev = pci_get_drvdata(pdev);
4977 if (!skdev) {
4978 pr_err("%s: no device data for PCI\n", pci_name(pdev));
4979 return;
4980 }
4981 skd_stop_device(skdev);
4982 skd_release_irq(skdev);
4983
4984 for (i = 0; i < SKD_MAX_BARS; i++)
4985 if (skdev->mem_map[i])
4986 iounmap((u32 *)skdev->mem_map[i]);
4987
4988 if (skdev->pcie_error_reporting_is_enabled)
4989 pci_disable_pcie_error_reporting(pdev);
4990
4991 skd_destruct(skdev);
4992
4993 pci_release_regions(pdev);
4994 pci_disable_device(pdev);
4995 pci_set_drvdata(pdev, NULL);
4996
4997 return;
4998 }
4999
skd_pci_suspend(struct pci_dev * pdev,pm_message_t state)5000 static int skd_pci_suspend(struct pci_dev *pdev, pm_message_t state)
5001 {
5002 int i;
5003 struct skd_device *skdev;
5004
5005 skdev = pci_get_drvdata(pdev);
5006 if (!skdev) {
5007 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5008 return -EIO;
5009 }
5010
5011 skd_stop_device(skdev);
5012
5013 skd_release_irq(skdev);
5014
5015 for (i = 0; i < SKD_MAX_BARS; i++)
5016 if (skdev->mem_map[i])
5017 iounmap((u32 *)skdev->mem_map[i]);
5018
5019 if (skdev->pcie_error_reporting_is_enabled)
5020 pci_disable_pcie_error_reporting(pdev);
5021
5022 pci_release_regions(pdev);
5023 pci_save_state(pdev);
5024 pci_disable_device(pdev);
5025 pci_set_power_state(pdev, pci_choose_state(pdev, state));
5026 return 0;
5027 }
5028
skd_pci_resume(struct pci_dev * pdev)5029 static int skd_pci_resume(struct pci_dev *pdev)
5030 {
5031 int i;
5032 int rc = 0;
5033 struct skd_device *skdev;
5034
5035 skdev = pci_get_drvdata(pdev);
5036 if (!skdev) {
5037 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5038 return -1;
5039 }
5040
5041 pci_set_power_state(pdev, PCI_D0);
5042 pci_enable_wake(pdev, PCI_D0, 0);
5043 pci_restore_state(pdev);
5044
5045 rc = pci_enable_device(pdev);
5046 if (rc)
5047 return rc;
5048 rc = pci_request_regions(pdev, DRV_NAME);
5049 if (rc)
5050 goto err_out;
5051 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
5052 if (!rc) {
5053 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
5054
5055 pr_err("(%s): consistent DMA mask error %d\n",
5056 pci_name(pdev), rc);
5057 }
5058 } else {
5059 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5060 if (rc) {
5061
5062 pr_err("(%s): DMA mask error %d\n",
5063 pci_name(pdev), rc);
5064 goto err_out_regions;
5065 }
5066 }
5067
5068 pci_set_master(pdev);
5069 rc = pci_enable_pcie_error_reporting(pdev);
5070 if (rc) {
5071 pr_err("(%s): bad enable of PCIe error reporting rc=%d\n",
5072 skdev->name, rc);
5073 skdev->pcie_error_reporting_is_enabled = 0;
5074 } else
5075 skdev->pcie_error_reporting_is_enabled = 1;
5076
5077 for (i = 0; i < SKD_MAX_BARS; i++) {
5078
5079 skdev->mem_phys[i] = pci_resource_start(pdev, i);
5080 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
5081 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
5082 skdev->mem_size[i]);
5083 if (!skdev->mem_map[i]) {
5084 pr_err("(%s): Unable to map adapter memory!\n",
5085 skd_name(skdev));
5086 rc = -ENODEV;
5087 goto err_out_iounmap;
5088 }
5089 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
5090 skdev->name, __func__, __LINE__,
5091 skdev->mem_map[i],
5092 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
5093 }
5094 rc = skd_acquire_irq(skdev);
5095 if (rc) {
5096
5097 pr_err("(%s): interrupt resource error %d\n",
5098 pci_name(pdev), rc);
5099 goto err_out_iounmap;
5100 }
5101
5102 rc = skd_start_timer(skdev);
5103 if (rc)
5104 goto err_out_timer;
5105
5106 init_waitqueue_head(&skdev->waitq);
5107
5108 skd_start_device(skdev);
5109
5110 return rc;
5111
5112 err_out_timer:
5113 skd_stop_device(skdev);
5114 skd_release_irq(skdev);
5115
5116 err_out_iounmap:
5117 for (i = 0; i < SKD_MAX_BARS; i++)
5118 if (skdev->mem_map[i])
5119 iounmap(skdev->mem_map[i]);
5120
5121 if (skdev->pcie_error_reporting_is_enabled)
5122 pci_disable_pcie_error_reporting(pdev);
5123
5124 err_out_regions:
5125 pci_release_regions(pdev);
5126
5127 err_out:
5128 pci_disable_device(pdev);
5129 return rc;
5130 }
5131
skd_pci_shutdown(struct pci_dev * pdev)5132 static void skd_pci_shutdown(struct pci_dev *pdev)
5133 {
5134 struct skd_device *skdev;
5135
5136 pr_err("skd_pci_shutdown called\n");
5137
5138 skdev = pci_get_drvdata(pdev);
5139 if (!skdev) {
5140 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5141 return;
5142 }
5143
5144 pr_err("%s: calling stop\n", skd_name(skdev));
5145 skd_stop_device(skdev);
5146 }
5147
5148 static struct pci_driver skd_driver = {
5149 .name = DRV_NAME,
5150 .id_table = skd_pci_tbl,
5151 .probe = skd_pci_probe,
5152 .remove = skd_pci_remove,
5153 .suspend = skd_pci_suspend,
5154 .resume = skd_pci_resume,
5155 .shutdown = skd_pci_shutdown,
5156 };
5157
5158 /*
5159 *****************************************************************************
5160 * LOGGING SUPPORT
5161 *****************************************************************************
5162 */
5163
skd_name(struct skd_device * skdev)5164 static const char *skd_name(struct skd_device *skdev)
5165 {
5166 memset(skdev->id_str, 0, sizeof(skdev->id_str));
5167
5168 if (skdev->inquiry_is_valid)
5169 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:%s:[%s]",
5170 skdev->name, skdev->inq_serial_num,
5171 pci_name(skdev->pdev));
5172 else
5173 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:??:[%s]",
5174 skdev->name, pci_name(skdev->pdev));
5175
5176 return skdev->id_str;
5177 }
5178
skd_drive_state_to_str(int state)5179 const char *skd_drive_state_to_str(int state)
5180 {
5181 switch (state) {
5182 case FIT_SR_DRIVE_OFFLINE:
5183 return "OFFLINE";
5184 case FIT_SR_DRIVE_INIT:
5185 return "INIT";
5186 case FIT_SR_DRIVE_ONLINE:
5187 return "ONLINE";
5188 case FIT_SR_DRIVE_BUSY:
5189 return "BUSY";
5190 case FIT_SR_DRIVE_FAULT:
5191 return "FAULT";
5192 case FIT_SR_DRIVE_DEGRADED:
5193 return "DEGRADED";
5194 case FIT_SR_PCIE_LINK_DOWN:
5195 return "INK_DOWN";
5196 case FIT_SR_DRIVE_SOFT_RESET:
5197 return "SOFT_RESET";
5198 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
5199 return "NEED_FW";
5200 case FIT_SR_DRIVE_INIT_FAULT:
5201 return "INIT_FAULT";
5202 case FIT_SR_DRIVE_BUSY_SANITIZE:
5203 return "BUSY_SANITIZE";
5204 case FIT_SR_DRIVE_BUSY_ERASE:
5205 return "BUSY_ERASE";
5206 case FIT_SR_DRIVE_FW_BOOTING:
5207 return "FW_BOOTING";
5208 default:
5209 return "???";
5210 }
5211 }
5212
skd_skdev_state_to_str(enum skd_drvr_state state)5213 const char *skd_skdev_state_to_str(enum skd_drvr_state state)
5214 {
5215 switch (state) {
5216 case SKD_DRVR_STATE_LOAD:
5217 return "LOAD";
5218 case SKD_DRVR_STATE_IDLE:
5219 return "IDLE";
5220 case SKD_DRVR_STATE_BUSY:
5221 return "BUSY";
5222 case SKD_DRVR_STATE_STARTING:
5223 return "STARTING";
5224 case SKD_DRVR_STATE_ONLINE:
5225 return "ONLINE";
5226 case SKD_DRVR_STATE_PAUSING:
5227 return "PAUSING";
5228 case SKD_DRVR_STATE_PAUSED:
5229 return "PAUSED";
5230 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
5231 return "DRAINING_TIMEOUT";
5232 case SKD_DRVR_STATE_RESTARTING:
5233 return "RESTARTING";
5234 case SKD_DRVR_STATE_RESUMING:
5235 return "RESUMING";
5236 case SKD_DRVR_STATE_STOPPING:
5237 return "STOPPING";
5238 case SKD_DRVR_STATE_SYNCING:
5239 return "SYNCING";
5240 case SKD_DRVR_STATE_FAULT:
5241 return "FAULT";
5242 case SKD_DRVR_STATE_DISAPPEARED:
5243 return "DISAPPEARED";
5244 case SKD_DRVR_STATE_BUSY_ERASE:
5245 return "BUSY_ERASE";
5246 case SKD_DRVR_STATE_BUSY_SANITIZE:
5247 return "BUSY_SANITIZE";
5248 case SKD_DRVR_STATE_BUSY_IMMINENT:
5249 return "BUSY_IMMINENT";
5250 case SKD_DRVR_STATE_WAIT_BOOT:
5251 return "WAIT_BOOT";
5252
5253 default:
5254 return "???";
5255 }
5256 }
5257
skd_skmsg_state_to_str(enum skd_fit_msg_state state)5258 static const char *skd_skmsg_state_to_str(enum skd_fit_msg_state state)
5259 {
5260 switch (state) {
5261 case SKD_MSG_STATE_IDLE:
5262 return "IDLE";
5263 case SKD_MSG_STATE_BUSY:
5264 return "BUSY";
5265 default:
5266 return "???";
5267 }
5268 }
5269
skd_skreq_state_to_str(enum skd_req_state state)5270 static const char *skd_skreq_state_to_str(enum skd_req_state state)
5271 {
5272 switch (state) {
5273 case SKD_REQ_STATE_IDLE:
5274 return "IDLE";
5275 case SKD_REQ_STATE_SETUP:
5276 return "SETUP";
5277 case SKD_REQ_STATE_BUSY:
5278 return "BUSY";
5279 case SKD_REQ_STATE_COMPLETED:
5280 return "COMPLETED";
5281 case SKD_REQ_STATE_TIMEOUT:
5282 return "TIMEOUT";
5283 case SKD_REQ_STATE_ABORTED:
5284 return "ABORTED";
5285 default:
5286 return "???";
5287 }
5288 }
5289
skd_log_skdev(struct skd_device * skdev,const char * event)5290 static void skd_log_skdev(struct skd_device *skdev, const char *event)
5291 {
5292 pr_debug("%s:%s:%d (%s) skdev=%p event='%s'\n",
5293 skdev->name, __func__, __LINE__, skdev->name, skdev, event);
5294 pr_debug("%s:%s:%d drive_state=%s(%d) driver_state=%s(%d)\n",
5295 skdev->name, __func__, __LINE__,
5296 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
5297 skd_skdev_state_to_str(skdev->state), skdev->state);
5298 pr_debug("%s:%s:%d busy=%d limit=%d dev=%d lowat=%d\n",
5299 skdev->name, __func__, __LINE__,
5300 skdev->in_flight, skdev->cur_max_queue_depth,
5301 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
5302 pr_debug("%s:%s:%d timestamp=0x%x cycle=%d cycle_ix=%d\n",
5303 skdev->name, __func__, __LINE__,
5304 skdev->timeout_stamp, skdev->skcomp_cycle, skdev->skcomp_ix);
5305 }
5306
skd_log_skmsg(struct skd_device * skdev,struct skd_fitmsg_context * skmsg,const char * event)5307 static void skd_log_skmsg(struct skd_device *skdev,
5308 struct skd_fitmsg_context *skmsg, const char *event)
5309 {
5310 pr_debug("%s:%s:%d (%s) skmsg=%p event='%s'\n",
5311 skdev->name, __func__, __LINE__, skdev->name, skmsg, event);
5312 pr_debug("%s:%s:%d state=%s(%d) id=0x%04x length=%d\n",
5313 skdev->name, __func__, __LINE__,
5314 skd_skmsg_state_to_str(skmsg->state), skmsg->state,
5315 skmsg->id, skmsg->length);
5316 }
5317
skd_log_skreq(struct skd_device * skdev,struct skd_request_context * skreq,const char * event)5318 static void skd_log_skreq(struct skd_device *skdev,
5319 struct skd_request_context *skreq, const char *event)
5320 {
5321 pr_debug("%s:%s:%d (%s) skreq=%p event='%s'\n",
5322 skdev->name, __func__, __LINE__, skdev->name, skreq, event);
5323 pr_debug("%s:%s:%d state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
5324 skdev->name, __func__, __LINE__,
5325 skd_skreq_state_to_str(skreq->state), skreq->state,
5326 skreq->id, skreq->fitmsg_id);
5327 pr_debug("%s:%s:%d timo=0x%x sg_dir=%d n_sg=%d\n",
5328 skdev->name, __func__, __LINE__,
5329 skreq->timeout_stamp, skreq->sg_data_dir, skreq->n_sg);
5330
5331 if (skreq->req != NULL) {
5332 struct request *req = skreq->req;
5333 u32 lba = (u32)blk_rq_pos(req);
5334 u32 count = blk_rq_sectors(req);
5335
5336 pr_debug("%s:%s:%d "
5337 "req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n",
5338 skdev->name, __func__, __LINE__,
5339 req, lba, lba, count, count,
5340 (int)rq_data_dir(req));
5341 } else
5342 pr_debug("%s:%s:%d req=NULL\n",
5343 skdev->name, __func__, __LINE__);
5344 }
5345
5346 /*
5347 *****************************************************************************
5348 * MODULE GLUE
5349 *****************************************************************************
5350 */
5351
skd_init(void)5352 static int __init skd_init(void)
5353 {
5354 pr_info(PFX " v%s-b%s loaded\n", DRV_VERSION, DRV_BUILD_ID);
5355
5356 switch (skd_isr_type) {
5357 case SKD_IRQ_LEGACY:
5358 case SKD_IRQ_MSI:
5359 case SKD_IRQ_MSIX:
5360 break;
5361 default:
5362 pr_err(PFX "skd_isr_type %d invalid, re-set to %d\n",
5363 skd_isr_type, SKD_IRQ_DEFAULT);
5364 skd_isr_type = SKD_IRQ_DEFAULT;
5365 }
5366
5367 if (skd_max_queue_depth < 1 ||
5368 skd_max_queue_depth > SKD_MAX_QUEUE_DEPTH) {
5369 pr_err(PFX "skd_max_queue_depth %d invalid, re-set to %d\n",
5370 skd_max_queue_depth, SKD_MAX_QUEUE_DEPTH_DEFAULT);
5371 skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
5372 }
5373
5374 if (skd_max_req_per_msg < 1 || skd_max_req_per_msg > 14) {
5375 pr_err(PFX "skd_max_req_per_msg %d invalid, re-set to %d\n",
5376 skd_max_req_per_msg, SKD_MAX_REQ_PER_MSG_DEFAULT);
5377 skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
5378 }
5379
5380 if (skd_sgs_per_request < 1 || skd_sgs_per_request > 4096) {
5381 pr_err(PFX "skd_sg_per_request %d invalid, re-set to %d\n",
5382 skd_sgs_per_request, SKD_N_SG_PER_REQ_DEFAULT);
5383 skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
5384 }
5385
5386 if (skd_dbg_level < 0 || skd_dbg_level > 2) {
5387 pr_err(PFX "skd_dbg_level %d invalid, re-set to %d\n",
5388 skd_dbg_level, 0);
5389 skd_dbg_level = 0;
5390 }
5391
5392 if (skd_isr_comp_limit < 0) {
5393 pr_err(PFX "skd_isr_comp_limit %d invalid, set to %d\n",
5394 skd_isr_comp_limit, 0);
5395 skd_isr_comp_limit = 0;
5396 }
5397
5398 if (skd_max_pass_thru < 1 || skd_max_pass_thru > 50) {
5399 pr_err(PFX "skd_max_pass_thru %d invalid, re-set to %d\n",
5400 skd_max_pass_thru, SKD_N_SPECIAL_CONTEXT);
5401 skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
5402 }
5403
5404 return pci_register_driver(&skd_driver);
5405 }
5406
skd_exit(void)5407 static void __exit skd_exit(void)
5408 {
5409 pr_info(PFX " v%s-b%s unloading\n", DRV_VERSION, DRV_BUILD_ID);
5410
5411 pci_unregister_driver(&skd_driver);
5412
5413 if (skd_major)
5414 unregister_blkdev(skd_major, DRV_NAME);
5415 }
5416
5417 module_init(skd_init);
5418 module_exit(skd_exit);
5419