1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2022 Qualcomm Innovation Center. All rights reserved.
4 *
5 * Authors:
6 * Asutosh Das <quic_asutoshd@quicinc.com>
7 * Can Guo <quic_cang@quicinc.com>
8 */
9
10 #include <asm/unaligned.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/module.h>
13 #include <linux/platform_device.h>
14 #include "ufshcd-priv.h"
15 #include <linux/delay.h>
16 #include <scsi/scsi_cmnd.h>
17 #include <linux/bitfield.h>
18 #include <linux/iopoll.h>
19
20 #define MAX_QUEUE_SUP GENMASK(7, 0)
21 #define UFS_MCQ_MIN_RW_QUEUES 2
22 #define UFS_MCQ_MIN_READ_QUEUES 0
23 #define UFS_MCQ_MIN_POLL_QUEUES 0
24 #define QUEUE_EN_OFFSET 31
25 #define QUEUE_ID_OFFSET 16
26
27 #define MCQ_CFG_MAC_MASK GENMASK(16, 8)
28 #define MCQ_QCFG_SIZE 0x40
29 #define MCQ_ENTRY_SIZE_IN_DWORD 8
30 #define CQE_UCD_BA GENMASK_ULL(63, 7)
31
32 /* Max mcq register polling time in microseconds */
33 #define MCQ_POLL_US 500000
34
rw_queue_count_set(const char * val,const struct kernel_param * kp)35 static int rw_queue_count_set(const char *val, const struct kernel_param *kp)
36 {
37 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_RW_QUEUES,
38 num_possible_cpus());
39 }
40
41 static const struct kernel_param_ops rw_queue_count_ops = {
42 .set = rw_queue_count_set,
43 .get = param_get_uint,
44 };
45
46 static unsigned int rw_queues;
47 module_param_cb(rw_queues, &rw_queue_count_ops, &rw_queues, 0644);
48 MODULE_PARM_DESC(rw_queues,
49 "Number of interrupt driven I/O queues used for rw. Default value is nr_cpus");
50
read_queue_count_set(const char * val,const struct kernel_param * kp)51 static int read_queue_count_set(const char *val, const struct kernel_param *kp)
52 {
53 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_READ_QUEUES,
54 num_possible_cpus());
55 }
56
57 static const struct kernel_param_ops read_queue_count_ops = {
58 .set = read_queue_count_set,
59 .get = param_get_uint,
60 };
61
62 static unsigned int read_queues;
63 module_param_cb(read_queues, &read_queue_count_ops, &read_queues, 0644);
64 MODULE_PARM_DESC(read_queues,
65 "Number of interrupt driven read queues used for read. Default value is 0");
66
poll_queue_count_set(const char * val,const struct kernel_param * kp)67 static int poll_queue_count_set(const char *val, const struct kernel_param *kp)
68 {
69 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_POLL_QUEUES,
70 num_possible_cpus());
71 }
72
73 static const struct kernel_param_ops poll_queue_count_ops = {
74 .set = poll_queue_count_set,
75 .get = param_get_uint,
76 };
77
78 static unsigned int poll_queues = 1;
79 module_param_cb(poll_queues, &poll_queue_count_ops, &poll_queues, 0644);
80 MODULE_PARM_DESC(poll_queues,
81 "Number of poll queues used for r/w. Default value is 1");
82
83 /**
84 * ufshcd_mcq_config_mac - Set the #Max Activ Cmds.
85 * @hba: per adapter instance
86 * @max_active_cmds: maximum # of active commands to the device at any time.
87 *
88 * The controller won't send more than the max_active_cmds to the device at
89 * any time.
90 */
ufshcd_mcq_config_mac(struct ufs_hba * hba,u32 max_active_cmds)91 void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds)
92 {
93 u32 val;
94
95 val = ufshcd_readl(hba, REG_UFS_MCQ_CFG);
96 val &= ~MCQ_CFG_MAC_MASK;
97 val |= FIELD_PREP(MCQ_CFG_MAC_MASK, max_active_cmds);
98 ufshcd_writel(hba, val, REG_UFS_MCQ_CFG);
99 }
100 EXPORT_SYMBOL_GPL(ufshcd_mcq_config_mac);
101
102 /**
103 * ufshcd_mcq_req_to_hwq - find the hardware queue on which the
104 * request would be issued.
105 * @hba: per adapter instance
106 * @req: pointer to the request to be issued
107 *
108 * Returns the hardware queue instance on which the request would
109 * be queued.
110 */
ufshcd_mcq_req_to_hwq(struct ufs_hba * hba,struct request * req)111 struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba,
112 struct request *req)
113 {
114 u32 utag = blk_mq_unique_tag(req);
115 u32 hwq = blk_mq_unique_tag_to_hwq(utag);
116
117 return &hba->uhq[hwq];
118 }
119
120 /**
121 * ufshcd_mcq_decide_queue_depth - decide the queue depth
122 * @hba: per adapter instance
123 *
124 * Returns queue-depth on success, non-zero on error
125 *
126 * MAC - Max. Active Command of the Host Controller (HC)
127 * HC wouldn't send more than this commands to the device.
128 * It is mandatory to implement get_hba_mac() to enable MCQ mode.
129 * Calculates and adjusts the queue depth based on the depth
130 * supported by the HC and ufs device.
131 */
ufshcd_mcq_decide_queue_depth(struct ufs_hba * hba)132 int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba)
133 {
134 int mac;
135
136 /* Mandatory to implement get_hba_mac() */
137 mac = ufshcd_mcq_vops_get_hba_mac(hba);
138 if (mac < 0) {
139 dev_err(hba->dev, "Failed to get mac, err=%d\n", mac);
140 return mac;
141 }
142
143 WARN_ON_ONCE(!hba->dev_info.bqueuedepth);
144 /*
145 * max. value of bqueuedepth = 256, mac is host dependent.
146 * It is mandatory for UFS device to define bQueueDepth if
147 * shared queuing architecture is enabled.
148 */
149 return min_t(int, mac, hba->dev_info.bqueuedepth);
150 }
151
ufshcd_mcq_config_nr_queues(struct ufs_hba * hba)152 static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba)
153 {
154 int i;
155 u32 hba_maxq, rem, tot_queues;
156 struct Scsi_Host *host = hba->host;
157
158 /* maxq is 0 based value */
159 hba_maxq = FIELD_GET(MAX_QUEUE_SUP, hba->mcq_capabilities) + 1;
160
161 tot_queues = read_queues + poll_queues + rw_queues;
162
163 if (hba_maxq < tot_queues) {
164 dev_err(hba->dev, "Total queues (%d) exceeds HC capacity (%d)\n",
165 tot_queues, hba_maxq);
166 return -EOPNOTSUPP;
167 }
168
169 rem = hba_maxq;
170
171 if (rw_queues) {
172 hba->nr_queues[HCTX_TYPE_DEFAULT] = rw_queues;
173 rem -= hba->nr_queues[HCTX_TYPE_DEFAULT];
174 } else {
175 rw_queues = num_possible_cpus();
176 }
177
178 if (poll_queues) {
179 hba->nr_queues[HCTX_TYPE_POLL] = poll_queues;
180 rem -= hba->nr_queues[HCTX_TYPE_POLL];
181 }
182
183 if (read_queues) {
184 hba->nr_queues[HCTX_TYPE_READ] = read_queues;
185 rem -= hba->nr_queues[HCTX_TYPE_READ];
186 }
187
188 if (!hba->nr_queues[HCTX_TYPE_DEFAULT])
189 hba->nr_queues[HCTX_TYPE_DEFAULT] = min3(rem, rw_queues,
190 num_possible_cpus());
191
192 for (i = 0; i < HCTX_MAX_TYPES; i++)
193 host->nr_hw_queues += hba->nr_queues[i];
194
195 hba->nr_hw_queues = host->nr_hw_queues;
196 return 0;
197 }
198
ufshcd_mcq_memory_alloc(struct ufs_hba * hba)199 int ufshcd_mcq_memory_alloc(struct ufs_hba *hba)
200 {
201 struct ufs_hw_queue *hwq;
202 size_t utrdl_size, cqe_size;
203 int i;
204
205 for (i = 0; i < hba->nr_hw_queues; i++) {
206 hwq = &hba->uhq[i];
207
208 utrdl_size = sizeof(struct utp_transfer_req_desc) *
209 hwq->max_entries;
210 hwq->sqe_base_addr = dmam_alloc_coherent(hba->dev, utrdl_size,
211 &hwq->sqe_dma_addr,
212 GFP_KERNEL);
213 if (!hwq->sqe_dma_addr) {
214 dev_err(hba->dev, "SQE allocation failed\n");
215 return -ENOMEM;
216 }
217
218 cqe_size = sizeof(struct cq_entry) * hwq->max_entries;
219 hwq->cqe_base_addr = dmam_alloc_coherent(hba->dev, cqe_size,
220 &hwq->cqe_dma_addr,
221 GFP_KERNEL);
222 if (!hwq->cqe_dma_addr) {
223 dev_err(hba->dev, "CQE allocation failed\n");
224 return -ENOMEM;
225 }
226 }
227
228 return 0;
229 }
230
231
232 /* Operation and runtime registers configuration */
233 #define MCQ_CFG_n(r, i) ((r) + MCQ_QCFG_SIZE * (i))
234 #define MCQ_OPR_OFFSET_n(p, i) \
235 (hba->mcq_opr[(p)].offset + hba->mcq_opr[(p)].stride * (i))
236
mcq_opr_base(struct ufs_hba * hba,enum ufshcd_mcq_opr n,int i)237 static void __iomem *mcq_opr_base(struct ufs_hba *hba,
238 enum ufshcd_mcq_opr n, int i)
239 {
240 struct ufshcd_mcq_opr_info_t *opr = &hba->mcq_opr[n];
241
242 return opr->base + opr->stride * i;
243 }
244
ufshcd_mcq_read_cqis(struct ufs_hba * hba,int i)245 u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i)
246 {
247 return readl(mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS);
248 }
249 EXPORT_SYMBOL_GPL(ufshcd_mcq_read_cqis);
250
ufshcd_mcq_write_cqis(struct ufs_hba * hba,u32 val,int i)251 void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i)
252 {
253 writel(val, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS);
254 }
255 EXPORT_SYMBOL_GPL(ufshcd_mcq_write_cqis);
256
257 /*
258 * Current MCQ specification doesn't provide a Task Tag or its equivalent in
259 * the Completion Queue Entry. Find the Task Tag using an indirect method.
260 */
ufshcd_mcq_get_tag(struct ufs_hba * hba,struct ufs_hw_queue * hwq,struct cq_entry * cqe)261 static int ufshcd_mcq_get_tag(struct ufs_hba *hba,
262 struct ufs_hw_queue *hwq,
263 struct cq_entry *cqe)
264 {
265 u64 addr;
266
267 /* sizeof(struct utp_transfer_cmd_desc) must be a multiple of 128 */
268 BUILD_BUG_ON(sizeof(struct utp_transfer_cmd_desc) & GENMASK(6, 0));
269
270 /* Bits 63:7 UCD base address, 6:5 are reserved, 4:0 is SQ ID */
271 addr = (le64_to_cpu(cqe->command_desc_base_addr) & CQE_UCD_BA) -
272 hba->ucdl_dma_addr;
273
274 return div_u64(addr, ufshcd_get_ucd_size(hba));
275 }
276
ufshcd_mcq_process_cqe(struct ufs_hba * hba,struct ufs_hw_queue * hwq)277 static void ufshcd_mcq_process_cqe(struct ufs_hba *hba,
278 struct ufs_hw_queue *hwq)
279 {
280 struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq);
281 int tag = ufshcd_mcq_get_tag(hba, hwq, cqe);
282
283 if (cqe->command_desc_base_addr) {
284 ufshcd_compl_one_cqe(hba, tag, cqe);
285 /* After processed the cqe, mark it empty (invalid) entry */
286 cqe->command_desc_base_addr = 0;
287 }
288 }
289
ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba * hba,struct ufs_hw_queue * hwq)290 void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba,
291 struct ufs_hw_queue *hwq)
292 {
293 unsigned long flags;
294 u32 entries = hwq->max_entries;
295
296 spin_lock_irqsave(&hwq->cq_lock, flags);
297 while (entries > 0) {
298 ufshcd_mcq_process_cqe(hba, hwq);
299 ufshcd_mcq_inc_cq_head_slot(hwq);
300 entries--;
301 }
302
303 ufshcd_mcq_update_cq_tail_slot(hwq);
304 hwq->cq_head_slot = hwq->cq_tail_slot;
305 spin_unlock_irqrestore(&hwq->cq_lock, flags);
306 }
307
ufshcd_mcq_poll_cqe_nolock(struct ufs_hba * hba,struct ufs_hw_queue * hwq)308 unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba,
309 struct ufs_hw_queue *hwq)
310 {
311 unsigned long completed_reqs = 0;
312
313 ufshcd_mcq_update_cq_tail_slot(hwq);
314 while (!ufshcd_mcq_is_cq_empty(hwq)) {
315 ufshcd_mcq_process_cqe(hba, hwq);
316 ufshcd_mcq_inc_cq_head_slot(hwq);
317 completed_reqs++;
318 }
319
320 if (completed_reqs)
321 ufshcd_mcq_update_cq_head(hwq);
322
323 return completed_reqs;
324 }
325 EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_nolock);
326
ufshcd_mcq_poll_cqe_lock(struct ufs_hba * hba,struct ufs_hw_queue * hwq)327 unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
328 struct ufs_hw_queue *hwq)
329 {
330 unsigned long completed_reqs, flags;
331
332 spin_lock_irqsave(&hwq->cq_lock, flags);
333 completed_reqs = ufshcd_mcq_poll_cqe_nolock(hba, hwq);
334 spin_unlock_irqrestore(&hwq->cq_lock, flags);
335
336 return completed_reqs;
337 }
338 EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_lock);
339
ufshcd_mcq_make_queues_operational(struct ufs_hba * hba)340 void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba)
341 {
342 struct ufs_hw_queue *hwq;
343 u16 qsize;
344 int i;
345
346 for (i = 0; i < hba->nr_hw_queues; i++) {
347 hwq = &hba->uhq[i];
348 hwq->id = i;
349 qsize = hwq->max_entries * MCQ_ENTRY_SIZE_IN_DWORD - 1;
350
351 /* Submission Queue Lower Base Address */
352 ufsmcq_writelx(hba, lower_32_bits(hwq->sqe_dma_addr),
353 MCQ_CFG_n(REG_SQLBA, i));
354 /* Submission Queue Upper Base Address */
355 ufsmcq_writelx(hba, upper_32_bits(hwq->sqe_dma_addr),
356 MCQ_CFG_n(REG_SQUBA, i));
357 /* Submission Queue Doorbell Address Offset */
358 ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_SQD, i),
359 MCQ_CFG_n(REG_SQDAO, i));
360 /* Submission Queue Interrupt Status Address Offset */
361 ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_SQIS, i),
362 MCQ_CFG_n(REG_SQISAO, i));
363
364 /* Completion Queue Lower Base Address */
365 ufsmcq_writelx(hba, lower_32_bits(hwq->cqe_dma_addr),
366 MCQ_CFG_n(REG_CQLBA, i));
367 /* Completion Queue Upper Base Address */
368 ufsmcq_writelx(hba, upper_32_bits(hwq->cqe_dma_addr),
369 MCQ_CFG_n(REG_CQUBA, i));
370 /* Completion Queue Doorbell Address Offset */
371 ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_CQD, i),
372 MCQ_CFG_n(REG_CQDAO, i));
373 /* Completion Queue Interrupt Status Address Offset */
374 ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_CQIS, i),
375 MCQ_CFG_n(REG_CQISAO, i));
376
377 /* Save the base addresses for quicker access */
378 hwq->mcq_sq_head = mcq_opr_base(hba, OPR_SQD, i) + REG_SQHP;
379 hwq->mcq_sq_tail = mcq_opr_base(hba, OPR_SQD, i) + REG_SQTP;
380 hwq->mcq_cq_head = mcq_opr_base(hba, OPR_CQD, i) + REG_CQHP;
381 hwq->mcq_cq_tail = mcq_opr_base(hba, OPR_CQD, i) + REG_CQTP;
382
383 /* Reinitializing is needed upon HC reset */
384 hwq->sq_tail_slot = hwq->cq_tail_slot = hwq->cq_head_slot = 0;
385
386 /* Enable Tail Entry Push Status interrupt only for non-poll queues */
387 if (i < hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL])
388 writel(1, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIE);
389
390 /* Completion Queue Enable|Size to Completion Queue Attribute */
391 ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize,
392 MCQ_CFG_n(REG_CQATTR, i));
393
394 /*
395 * Submission Qeueue Enable|Size|Completion Queue ID to
396 * Submission Queue Attribute
397 */
398 ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize |
399 (i << QUEUE_ID_OFFSET),
400 MCQ_CFG_n(REG_SQATTR, i));
401 }
402 }
403 EXPORT_SYMBOL_GPL(ufshcd_mcq_make_queues_operational);
404
ufshcd_mcq_enable_esi(struct ufs_hba * hba)405 void ufshcd_mcq_enable_esi(struct ufs_hba *hba)
406 {
407 ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x2,
408 REG_UFS_MEM_CFG);
409 }
410 EXPORT_SYMBOL_GPL(ufshcd_mcq_enable_esi);
411
ufshcd_mcq_config_esi(struct ufs_hba * hba,struct msi_msg * msg)412 void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg)
413 {
414 ufshcd_writel(hba, msg->address_lo, REG_UFS_ESILBA);
415 ufshcd_writel(hba, msg->address_hi, REG_UFS_ESIUBA);
416 }
417 EXPORT_SYMBOL_GPL(ufshcd_mcq_config_esi);
418
ufshcd_mcq_init(struct ufs_hba * hba)419 int ufshcd_mcq_init(struct ufs_hba *hba)
420 {
421 struct Scsi_Host *host = hba->host;
422 struct ufs_hw_queue *hwq;
423 int ret, i;
424
425 ret = ufshcd_mcq_config_nr_queues(hba);
426 if (ret)
427 return ret;
428
429 ret = ufshcd_vops_mcq_config_resource(hba);
430 if (ret)
431 return ret;
432
433 ret = ufshcd_mcq_vops_op_runtime_config(hba);
434 if (ret) {
435 dev_err(hba->dev, "Operation runtime config failed, ret=%d\n",
436 ret);
437 return ret;
438 }
439 hba->uhq = devm_kzalloc(hba->dev,
440 hba->nr_hw_queues * sizeof(struct ufs_hw_queue),
441 GFP_KERNEL);
442 if (!hba->uhq) {
443 dev_err(hba->dev, "ufs hw queue memory allocation failed\n");
444 return -ENOMEM;
445 }
446
447 for (i = 0; i < hba->nr_hw_queues; i++) {
448 hwq = &hba->uhq[i];
449 hwq->max_entries = hba->nutrs + 1;
450 spin_lock_init(&hwq->sq_lock);
451 spin_lock_init(&hwq->cq_lock);
452 mutex_init(&hwq->sq_mutex);
453 }
454
455 /* The very first HW queue serves device commands */
456 hba->dev_cmd_queue = &hba->uhq[0];
457
458 host->host_tagset = 1;
459 return 0;
460 }
461
ufshcd_mcq_sq_stop(struct ufs_hba * hba,struct ufs_hw_queue * hwq)462 static int ufshcd_mcq_sq_stop(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
463 {
464 void __iomem *reg;
465 u32 id = hwq->id, val;
466 int err;
467
468 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
469 return -ETIMEDOUT;
470
471 writel(SQ_STOP, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC);
472 reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS;
473 err = read_poll_timeout(readl, val, val & SQ_STS, 20,
474 MCQ_POLL_US, false, reg);
475 if (err)
476 dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n",
477 __func__, id, err);
478 return err;
479 }
480
ufshcd_mcq_sq_start(struct ufs_hba * hba,struct ufs_hw_queue * hwq)481 static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
482 {
483 void __iomem *reg;
484 u32 id = hwq->id, val;
485 int err;
486
487 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
488 return -ETIMEDOUT;
489
490 writel(SQ_START, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC);
491 reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS;
492 err = read_poll_timeout(readl, val, !(val & SQ_STS), 20,
493 MCQ_POLL_US, false, reg);
494 if (err)
495 dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n",
496 __func__, id, err);
497 return err;
498 }
499
500 /**
501 * ufshcd_mcq_sq_cleanup - Clean up submission queue resources
502 * associated with the pending command.
503 * @hba - per adapter instance.
504 * @task_tag - The command's task tag.
505 *
506 * Returns 0 for success; error code otherwise.
507 */
ufshcd_mcq_sq_cleanup(struct ufs_hba * hba,int task_tag)508 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)
509 {
510 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
511 struct scsi_cmnd *cmd = lrbp->cmd;
512 struct ufs_hw_queue *hwq;
513 void __iomem *reg, *opr_sqd_base;
514 u32 nexus, id, val;
515 int err;
516
517 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
518 return -ETIMEDOUT;
519
520 if (task_tag != hba->nutrs - UFSHCD_NUM_RESERVED) {
521 if (!cmd)
522 return -EINVAL;
523 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
524 } else {
525 hwq = hba->dev_cmd_queue;
526 }
527
528 id = hwq->id;
529
530 mutex_lock(&hwq->sq_mutex);
531
532 /* stop the SQ fetching before working on it */
533 err = ufshcd_mcq_sq_stop(hba, hwq);
534 if (err)
535 goto unlock;
536
537 /* SQCTI = EXT_IID, IID, LUN, Task Tag */
538 nexus = lrbp->lun << 8 | task_tag;
539 opr_sqd_base = mcq_opr_base(hba, OPR_SQD, id);
540 writel(nexus, opr_sqd_base + REG_SQCTI);
541
542 /* SQRTCy.ICU = 1 */
543 writel(SQ_ICU, opr_sqd_base + REG_SQRTC);
544
545 /* Poll SQRTSy.CUS = 1. Return result from SQRTSy.RTC */
546 reg = opr_sqd_base + REG_SQRTS;
547 err = read_poll_timeout(readl, val, val & SQ_CUS, 20,
548 MCQ_POLL_US, false, reg);
549 if (err)
550 dev_err(hba->dev, "%s: failed. hwq=%d, tag=%d err=%ld\n",
551 __func__, id, task_tag,
552 FIELD_GET(SQ_ICU_ERR_CODE_MASK, readl(reg)));
553
554 if (ufshcd_mcq_sq_start(hba, hwq))
555 err = -ETIMEDOUT;
556
557 unlock:
558 mutex_unlock(&hwq->sq_mutex);
559 return err;
560 }
561
562 /**
563 * ufshcd_mcq_nullify_sqe - Nullify the submission queue entry.
564 * Write the sqe's Command Type to 0xF. The host controller will not
565 * fetch any sqe with Command Type = 0xF.
566 *
567 * @utrd - UTP Transfer Request Descriptor to be nullified.
568 */
ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc * utrd)569 static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd)
570 {
571 u32 dword_0;
572
573 dword_0 = le32_to_cpu(utrd->header.dword_0);
574 dword_0 &= ~UPIU_COMMAND_TYPE_MASK;
575 dword_0 |= FIELD_PREP(UPIU_COMMAND_TYPE_MASK, 0xF);
576 utrd->header.dword_0 = cpu_to_le32(dword_0);
577 }
578
579 /**
580 * ufshcd_mcq_sqe_search - Search for the command in the submission queue
581 * If the command is in the submission queue and not issued to the device yet,
582 * nullify the sqe so the host controller will skip fetching the sqe.
583 *
584 * @hba - per adapter instance.
585 * @hwq - Hardware Queue to be searched.
586 * @task_tag - The command's task tag.
587 *
588 * Returns true if the SQE containing the command is present in the SQ
589 * (not fetched by the controller); returns false if the SQE is not in the SQ.
590 */
ufshcd_mcq_sqe_search(struct ufs_hba * hba,struct ufs_hw_queue * hwq,int task_tag)591 static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,
592 struct ufs_hw_queue *hwq, int task_tag)
593 {
594 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
595 struct utp_transfer_req_desc *utrd;
596 __le64 cmd_desc_base_addr;
597 bool ret = false;
598 u64 addr, match;
599 u32 sq_head_slot;
600
601 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
602 return true;
603
604 mutex_lock(&hwq->sq_mutex);
605
606 ufshcd_mcq_sq_stop(hba, hwq);
607 sq_head_slot = ufshcd_mcq_get_sq_head_slot(hwq);
608 if (sq_head_slot == hwq->sq_tail_slot)
609 goto out;
610
611 cmd_desc_base_addr = lrbp->utr_descriptor_ptr->command_desc_base_addr;
612 addr = le64_to_cpu(cmd_desc_base_addr) & CQE_UCD_BA;
613
614 while (sq_head_slot != hwq->sq_tail_slot) {
615 utrd = hwq->sqe_base_addr +
616 sq_head_slot * sizeof(struct utp_transfer_req_desc);
617 match = le64_to_cpu(utrd->command_desc_base_addr) & CQE_UCD_BA;
618 if (addr == match) {
619 ufshcd_mcq_nullify_sqe(utrd);
620 ret = true;
621 goto out;
622 }
623
624 sq_head_slot++;
625 if (sq_head_slot == hwq->max_entries)
626 sq_head_slot = 0;
627 }
628
629 out:
630 ufshcd_mcq_sq_start(hba, hwq);
631 mutex_unlock(&hwq->sq_mutex);
632 return ret;
633 }
634
635 /**
636 * ufshcd_mcq_abort - Abort the command in MCQ.
637 * @cmd - The command to be aborted.
638 *
639 * Returns SUCCESS or FAILED error codes
640 */
ufshcd_mcq_abort(struct scsi_cmnd * cmd)641 int ufshcd_mcq_abort(struct scsi_cmnd *cmd)
642 {
643 struct Scsi_Host *host = cmd->device->host;
644 struct ufs_hba *hba = shost_priv(host);
645 int tag = scsi_cmd_to_rq(cmd)->tag;
646 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
647 struct ufs_hw_queue *hwq;
648 unsigned long flags;
649 int err = FAILED;
650
651 if (!ufshcd_cmd_inflight(lrbp->cmd)) {
652 dev_err(hba->dev,
653 "%s: skip abort. cmd at tag %d already completed.\n",
654 __func__, tag);
655 goto out;
656 }
657
658 /* Skip task abort in case previous aborts failed and report failure */
659 if (lrbp->req_abort_skip) {
660 dev_err(hba->dev, "%s: skip abort. tag %d failed earlier\n",
661 __func__, tag);
662 goto out;
663 }
664
665 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
666
667 if (ufshcd_mcq_sqe_search(hba, hwq, tag)) {
668 /*
669 * Failure. The command should not be "stuck" in SQ for
670 * a long time which resulted in command being aborted.
671 */
672 dev_err(hba->dev, "%s: cmd found in sq. hwq=%d, tag=%d\n",
673 __func__, hwq->id, tag);
674 goto out;
675 }
676
677 /*
678 * The command is not in the submission queue, and it is not
679 * in the completion queue either. Query the device to see if
680 * the command is being processed in the device.
681 */
682 if (ufshcd_try_to_abort_task(hba, tag)) {
683 dev_err(hba->dev, "%s: device abort failed %d\n", __func__, err);
684 lrbp->req_abort_skip = true;
685 goto out;
686 }
687
688 err = SUCCESS;
689 spin_lock_irqsave(&hwq->cq_lock, flags);
690 if (ufshcd_cmd_inflight(lrbp->cmd))
691 ufshcd_release_scsi_cmd(hba, lrbp);
692 spin_unlock_irqrestore(&hwq->cq_lock, flags);
693
694 out:
695 return err;
696 }
697