1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * AMD Passthru DMA device driver
4 * -- Based on the CCP driver
5 *
6 * Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
7 *
8 * Author: Sanjay R Mehta <sanju.mehta@amd.com>
9 * Author: Gary R Hook <gary.hook@amd.com>
10 */
11
12 #include <linux/bitfield.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/debugfs.h>
15 #include <linux/interrupt.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18 #include <linux/pci.h>
19
20 #include "ptdma.h"
21
22 /* Human-readable error strings */
23 static char *pt_error_codes[] = {
24 "",
25 "ERR 01: ILLEGAL_ENGINE",
26 "ERR 03: ILLEGAL_FUNCTION_TYPE",
27 "ERR 04: ILLEGAL_FUNCTION_MODE",
28 "ERR 06: ILLEGAL_FUNCTION_SIZE",
29 "ERR 08: ILLEGAL_FUNCTION_RSVD",
30 "ERR 09: ILLEGAL_BUFFER_LENGTH",
31 "ERR 10: VLSB_FAULT",
32 "ERR 11: ILLEGAL_MEM_ADDR",
33 "ERR 12: ILLEGAL_MEM_SEL",
34 "ERR 13: ILLEGAL_CONTEXT_ID",
35 "ERR 15: 0xF Reserved",
36 "ERR 18: CMD_TIMEOUT",
37 "ERR 19: IDMA0_AXI_SLVERR",
38 "ERR 20: IDMA0_AXI_DECERR",
39 "ERR 21: 0x15 Reserved",
40 "ERR 22: IDMA1_AXI_SLAVE_FAULT",
41 "ERR 23: IDMA1_AIXI_DECERR",
42 "ERR 24: 0x18 Reserved",
43 "ERR 27: 0x1B Reserved",
44 "ERR 38: ODMA0_AXI_SLVERR",
45 "ERR 39: ODMA0_AXI_DECERR",
46 "ERR 40: 0x28 Reserved",
47 "ERR 41: ODMA1_AXI_SLVERR",
48 "ERR 42: ODMA1_AXI_DECERR",
49 "ERR 43: LSB_PARITY_ERR",
50 };
51
pt_log_error(struct pt_device * d,int e)52 static void pt_log_error(struct pt_device *d, int e)
53 {
54 dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e);
55 }
56
pt_start_queue(struct pt_cmd_queue * cmd_q)57 void pt_start_queue(struct pt_cmd_queue *cmd_q)
58 {
59 /* Turn on the run bit */
60 iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control);
61 }
62
pt_stop_queue(struct pt_cmd_queue * cmd_q)63 void pt_stop_queue(struct pt_cmd_queue *cmd_q)
64 {
65 /* Turn off the run bit */
66 iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control);
67 }
68
pt_core_execute_cmd(struct ptdma_desc * desc,struct pt_cmd_queue * cmd_q)69 static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q)
70 {
71 bool soc = FIELD_GET(DWORD0_SOC, desc->dw0);
72 u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx];
73 u32 tail;
74 unsigned long flags;
75
76 if (soc) {
77 desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0);
78 desc->dw0 &= ~DWORD0_SOC;
79 }
80 spin_lock_irqsave(&cmd_q->q_lock, flags);
81
82 /* Copy 32-byte command descriptor to hw queue. */
83 memcpy(q_desc, desc, 32);
84 cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN;
85
86 /* The data used by this command must be flushed to memory */
87 wmb();
88
89 /* Write the new tail address back to the queue register */
90 tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
91 iowrite32(tail, cmd_q->reg_control + 0x0004);
92
93 /* Turn the queue back on using our cached control register */
94 pt_start_queue(cmd_q);
95 spin_unlock_irqrestore(&cmd_q->q_lock, flags);
96
97 return 0;
98 }
99
pt_core_perform_passthru(struct pt_cmd_queue * cmd_q,struct pt_passthru_engine * pt_engine)100 int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q,
101 struct pt_passthru_engine *pt_engine)
102 {
103 struct ptdma_desc desc;
104
105 cmd_q->cmd_error = 0;
106 cmd_q->total_pt_ops++;
107 memset(&desc, 0, sizeof(desc));
108 desc.dw0 = CMD_DESC_DW0_VAL;
109 desc.length = pt_engine->src_len;
110 desc.src_lo = lower_32_bits(pt_engine->src_dma);
111 desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma);
112 desc.dst_lo = lower_32_bits(pt_engine->dst_dma);
113 desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma);
114
115 return pt_core_execute_cmd(&desc, cmd_q);
116 }
117
pt_core_disable_queue_interrupts(struct pt_device * pt)118 static inline void pt_core_disable_queue_interrupts(struct pt_device *pt)
119 {
120 iowrite32(0, pt->cmd_q.reg_control + 0x000C);
121 }
122
pt_core_enable_queue_interrupts(struct pt_device * pt)123 static inline void pt_core_enable_queue_interrupts(struct pt_device *pt)
124 {
125 iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C);
126 }
127
pt_do_cmd_complete(unsigned long data)128 static void pt_do_cmd_complete(unsigned long data)
129 {
130 struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data;
131 struct pt_cmd *cmd = tdata->cmd;
132 struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q;
133 u32 tail;
134
135 if (cmd_q->cmd_error) {
136 /*
137 * Log the error and flush the queue by
138 * moving the head pointer
139 */
140 tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
141 pt_log_error(cmd_q->pt, cmd_q->cmd_error);
142 iowrite32(tail, cmd_q->reg_control + 0x0008);
143 }
144
145 cmd->pt_cmd_callback(cmd->data, cmd->ret);
146 }
147
pt_core_irq_handler(int irq,void * data)148 static irqreturn_t pt_core_irq_handler(int irq, void *data)
149 {
150 struct pt_device *pt = data;
151 struct pt_cmd_queue *cmd_q = &pt->cmd_q;
152 u32 status;
153
154 pt_core_disable_queue_interrupts(pt);
155 pt->total_interrupts++;
156 status = ioread32(cmd_q->reg_control + 0x0010);
157 if (status) {
158 cmd_q->int_status = status;
159 cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100);
160 cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104);
161
162 /* On error, only save the first error value */
163 if ((status & INT_ERROR) && !cmd_q->cmd_error)
164 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
165
166 /* Acknowledge the interrupt */
167 iowrite32(status, cmd_q->reg_control + 0x0010);
168 pt_core_enable_queue_interrupts(pt);
169 pt_do_cmd_complete((ulong)&pt->tdata);
170 }
171 return IRQ_HANDLED;
172 }
173
pt_core_init(struct pt_device * pt)174 int pt_core_init(struct pt_device *pt)
175 {
176 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
177 struct pt_cmd_queue *cmd_q = &pt->cmd_q;
178 u32 dma_addr_lo, dma_addr_hi;
179 struct device *dev = pt->dev;
180 struct dma_pool *dma_pool;
181 int ret;
182
183 /* Allocate a dma pool for the queue */
184 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev));
185
186 dma_pool = dma_pool_create(dma_pool_name, dev,
187 PT_DMAPOOL_MAX_SIZE,
188 PT_DMAPOOL_ALIGN, 0);
189 if (!dma_pool)
190 return -ENOMEM;
191
192 /* ptdma core initialisation */
193 iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET);
194 iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET);
195 iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET);
196 iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET);
197 iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET);
198
199 cmd_q->pt = pt;
200 cmd_q->dma_pool = dma_pool;
201 spin_lock_init(&cmd_q->q_lock);
202
203 /* Page alignment satisfies our needs for N <= 128 */
204 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
205 cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize,
206 &cmd_q->qbase_dma,
207 GFP_KERNEL);
208 if (!cmd_q->qbase) {
209 dev_err(dev, "unable to allocate command queue\n");
210 ret = -ENOMEM;
211 goto e_destroy_pool;
212 }
213
214 cmd_q->qidx = 0;
215
216 /* Preset some register values */
217 cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR;
218
219 /* Turn off the queues and disable interrupts until ready */
220 pt_core_disable_queue_interrupts(pt);
221
222 cmd_q->qcontrol = 0; /* Start with nothing */
223 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
224
225 ioread32(cmd_q->reg_control + 0x0104);
226 ioread32(cmd_q->reg_control + 0x0100);
227
228 /* Clear the interrupt status */
229 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
230
231 /* Request an irq */
232 ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt);
233 if (ret) {
234 dev_err(dev, "unable to allocate an IRQ\n");
235 goto e_free_dma;
236 }
237
238 /* Update the device registers with queue information. */
239 cmd_q->qcontrol &= ~CMD_Q_SIZE;
240 cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL);
241
242 cmd_q->qdma_tail = cmd_q->qbase_dma;
243 dma_addr_lo = lower_32_bits(cmd_q->qdma_tail);
244 iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004);
245 iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008);
246
247 dma_addr_hi = upper_32_bits(cmd_q->qdma_tail);
248 cmd_q->qcontrol |= (dma_addr_hi << 16);
249 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
250
251 pt_core_enable_queue_interrupts(pt);
252
253 /* Register the DMA engine support */
254 ret = pt_dmaengine_register(pt);
255 if (ret)
256 goto e_free_irq;
257
258 /* Set up debugfs entries */
259 ptdma_debugfs_setup(pt);
260
261 return 0;
262
263 e_free_irq:
264 free_irq(pt->pt_irq, pt);
265
266 e_free_dma:
267 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma);
268
269 e_destroy_pool:
270 dma_pool_destroy(pt->cmd_q.dma_pool);
271
272 return ret;
273 }
274
pt_core_destroy(struct pt_device * pt)275 void pt_core_destroy(struct pt_device *pt)
276 {
277 struct device *dev = pt->dev;
278 struct pt_cmd_queue *cmd_q = &pt->cmd_q;
279 struct pt_cmd *cmd;
280
281 /* Unregister the DMA engine */
282 pt_dmaengine_unregister(pt);
283
284 /* Disable and clear interrupts */
285 pt_core_disable_queue_interrupts(pt);
286
287 /* Turn off the run bit */
288 pt_stop_queue(cmd_q);
289
290 /* Clear the interrupt status */
291 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
292 ioread32(cmd_q->reg_control + 0x0104);
293 ioread32(cmd_q->reg_control + 0x0100);
294
295 free_irq(pt->pt_irq, pt);
296
297 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
298 cmd_q->qbase_dma);
299
300 /* Flush the cmd queue */
301 while (!list_empty(&pt->cmd)) {
302 /* Invoke the callback directly with an error code */
303 cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry);
304 list_del(&cmd->entry);
305 cmd->pt_cmd_callback(cmd->data, -ENODEV);
306 }
307 }
308