1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) Rockchip Electronics Co., Ltd.
4 *
5 * Author: Huang Lee <Putin.li@rock-chips.com>
6 */
7
8 #ifndef __LINUX_RGA_DRV_H_
9 #define __LINUX_RGA_DRV_H_
10
11 #include <linux/clk.h>
12 #include <linux/completion.h>
13 #include <linux/debugfs.h>
14 #include <linux/delay.h>
15 #include <linux/device.h>
16 #include <linux/dma-buf-cache.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/err.h>
19 #include <linux/fb.h>
20 #include <linux/fs.h>
21 #include <linux/init.h>
22 #include <linux/interrupt.h>
23 #include <linux/io.h>
24 #include <linux/irq.h>
25 #include <linux/kernel.h>
26 #include <linux/kref.h>
27 #include <linux/miscdevice.h>
28 #include <linux/module.h>
29 #include <linux/mutex.h>
30 #include <linux/of_device.h>
31 #include <linux/platform_device.h>
32 #include <linux/poll.h>
33 #include <linux/regulator/consumer.h>
34 #include <linux/scatterlist.h>
35 #include <linux/sched.h>
36 #include <linux/slab.h>
37 #include <linux/spinlock.h>
38 #include <linux/syscalls.h>
39 #include <linux/time.h>
40 #include <linux/timer.h>
41 #include <linux/uaccess.h>
42 #include <linux/version.h>
43 #include <linux/wait.h>
44 #include <linux/wakelock.h>
45 #include <linux/pm_runtime.h>
46 #include <linux/sched/mm.h>
47
48 #include <asm/cacheflush.h>
49
50 #include <linux/iommu.h>
51 #include <linux/iova.h>
52 #include <linux/dma-iommu.h>
53 #include <linux/dma-map-ops.h>
54 #include <linux/hrtimer.h>
55
56 #include "rga.h"
57 #include "rga_debugger.h"
58
59 #define RGA_CORE_REG_OFFSET 0x10000
60
61 /* sample interval: 1000ms */
62 #define RGA_LOAD_INTERVAL 1000000000
63
64 #if ((defined(CONFIG_RK_IOMMU) || defined(CONFIG_ROCKCHIP_IOMMU)) \
65 && defined(CONFIG_ION_ROCKCHIP))
66 #define CONFIG_RGA_IOMMU
67 #endif
68
69 /* Driver information */
70 #define DRIVER_DESC "RGA multicore Device Driver"
71 #define DRIVER_NAME "rga_multicore"
72
73 #define STR_HELPER(x) #x
74 #define STR(x) STR_HELPER(x)
75
76 #define DRIVER_MAJOR_VERISON 1
77 #define DRIVER_MINOR_VERSION 2
78 #define DRIVER_REVISION_VERSION 0
79
80 #define DRIVER_VERSION (STR(DRIVER_MAJOR_VERISON) "." STR(DRIVER_MINOR_VERSION) \
81 "." STR(DRIVER_REVISION_VERSION))
82
83 /* time limit */
84 #define RGA_ASYNC_TIMEOUT_DELAY 500
85 #define RGA_SYNC_TIMEOUT_DELAY HZ
86 #define RGA_RESET_TIMEOUT 1000
87
88 #define RGA_MAX_SCHEDULER 3
89 #define RGA_MAX_BUS_CLK 10
90
91 #define RGA_BUFFER_POOL_MAX_SIZE 64
92
93 #ifndef ABS
94 #define ABS(X) (((X) < 0) ? (-(X)) : (X))
95 #endif
96
97 #ifndef CLIP
98 #define CLIP(x, a, b) (((x) < (a)) \
99 ? (a) : (((x) > (b)) ? (b) : (x)))
100 #endif
101
102 extern struct rga_drvdata_t *rga_drvdata;
103
104 enum {
105 RGA3_SCHEDULER_CORE0 = 1 << 0,
106 RGA3_SCHEDULER_CORE1 = 1 << 1,
107 RGA2_SCHEDULER_CORE0 = 1 << 2,
108 RGA_CORE_MASK = 0x7,
109 RGA_NONE_CORE = 0x0,
110 };
111
112 enum iommu_dma_cookie_type {
113 IOMMU_DMA_IOVA_COOKIE,
114 IOMMU_DMA_MSI_COOKIE,
115 };
116
117 struct rga_fence_context {
118 unsigned int context;
119 unsigned int seqno;
120 spinlock_t spinlock;
121 };
122
123 struct rga_fence_waiter {
124 /* Base sync driver waiter structure */
125 struct dma_fence_cb waiter;
126
127 struct rga_job *job;
128 };
129
130 struct rga_iommu_dma_cookie {
131 enum iommu_dma_cookie_type type;
132
133 /* Full allocator for IOMMU_DMA_IOVA_COOKIE */
134 struct iova_domain iovad;
135 };
136
137 /*
138 * legacy: Wait for the import process to completely replace the current
139 * dma_map and remove it
140 */
141 struct rga_dma_buffer_t {
142 /* DMABUF information */
143 struct dma_buf *dma_buf;
144 struct dma_buf_attachment *attach;
145 struct sg_table *sgt;
146
147 dma_addr_t iova;
148 unsigned long size;
149 void *vaddr;
150 enum dma_data_direction dir;
151
152 /* It indicates whether the buffer is cached */
153 bool cached;
154
155 struct list_head link;
156 struct kref refcount;
157
158 struct iommu_domain *domain;
159 struct rga_iommu_dma_cookie *cookie;
160
161 /*
162 * use dma_buf directly,
163 * do not call dma_buf_put, such as mpi
164 */
165 bool use_dma_buf;
166
167 bool use_viraddr;
168 };
169
170 struct rga_dma_buffer {
171 /* DMABUF information */
172 struct dma_buf *dma_buf;
173 struct dma_buf_attachment *attach;
174 struct sg_table *sgt;
175 void *vmap_ptr;
176
177 struct iommu_domain *domain;
178 struct rga_iommu_dma_cookie *cookie;
179
180 enum dma_data_direction dir;
181
182 dma_addr_t iova;
183 unsigned long size;
184
185 /* The core of the mapping */
186 int core;
187 };
188
189 struct rga_virt_addr {
190 uint64_t addr;
191
192 struct page **pages;
193 int pages_order;
194 int page_count;
195 unsigned long size;
196
197 int result;
198 };
199
200 struct rga_internal_buffer {
201 /* DMA buffer */
202 struct rga_dma_buffer *dma_buffer;
203 uint32_t dma_buffer_size;
204
205 /* virtual address */
206 struct rga_virt_addr *virt_addr;
207
208 /* physical address */
209 uint64_t phys_addr;
210
211 struct rga_memory_parm memory_parm;
212
213
214 struct mm_struct *current_mm;
215
216 /* memory type. */
217 uint32_t type;
218
219 uint32_t handle;
220
221 uint32_t mm_flag;
222
223 struct kref refcount;
224 };
225
226 /*
227 * yqw add:
228 * In order to use the virtual address to refresh the cache,
229 * it may be merged into sgt later.
230 */
231 struct rga2_mmu_other_t {
232 uint32_t *MMU_src0_base;
233 uint32_t *MMU_src1_base;
234 uint32_t *MMU_dst_base;
235 uint32_t MMU_src0_count;
236 uint32_t MMU_src1_count;
237 uint32_t MMU_dst_count;
238
239 uint32_t MMU_len;
240 bool MMU_map;
241 };
242
243 struct rga_job {
244 struct list_head head;
245 struct rga_req rga_command_base;
246 uint32_t cmd_reg[32 * 8];
247 uint32_t csc_reg[12];
248
249 struct rga_dma_buffer_t *rga_dma_buffer_src0;
250 struct rga_dma_buffer_t *rga_dma_buffer_src1;
251 struct rga_dma_buffer_t *rga_dma_buffer_dst;
252 /* used by rga2 */
253 struct rga_dma_buffer_t *rga_dma_buffer_els;
254
255 struct rga_internal_buffer *src_buffer;
256 struct rga_internal_buffer *src1_buffer;
257 struct rga_internal_buffer *dst_buffer;
258 /* used by rga2 */
259 struct rga_internal_buffer *els_buffer;
260
261 struct dma_buf *dma_buf_src0;
262 struct dma_buf *dma_buf_src1;
263 struct dma_buf *dma_buf_dst;
264 struct dma_buf *dma_buf_els;
265
266 /* for rga2 virtual_address */
267 struct mm_struct *mm;
268 struct rga2_mmu_other_t vir_page_table;
269
270 struct dma_fence *out_fence;
271 struct dma_fence *in_fence;
272 spinlock_t fence_lock;
273 ktime_t timestamp;
274 ktime_t running_time;
275 unsigned int flags;
276 int job_id;
277 int priority;
278 int core;
279 int ret;
280 };
281
282 struct rga_scheduler_t;
283
284 struct rga_backend_ops {
285 int (*get_version)(struct rga_scheduler_t *scheduler);
286 int (*set_reg)(struct rga_job *job, struct rga_scheduler_t *scheduler);
287 int (*init_reg)(struct rga_job *job);
288 void (*soft_reset)(struct rga_scheduler_t *scheduler);
289 };
290
291 struct rga_timer {
292 u32 busy_time;
293 u32 busy_time_record;
294 };
295
296 struct rga_scheduler_t {
297 struct device *dev;
298 void __iomem *rga_base;
299
300 struct clk *clks[RGA_MAX_BUS_CLK];
301 int num_clks;
302
303 int pd_refcount;
304
305 struct rga_job *running_job;
306 struct list_head todo_list;
307 spinlock_t irq_lock;
308 wait_queue_head_t job_done_wq;
309 const struct rga_backend_ops *ops;
310 const struct rga_hw_data *data;
311 int job_count;
312 int irq;
313 struct rga_version_t version;
314 int core;
315
316 struct rga_timer timer;
317 };
318
319 struct rga_drvdata_t {
320 struct miscdevice miscdev;
321
322 struct rga_fence_context *fence_ctx;
323
324 /* used by rga2's mmu lock */
325 struct mutex lock;
326
327 struct rga_scheduler_t *rga_scheduler[RGA_MAX_SCHEDULER];
328 int num_of_scheduler;
329
330 struct delayed_work power_off_work;
331 struct wake_lock wake_lock;
332
333 struct rga_mm *mm;
334
335 #ifdef CONFIG_ROCKCHIP_RGA_DEBUGGER
336 struct rga_debugger *debugger;
337 #endif
338 };
339
340 struct rga_irqs_data_t {
341 const char *name;
342 irqreturn_t (*irq_hdl)(int irq, void *ctx);
343 irqreturn_t (*irq_thread)(int irq, void *ctx);
344 };
345
346 struct rga_match_data_t {
347 const char * const *clks;
348 int num_clks;
349 const struct rga_irqs_data_t *irqs;
350 int num_irqs;
351 };
352
rga_read(int offset,struct rga_scheduler_t * rga_scheduler)353 static inline int rga_read(int offset, struct rga_scheduler_t *rga_scheduler)
354 {
355 return readl(rga_scheduler->rga_base + offset);
356 }
357
rga_write(int value,int offset,struct rga_scheduler_t * rga_scheduler)358 static inline void rga_write(int value, int offset, struct rga_scheduler_t *rga_scheduler)
359 {
360 writel(value, rga_scheduler->rga_base + offset);
361 }
362
363 int rga_power_enable(struct rga_scheduler_t *rga_scheduler);
364 int rga_power_disable(struct rga_scheduler_t *rga_scheduler);
365
366 #endif /* __LINUX_RGA_FENCE_H_ */
367