1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2020 Rockchip Electronics Co., Ltd */
3
4 #include <linux/clk.h>
5 #include <linux/delay.h>
6 #include <linux/interrupt.h>
7 #include <linux/io.h>
8 #include <linux/iommu.h>
9 #include <linux/mfd/syscon.h>
10 #include <linux/module.h>
11 #include <linux/of.h>
12 #include <linux/of_graph.h>
13 #include <linux/of_platform.h>
14 #include <linux/of_reserved_mem.h>
15 #include <linux/pinctrl/consumer.h>
16 #include <linux/pm_runtime.h>
17 #include <linux/reset.h>
18 #include <media/videobuf2-dma-contig.h>
19 #include <media/videobuf2-dma-sg.h>
20 #include <soc/rockchip/rockchip_iommu.h>
21
22 #include "common.h"
23 #include "dev.h"
24 #include "hw.h"
25 #include "regs.h"
26
27 /*
28 * rkisp_hw share hardware resource with rkisp virtual device
29 * rkisp_device rkisp_device rkisp_device rkisp_device
30 * | | | |
31 * \ | | /
32 * --------------------------------------
33 * |
34 * rkisp_hw
35 */
36
37 struct isp_irqs_data {
38 const char *name;
39 irqreturn_t (*irq_hdl)(int irq, void *ctx);
40 };
41
42 /* using default value if reg no write for multi device */
default_sw_reg_flag(struct rkisp_device * dev)43 static void default_sw_reg_flag(struct rkisp_device *dev)
44 {
45 u32 v20_reg[] = {
46 CTRL_VI_ISP_PATH, IMG_EFF_CTRL, ISP_CCM_CTRL,
47 CPROC_CTRL, DUAL_CROP_CTRL, ISP_GAMMA_OUT_CTRL,
48 ISP_LSC_CTRL, ISP_DEBAYER_CONTROL, ISP_WDR_CTRL,
49 ISP_GIC_CONTROL, ISP_BLS_CTRL, ISP_DPCC0_MODE,
50 ISP_DPCC1_MODE, ISP_DPCC2_MODE, ISP_HDRMGE_CTRL,
51 ISP_HDRTMO_CTRL, ISP_RAWNR_CTRL, ISP_LDCH_STS,
52 ISP_DHAZ_CTRL, ISP_3DLUT_CTRL, ISP_GAIN_CTRL,
53 ISP_AFM_CTRL, ISP_HIST_HIST_CTRL, RAWAE_BIG1_BASE,
54 RAWAE_BIG2_BASE, RAWAE_BIG3_BASE, ISP_RAWAE_LITE_CTRL,
55 ISP_RAWHIST_LITE_CTRL, ISP_RAWHIST_BIG1_BASE,
56 ISP_RAWHIST_BIG2_BASE, ISP_RAWHIST_BIG3_BASE,
57 ISP_YUVAE_CTRL, ISP_RAWAF_CTRL, ISP_RAWAWB_CTRL,
58 };
59 u32 v21_reg[] = {
60 CTRL_VI_ISP_PATH, IMG_EFF_CTRL, ISP_CCM_CTRL,
61 CPROC_CTRL, DUAL_CROP_CTRL, ISP_GAMMA_OUT_CTRL,
62 SELF_RESIZE_CTRL, MAIN_RESIZE_CTRL, ISP_LSC_CTRL,
63 ISP_DEBAYER_CONTROL, ISP21_YNR_GLOBAL_CTRL,
64 ISP21_CNR_CTRL, ISP21_SHARP_SHARP_EN, ISP_GIC_CONTROL,
65 ISP_BLS_CTRL, ISP_DPCC0_MODE, ISP_DPCC1_MODE,
66 ISP_HDRMGE_CTRL, ISP21_DRC_CTRL0, ISP21_BAYNR_CTRL,
67 ISP21_BAY3D_CTRL, ISP_LDCH_STS, ISP21_DHAZ_CTRL,
68 ISP_3DLUT_CTRL, ISP_AFM_CTRL, ISP_HIST_HIST_CTRL,
69 RAWAE_BIG1_BASE, RAWAE_BIG2_BASE, RAWAE_BIG3_BASE,
70 ISP_RAWAE_LITE_CTRL, ISP_RAWHIST_LITE_CTRL,
71 ISP_RAWHIST_BIG1_BASE, ISP_RAWHIST_BIG2_BASE,
72 ISP_RAWHIST_BIG3_BASE, ISP_YUVAE_CTRL, ISP_RAWAF_CTRL,
73 ISP21_RAWAWB_CTRL,
74 };
75 u32 v30_reg[] = {
76 ISP3X_VI_ISP_PATH, ISP3X_IMG_EFF_CTRL, ISP3X_CMSK_CTRL0,
77 ISP3X_CCM_CTRL, ISP3X_CPROC_CTRL, ISP3X_DUAL_CROP_CTRL,
78 ISP3X_GAMMA_OUT_CTRL, ISP3X_SELF_RESIZE_CTRL, ISP3X_MAIN_RESIZE_CTRL,
79 ISP3X_LSC_CTRL, ISP3X_DEBAYER_CONTROL, ISP3X_CAC_CTRL,
80 ISP3X_YNR_GLOBAL_CTRL, ISP3X_CNR_CTRL, ISP3X_SHARP_EN,
81 ISP3X_BAY3D_CTRL, ISP3X_GIC_CONTROL, ISP3X_BLS_CTRL,
82 ISP3X_DPCC0_MODE, ISP3X_DPCC1_MODE, ISP3X_DPCC2_MODE,
83 ISP3X_HDRMGE_CTRL, ISP3X_DRC_CTRL0, ISP3X_BAYNR_CTRL,
84 ISP3X_LDCH_STS, ISP3X_DHAZ_CTRL, ISP3X_3DLUT_CTRL,
85 ISP3X_GAIN_CTRL, ISP3X_RAWAE_LITE_CTRL, ISP3X_RAWAE_BIG1_BASE,
86 ISP3X_RAWAE_BIG2_BASE, ISP3X_RAWAE_BIG3_BASE, ISP3X_RAWHIST_LITE_CTRL,
87 ISP3X_RAWHIST_BIG1_BASE, ISP3X_RAWHIST_BIG2_BASE, ISP3X_RAWHIST_BIG3_BASE,
88 ISP3X_RAWAF_CTRL, ISP3X_RAWAWB_CTRL,
89 };
90 u32 i, *flag, *reg, size;
91
92 switch (dev->isp_ver) {
93 case ISP_V20:
94 reg = v20_reg;
95 size = ARRAY_SIZE(v20_reg);
96 break;
97 case ISP_V21:
98 reg = v21_reg;
99 size = ARRAY_SIZE(v21_reg);
100 break;
101 case ISP_V30:
102 reg = v30_reg;
103 size = ARRAY_SIZE(v30_reg);
104 break;
105 default:
106 return;
107 }
108
109 for (i = 0; i < size; i++) {
110 flag = dev->sw_base_addr + reg[i] + RKISP_ISP_SW_REG_SIZE;
111 *flag = SW_REG_CACHE;
112 if (dev->hw_dev->is_unite) {
113 flag += RKISP_ISP_SW_MAX_SIZE / 4;
114 *flag = SW_REG_CACHE;
115 }
116 }
117 }
118
mipi_irq_hdl(int irq,void * ctx)119 static irqreturn_t mipi_irq_hdl(int irq, void *ctx)
120 {
121 struct device *dev = ctx;
122 struct rkisp_hw_dev *hw_dev = dev_get_drvdata(dev);
123 struct rkisp_device *isp = hw_dev->isp[hw_dev->mipi_dev_id];
124 void __iomem *base = !hw_dev->is_unite ?
125 hw_dev->base_addr : hw_dev->base_next_addr;
126
127 if (hw_dev->is_thunderboot)
128 return IRQ_HANDLED;
129
130 if (hw_dev->isp_ver == ISP_V13 || hw_dev->isp_ver == ISP_V12) {
131 u32 err1, err2, err3;
132
133 err1 = readl(base + CIF_ISP_CSI0_ERR1);
134 err2 = readl(base + CIF_ISP_CSI0_ERR2);
135 err3 = readl(base + CIF_ISP_CSI0_ERR3);
136
137 if (err1 || err2 || err3)
138 rkisp_mipi_v13_isr(err1, err2, err3, isp);
139 } else if (hw_dev->isp_ver == ISP_V20 ||
140 hw_dev->isp_ver == ISP_V21 ||
141 hw_dev->isp_ver == ISP_V30) {
142 u32 phy, packet, overflow, state;
143
144 state = readl(base + CSI2RX_ERR_STAT);
145 phy = readl(base + CSI2RX_ERR_PHY);
146 packet = readl(base + CSI2RX_ERR_PACKET);
147 overflow = readl(base + CSI2RX_ERR_OVERFLOW);
148 if (phy | packet | overflow | state) {
149 if (hw_dev->isp_ver == ISP_V20)
150 rkisp_mipi_v20_isr(phy, packet, overflow, state, isp);
151 else if (hw_dev->isp_ver == ISP_V21)
152 rkisp_mipi_v21_isr(phy, packet, overflow, state, isp);
153 else
154 rkisp_mipi_v30_isr(phy, packet, overflow, state, isp);
155 }
156 } else {
157 u32 mis_val = readl(base + CIF_MIPI_MIS);
158
159 if (mis_val)
160 rkisp_mipi_isr(mis_val, isp);
161 }
162
163 return IRQ_HANDLED;
164 }
165
mi_irq_hdl(int irq,void * ctx)166 static irqreturn_t mi_irq_hdl(int irq, void *ctx)
167 {
168 struct device *dev = ctx;
169 struct rkisp_hw_dev *hw_dev = dev_get_drvdata(dev);
170 struct rkisp_device *isp = hw_dev->isp[hw_dev->cur_dev_id];
171 void __iomem *base = !hw_dev->is_unite ?
172 hw_dev->base_addr : hw_dev->base_next_addr;
173 u32 mis_val, tx_isr = MI_RAW0_WR_FRAME | MI_RAW1_WR_FRAME |
174 MI_RAW2_WR_FRAME | MI_RAW3_WR_FRAME;
175
176 if (hw_dev->is_thunderboot)
177 return IRQ_HANDLED;
178
179 mis_val = readl(base + CIF_MI_MIS);
180 if (mis_val) {
181 if (mis_val & ~tx_isr)
182 rkisp_mi_isr(mis_val & ~tx_isr, isp);
183 if (mis_val & tx_isr) {
184 isp = hw_dev->isp[hw_dev->mipi_dev_id];
185 rkisp_mi_isr(mis_val & tx_isr, isp);
186 }
187 }
188 return IRQ_HANDLED;
189 }
190
isp_irq_hdl(int irq,void * ctx)191 static irqreturn_t isp_irq_hdl(int irq, void *ctx)
192 {
193 struct device *dev = ctx;
194 struct rkisp_hw_dev *hw_dev = dev_get_drvdata(dev);
195 struct rkisp_device *isp = hw_dev->isp[hw_dev->cur_dev_id];
196 void __iomem *base = !hw_dev->is_unite ?
197 hw_dev->base_addr : hw_dev->base_next_addr;
198 unsigned int mis_val, mis_3a = 0;
199
200 if (hw_dev->is_thunderboot)
201 return IRQ_HANDLED;
202
203 mis_val = readl(base + CIF_ISP_MIS);
204 if (hw_dev->isp_ver == ISP_V20 ||
205 hw_dev->isp_ver == ISP_V21 ||
206 hw_dev->isp_ver == ISP_V30)
207 mis_3a = readl(base + ISP_ISP3A_MIS);
208 if (mis_val || mis_3a)
209 rkisp_isp_isr(mis_val, mis_3a, isp);
210
211 return IRQ_HANDLED;
212 }
213
irq_handler(int irq,void * ctx)214 static irqreturn_t irq_handler(int irq, void *ctx)
215 {
216 struct device *dev = ctx;
217 struct rkisp_hw_dev *hw_dev = dev_get_drvdata(dev);
218 struct rkisp_device *isp = hw_dev->isp[hw_dev->cur_dev_id];
219 unsigned int mis_val, mis_3a = 0;
220
221 mis_val = readl(hw_dev->base_addr + CIF_ISP_MIS);
222 if (hw_dev->isp_ver == ISP_V20 ||
223 hw_dev->isp_ver == ISP_V21 ||
224 hw_dev->isp_ver == ISP_V30)
225 mis_3a = readl(hw_dev->base_addr + ISP_ISP3A_MIS);
226 if (mis_val || mis_3a)
227 rkisp_isp_isr(mis_val, mis_3a, isp);
228
229 mis_val = readl(hw_dev->base_addr + CIF_MIPI_MIS);
230 if (mis_val)
231 rkisp_mipi_isr(mis_val, isp);
232
233 mis_val = readl(hw_dev->base_addr + CIF_MI_MIS);
234 if (mis_val)
235 rkisp_mi_isr(mis_val, isp);
236
237 return IRQ_HANDLED;
238 }
239
rkisp_register_irq(struct rkisp_hw_dev * hw_dev)240 int rkisp_register_irq(struct rkisp_hw_dev *hw_dev)
241 {
242 const struct isp_match_data *match_data = hw_dev->match_data;
243 struct platform_device *pdev = hw_dev->pdev;
244 struct device *dev = &pdev->dev;
245 struct resource *res;
246 int i, ret, irq;
247
248 res = platform_get_resource_byname(pdev, IORESOURCE_IRQ,
249 match_data->irqs[0].name);
250 if (res) {
251 /* there are irq names in dts */
252 for (i = 0; i < match_data->num_irqs; i++) {
253 irq = platform_get_irq_byname(pdev, match_data->irqs[i].name);
254 if (irq < 0) {
255 dev_err(dev, "no irq %s in dts\n",
256 match_data->irqs[i].name);
257 return irq;
258 }
259
260 if (!strcmp(match_data->irqs[i].name, "mipi_irq"))
261 hw_dev->mipi_irq = irq;
262
263 ret = devm_request_irq(dev, irq,
264 match_data->irqs[i].irq_hdl,
265 IRQF_SHARED,
266 dev_driver_string(dev),
267 dev);
268 if (ret < 0) {
269 dev_err(dev, "request %s failed: %d\n",
270 match_data->irqs[i].name, ret);
271 return ret;
272 }
273
274 if (hw_dev->mipi_irq == irq &&
275 (hw_dev->isp_ver == ISP_V12 ||
276 hw_dev->isp_ver == ISP_V13))
277 disable_irq(hw_dev->mipi_irq);
278 }
279 } else {
280 /* no irq names in dts */
281 irq = platform_get_irq(pdev, 0);
282 if (irq < 0) {
283 dev_err(dev, "no isp irq in dts\n");
284 return irq;
285 }
286
287 ret = devm_request_irq(dev, irq,
288 irq_handler,
289 IRQF_SHARED,
290 dev_driver_string(dev),
291 dev);
292 if (ret < 0) {
293 dev_err(dev, "request irq failed: %d\n", ret);
294 return ret;
295 }
296 }
297
298 return 0;
299 }
300
301 static const char * const rk1808_isp_clks[] = {
302 "clk_isp",
303 "aclk_isp",
304 "hclk_isp",
305 "pclk_isp",
306 };
307
308 static const char * const rk3288_isp_clks[] = {
309 "clk_isp",
310 "aclk_isp",
311 "hclk_isp",
312 "pclk_isp_in",
313 "sclk_isp_jpe",
314 };
315
316 static const char * const rk3326_isp_clks[] = {
317 "clk_isp",
318 "aclk_isp",
319 "hclk_isp",
320 "pclk_isp",
321 };
322
323 static const char * const rk3368_isp_clks[] = {
324 "clk_isp",
325 "aclk_isp",
326 "hclk_isp",
327 "pclk_isp",
328 };
329
330 static const char * const rk3399_isp_clks[] = {
331 "clk_isp",
332 "aclk_isp",
333 "hclk_isp",
334 "aclk_isp_wrap",
335 "hclk_isp_wrap",
336 "pclk_isp_wrap"
337 };
338
339 static const char * const rk3568_isp_clks[] = {
340 "clk_isp",
341 "aclk_isp",
342 "hclk_isp",
343 };
344
345 static const char * const rk3588_isp_clks[] = {
346 "clk_isp_core",
347 "aclk_isp",
348 "hclk_isp",
349 "clk_isp_core_marvin",
350 "clk_isp_core_vicap",
351 };
352
353 static const char * const rk3588_isp_unite_clks[] = {
354 "clk_isp_core0",
355 "aclk_isp0",
356 "hclk_isp0",
357 "clk_isp_core_marvin0",
358 "clk_isp_core_vicap0",
359 "clk_isp_core1",
360 "aclk_isp1",
361 "hclk_isp1",
362 "clk_isp_core_marvin1",
363 "clk_isp_core_vicap1",
364 };
365
366 static const char * const rv1126_isp_clks[] = {
367 "clk_isp",
368 "aclk_isp",
369 "hclk_isp",
370 };
371
372 /* isp clock adjustment table (MHz) */
373 static const struct isp_clk_info rk1808_isp_clk_rate[] = {
374 {300, }, {400, }, {500, }, {600, }
375 };
376
377 /* isp clock adjustment table (MHz) */
378 static const struct isp_clk_info rk3288_isp_clk_rate[] = {
379 {150, }, {384, }, {500, }, {594, }
380 };
381
382 /* isp clock adjustment table (MHz) */
383 static const struct isp_clk_info rk3326_isp_clk_rate[] = {
384 {300, }, {347, }, {400, }, {520, }, {600, }
385 };
386
387 /* isp clock adjustment table (MHz) */
388 static const struct isp_clk_info rk3368_isp_clk_rate[] = {
389 {300, }, {400, }, {600, }
390 };
391
392 /* isp clock adjustment table (MHz) */
393 static const struct isp_clk_info rk3399_isp_clk_rate[] = {
394 {300, }, {400, }, {600, }
395 };
396
397 static const struct isp_clk_info rk3568_isp_clk_rate[] = {
398 {
399 .clk_rate = 300,
400 .refer_data = 1920, //width
401 }, {
402 .clk_rate = 400,
403 .refer_data = 2688,
404 }, {
405 .clk_rate = 500,
406 .refer_data = 3072,
407 }, {
408 .clk_rate = 600,
409 .refer_data = 3840,
410 }
411 };
412
413 static const struct isp_clk_info rk3588_isp_clk_rate[] = {
414 {
415 .clk_rate = 300,
416 .refer_data = 1920, //width
417 }, {
418 .clk_rate = 400,
419 .refer_data = 2688,
420 }, {
421 .clk_rate = 500,
422 .refer_data = 3072,
423 }, {
424 .clk_rate = 600,
425 .refer_data = 3840,
426 }, {
427 .clk_rate = 702,
428 .refer_data = 4672,
429 }
430 };
431
432 static const struct isp_clk_info rv1126_isp_clk_rate[] = {
433 {
434 .clk_rate = 20,
435 .refer_data = 0,
436 }, {
437 .clk_rate = 300,
438 .refer_data = 1920, //width
439 }, {
440 .clk_rate = 400,
441 .refer_data = 2688,
442 }, {
443 .clk_rate = 500,
444 .refer_data = 3072,
445 }, {
446 .clk_rate = 600,
447 .refer_data = 3840,
448 }
449 };
450
451 static struct isp_irqs_data rk1808_isp_irqs[] = {
452 {"isp_irq", isp_irq_hdl},
453 {"mi_irq", mi_irq_hdl},
454 {"mipi_irq", mipi_irq_hdl}
455 };
456
457 static struct isp_irqs_data rk3288_isp_irqs[] = {
458 {"isp_irq", irq_handler}
459 };
460
461 static struct isp_irqs_data rk3326_isp_irqs[] = {
462 {"isp_irq", isp_irq_hdl},
463 {"mi_irq", mi_irq_hdl},
464 {"mipi_irq", mipi_irq_hdl}
465 };
466
467 static struct isp_irqs_data rk3368_isp_irqs[] = {
468 {"isp_irq", irq_handler}
469 };
470
471 static struct isp_irqs_data rk3399_isp_irqs[] = {
472 {"isp_irq", irq_handler}
473 };
474
475 static struct isp_irqs_data rk3568_isp_irqs[] = {
476 {"isp_irq", isp_irq_hdl},
477 {"mi_irq", mi_irq_hdl},
478 {"mipi_irq", mipi_irq_hdl}
479 };
480
481 static struct isp_irqs_data rk3588_isp_irqs[] = {
482 {"isp_irq", isp_irq_hdl},
483 {"mi_irq", mi_irq_hdl},
484 {"mipi_irq", mipi_irq_hdl}
485 };
486
487 static struct isp_irqs_data rv1126_isp_irqs[] = {
488 {"isp_irq", isp_irq_hdl},
489 {"mi_irq", mi_irq_hdl},
490 {"mipi_irq", mipi_irq_hdl}
491 };
492
493 static const struct isp_match_data rv1126_isp_match_data = {
494 .clks = rv1126_isp_clks,
495 .num_clks = ARRAY_SIZE(rv1126_isp_clks),
496 .isp_ver = ISP_V20,
497 .clk_rate_tbl = rv1126_isp_clk_rate,
498 .num_clk_rate_tbl = ARRAY_SIZE(rv1126_isp_clk_rate),
499 .irqs = rv1126_isp_irqs,
500 .num_irqs = ARRAY_SIZE(rv1126_isp_irqs),
501 .unite = false,
502 };
503
504 static const struct isp_match_data rk1808_isp_match_data = {
505 .clks = rk1808_isp_clks,
506 .num_clks = ARRAY_SIZE(rk1808_isp_clks),
507 .isp_ver = ISP_V13,
508 .clk_rate_tbl = rk1808_isp_clk_rate,
509 .num_clk_rate_tbl = ARRAY_SIZE(rk1808_isp_clk_rate),
510 .irqs = rk1808_isp_irqs,
511 .num_irqs = ARRAY_SIZE(rk1808_isp_irqs),
512 .unite = false,
513 };
514
515 static const struct isp_match_data rk3288_isp_match_data = {
516 .clks = rk3288_isp_clks,
517 .num_clks = ARRAY_SIZE(rk3288_isp_clks),
518 .isp_ver = ISP_V10,
519 .clk_rate_tbl = rk3288_isp_clk_rate,
520 .num_clk_rate_tbl = ARRAY_SIZE(rk3288_isp_clk_rate),
521 .irqs = rk3288_isp_irqs,
522 .num_irqs = ARRAY_SIZE(rk3288_isp_irqs),
523 .unite = false,
524 };
525
526 static const struct isp_match_data rk3326_isp_match_data = {
527 .clks = rk3326_isp_clks,
528 .num_clks = ARRAY_SIZE(rk3326_isp_clks),
529 .isp_ver = ISP_V12,
530 .clk_rate_tbl = rk3326_isp_clk_rate,
531 .num_clk_rate_tbl = ARRAY_SIZE(rk3326_isp_clk_rate),
532 .irqs = rk3326_isp_irqs,
533 .num_irqs = ARRAY_SIZE(rk3326_isp_irqs),
534 .unite = false,
535 };
536
537 static const struct isp_match_data rk3368_isp_match_data = {
538 .clks = rk3368_isp_clks,
539 .num_clks = ARRAY_SIZE(rk3368_isp_clks),
540 .isp_ver = ISP_V10_1,
541 .clk_rate_tbl = rk3368_isp_clk_rate,
542 .num_clk_rate_tbl = ARRAY_SIZE(rk3368_isp_clk_rate),
543 .irqs = rk3368_isp_irqs,
544 .num_irqs = ARRAY_SIZE(rk3368_isp_irqs),
545 .unite = false,
546 };
547
548 static const struct isp_match_data rk3399_isp_match_data = {
549 .clks = rk3399_isp_clks,
550 .num_clks = ARRAY_SIZE(rk3399_isp_clks),
551 .isp_ver = ISP_V10,
552 .clk_rate_tbl = rk3399_isp_clk_rate,
553 .num_clk_rate_tbl = ARRAY_SIZE(rk3399_isp_clk_rate),
554 .irqs = rk3399_isp_irqs,
555 .num_irqs = ARRAY_SIZE(rk3399_isp_irqs),
556 .unite = false,
557 };
558
559 static const struct isp_match_data rk3568_isp_match_data = {
560 .clks = rk3568_isp_clks,
561 .num_clks = ARRAY_SIZE(rk3568_isp_clks),
562 .isp_ver = ISP_V21,
563 .clk_rate_tbl = rk3568_isp_clk_rate,
564 .num_clk_rate_tbl = ARRAY_SIZE(rk3568_isp_clk_rate),
565 .irqs = rk3568_isp_irqs,
566 .num_irqs = ARRAY_SIZE(rk3568_isp_irqs),
567 .unite = false,
568 };
569
570 static const struct isp_match_data rk3588_isp_match_data = {
571 .clks = rk3588_isp_clks,
572 .num_clks = ARRAY_SIZE(rk3588_isp_clks),
573 .isp_ver = ISP_V30,
574 .clk_rate_tbl = rk3588_isp_clk_rate,
575 .num_clk_rate_tbl = ARRAY_SIZE(rk3588_isp_clk_rate),
576 .irqs = rk3588_isp_irqs,
577 .num_irqs = ARRAY_SIZE(rk3588_isp_irqs),
578 .unite = false,
579 };
580
581 static const struct isp_match_data rk3588_isp_unite_match_data = {
582 .clks = rk3588_isp_unite_clks,
583 .num_clks = ARRAY_SIZE(rk3588_isp_unite_clks),
584 .isp_ver = ISP_V30,
585 .clk_rate_tbl = rk3588_isp_clk_rate,
586 .num_clk_rate_tbl = ARRAY_SIZE(rk3588_isp_clk_rate),
587 .irqs = rk3588_isp_irqs,
588 .num_irqs = ARRAY_SIZE(rk3588_isp_irqs),
589 .unite = true,
590 };
591
592 static const struct of_device_id rkisp_hw_of_match[] = {
593 {
594 .compatible = "rockchip,rk1808-rkisp1",
595 .data = &rk1808_isp_match_data,
596 }, {
597 .compatible = "rockchip,rk3288-rkisp1",
598 .data = &rk3288_isp_match_data,
599 }, {
600 .compatible = "rockchip,rk3326-rkisp1",
601 .data = &rk3326_isp_match_data,
602 }, {
603 .compatible = "rockchip,rk3368-rkisp1",
604 .data = &rk3368_isp_match_data,
605 }, {
606 .compatible = "rockchip,rk3399-rkisp1",
607 .data = &rk3399_isp_match_data,
608 }, {
609 .compatible = "rockchip,rk3568-rkisp",
610 .data = &rk3568_isp_match_data,
611 }, {
612 .compatible = "rockchip,rk3588-rkisp",
613 .data = &rk3588_isp_match_data,
614 }, {
615 .compatible = "rockchip,rk3588-rkisp-unite",
616 .data = &rk3588_isp_unite_match_data,
617 }, {
618 .compatible = "rockchip,rv1126-rkisp",
619 .data = &rv1126_isp_match_data,
620 },
621 {},
622 };
623
is_iommu_enable(struct device * dev)624 static inline bool is_iommu_enable(struct device *dev)
625 {
626 struct device_node *iommu;
627
628 iommu = of_parse_phandle(dev->of_node, "iommus", 0);
629 if (!iommu) {
630 dev_info(dev, "no iommu attached, using non-iommu buffers\n");
631 return false;
632 } else if (!of_device_is_available(iommu)) {
633 dev_info(dev, "iommu is disabled, using non-iommu buffers\n");
634 of_node_put(iommu);
635 return false;
636 }
637 of_node_put(iommu);
638
639 return true;
640 }
641
rkisp_soft_reset(struct rkisp_hw_dev * dev,bool is_secure)642 void rkisp_soft_reset(struct rkisp_hw_dev *dev, bool is_secure)
643 {
644 void __iomem *base = dev->base_addr;
645
646 if (is_secure) {
647 /* if isp working, cru reset isn't secure.
648 * isp soft reset first to protect isp reset.
649 */
650 writel(0xffff, base + CIF_IRCL);
651 if (dev->is_unite)
652 writel(0xffff, dev->base_next_addr + CIF_IRCL);
653 udelay(10);
654 }
655
656 if (dev->reset) {
657 reset_control_assert(dev->reset);
658 udelay(10);
659 reset_control_deassert(dev->reset);
660 udelay(10);
661 }
662
663 /* reset for Dehaze */
664 if (dev->isp_ver == ISP_V20)
665 writel(CIF_ISP_CTRL_ISP_MODE_BAYER_ITU601, base + CIF_ISP_CTRL);
666 writel(0xffff, base + CIF_IRCL);
667 if (dev->is_unite)
668 writel(0xffff, dev->base_next_addr + CIF_IRCL);
669 udelay(10);
670
671 /* refresh iommu after reset */
672 if (dev->is_mmu) {
673 rockchip_iommu_disable(dev->dev);
674 rockchip_iommu_enable(dev->dev);
675 }
676 }
677
isp_config_clk(struct rkisp_hw_dev * dev,int on)678 static void isp_config_clk(struct rkisp_hw_dev *dev, int on)
679 {
680 u32 val = !on ? 0 :
681 CIF_ICCL_ISP_CLK | CIF_ICCL_CP_CLK | CIF_ICCL_MRSZ_CLK |
682 CIF_ICCL_SRSZ_CLK | CIF_ICCL_JPEG_CLK | CIF_ICCL_MI_CLK |
683 CIF_ICCL_IE_CLK | CIF_ICCL_MIPI_CLK | CIF_ICCL_DCROP_CLK;
684
685 if ((dev->isp_ver == ISP_V20 || dev->isp_ver == ISP_V30) && on)
686 val |= ICCL_MPFBC_CLK;
687
688 writel(val, dev->base_addr + CIF_ICCL);
689 if (dev->is_unite)
690 writel(val, dev->base_next_addr + CIF_ICCL);
691
692 if (dev->isp_ver == ISP_V12 || dev->isp_ver == ISP_V13) {
693 val = !on ? 0 :
694 CIF_CLK_CTRL_MI_Y12 | CIF_CLK_CTRL_MI_SP |
695 CIF_CLK_CTRL_MI_RAW0 | CIF_CLK_CTRL_MI_RAW1 |
696 CIF_CLK_CTRL_MI_READ | CIF_CLK_CTRL_MI_RAWRD |
697 CIF_CLK_CTRL_CP | CIF_CLK_CTRL_IE;
698
699 writel(val, dev->base_addr + CIF_VI_ISP_CLK_CTRL_V12);
700 } else if (dev->isp_ver == ISP_V20 ||
701 dev->isp_ver == ISP_V21 ||
702 dev->isp_ver == ISP_V30) {
703 val = !on ? 0 :
704 CLK_CTRL_MI_LDC | CLK_CTRL_MI_MP |
705 CLK_CTRL_MI_JPEG | CLK_CTRL_MI_DP |
706 CLK_CTRL_MI_Y12 | CLK_CTRL_MI_SP |
707 CLK_CTRL_MI_RAW0 | CLK_CTRL_MI_RAW1 |
708 CLK_CTRL_MI_READ | CLK_CTRL_MI_RAWRD |
709 CLK_CTRL_ISP_RAW;
710
711 if ((dev->isp_ver == ISP_V20 || dev->isp_ver == ISP_V30) && on)
712 val |= CLK_CTRL_ISP_3A;
713 writel(val, dev->base_addr + CTRL_VI_ISP_CLK_CTRL);
714 if (dev->is_unite)
715 writel(val, dev->base_next_addr + CTRL_VI_ISP_CLK_CTRL);
716 }
717 }
718
disable_sys_clk(struct rkisp_hw_dev * dev)719 static void disable_sys_clk(struct rkisp_hw_dev *dev)
720 {
721 int i;
722
723 if (dev->isp_ver == ISP_V12 || dev->isp_ver == ISP_V13) {
724 if (dev->mipi_irq >= 0)
725 disable_irq(dev->mipi_irq);
726 }
727
728 isp_config_clk(dev, false);
729
730 for (i = dev->num_clks - 1; i >= 0; i--)
731 if (!IS_ERR(dev->clks[i]))
732 clk_disable_unprepare(dev->clks[i]);
733 }
734
enable_sys_clk(struct rkisp_hw_dev * dev)735 static int enable_sys_clk(struct rkisp_hw_dev *dev)
736 {
737 int i, ret = -EINVAL;
738 unsigned long rate;
739
740 for (i = 0; i < dev->num_clks; i++) {
741 if (!IS_ERR(dev->clks[i])) {
742 ret = clk_prepare_enable(dev->clks[i]);
743 if (ret < 0)
744 goto err;
745 }
746 }
747
748 rate = dev->clk_rate_tbl[0].clk_rate * 1000000UL;
749 rkisp_set_clk_rate(dev->clks[0], rate);
750 if (dev->is_unite)
751 rkisp_set_clk_rate(dev->clks[5], rate);
752 rkisp_soft_reset(dev, false);
753 isp_config_clk(dev, true);
754
755 if (dev->isp_ver == ISP_V12 || dev->isp_ver == ISP_V13) {
756 /* disable csi_rx interrupt */
757 writel(0, dev->base_addr + CIF_ISP_CSI0_CTRL0);
758 writel(0, dev->base_addr + CIF_ISP_CSI0_MASK1);
759 writel(0, dev->base_addr + CIF_ISP_CSI0_MASK2);
760 writel(0, dev->base_addr + CIF_ISP_CSI0_MASK3);
761 }
762
763 return 0;
764 err:
765 for (--i; i >= 0; --i)
766 if (!IS_ERR(dev->clks[i]))
767 clk_disable_unprepare(dev->clks[i]);
768 return ret;
769 }
770
rkisp_hw_probe(struct platform_device * pdev)771 static int rkisp_hw_probe(struct platform_device *pdev)
772 {
773 const struct of_device_id *match;
774 const struct isp_match_data *match_data;
775 struct device_node *node = pdev->dev.of_node;
776 struct device *dev = &pdev->dev;
777 struct rkisp_hw_dev *hw_dev;
778 struct resource *res;
779 int i, ret;
780 bool is_mem_reserved = true;
781
782 match = of_match_node(rkisp_hw_of_match, node);
783 if (IS_ERR(match))
784 return PTR_ERR(match);
785
786 hw_dev = devm_kzalloc(dev, sizeof(*hw_dev), GFP_KERNEL);
787 if (!hw_dev)
788 return -ENOMEM;
789
790 dev_set_drvdata(dev, hw_dev);
791 hw_dev->dev = dev;
792 hw_dev->is_thunderboot = IS_ENABLED(CONFIG_VIDEO_ROCKCHIP_THUNDER_BOOT_ISP);
793 dev_info(dev, "is_thunderboot: %d\n", hw_dev->is_thunderboot);
794 hw_dev->max_in.w = 0;
795 hw_dev->max_in.h = 0;
796 hw_dev->max_in.fps = 0;
797 of_property_read_u32_array(node, "max-input", &hw_dev->max_in.w, 3);
798 dev_info(dev, "max input:%dx%d@%dfps\n",
799 hw_dev->max_in.w, hw_dev->max_in.h, hw_dev->max_in.fps);
800 hw_dev->grf = syscon_regmap_lookup_by_phandle(node, "rockchip,grf");
801 if (IS_ERR(hw_dev->grf))
802 dev_warn(dev, "Missing rockchip,grf property\n");
803
804 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
805 if (!res) {
806 dev_err(dev, "get resource failed\n");
807 ret = -EINVAL;
808 goto err;
809 }
810 hw_dev->base_addr = devm_ioremap_resource(dev, res);
811 if (PTR_ERR(hw_dev->base_addr) == -EBUSY) {
812 resource_size_t offset = res->start;
813 resource_size_t size = resource_size(res);
814
815 hw_dev->base_addr = devm_ioremap(dev, offset, size);
816 }
817 if (IS_ERR(hw_dev->base_addr)) {
818 dev_err(dev, "ioremap failed\n");
819 ret = PTR_ERR(hw_dev->base_addr);
820 goto err;
821 }
822
823 match_data = match->data;
824 hw_dev->base_next_addr = NULL;
825 if (match_data->unite) {
826 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
827 if (!res) {
828 dev_err(dev, "get next resource failed\n");
829 ret = -EINVAL;
830 goto err;
831 }
832 hw_dev->base_next_addr = devm_ioremap_resource(dev, res);
833 if (PTR_ERR(hw_dev->base_next_addr) == -EBUSY) {
834 resource_size_t offset = res->start;
835 resource_size_t size = resource_size(res);
836
837 hw_dev->base_next_addr = devm_ioremap(dev, offset, size);
838 }
839
840 if (IS_ERR(hw_dev->base_next_addr)) {
841 dev_err(dev, "ioremap next failed\n");
842 ret = PTR_ERR(hw_dev->base_next_addr);
843 goto err;
844 }
845 }
846
847 rkisp_monitor = device_property_read_bool(dev, "rockchip,restart-monitor-en");
848 hw_dev->mipi_irq = -1;
849
850 hw_dev->pdev = pdev;
851 hw_dev->match_data = match_data;
852 if (!hw_dev->is_thunderboot)
853 rkisp_register_irq(hw_dev);
854
855 for (i = 0; i < match_data->num_clks; i++) {
856 struct clk *clk = devm_clk_get(dev, match_data->clks[i]);
857
858 if (IS_ERR(clk)) {
859 dev_err(dev, "failed to get %s\n", match_data->clks[i]);
860 ret = PTR_ERR(clk);
861 goto err;
862 }
863 hw_dev->clks[i] = clk;
864 }
865 hw_dev->num_clks = match_data->num_clks;
866 hw_dev->clk_rate_tbl = match_data->clk_rate_tbl;
867 hw_dev->num_clk_rate_tbl = match_data->num_clk_rate_tbl;
868
869 hw_dev->reset = devm_reset_control_array_get(dev, false, false);
870 if (IS_ERR(hw_dev->reset)) {
871 dev_dbg(dev, "failed to get reset\n");
872 hw_dev->reset = NULL;
873 }
874
875 ret = of_property_read_u64(node, "rockchip,iq-feature", &hw_dev->iq_feature);
876 if (!ret)
877 hw_dev->is_feature_on = true;
878 else
879 hw_dev->is_feature_on = false;
880
881 hw_dev->dev_num = 0;
882 hw_dev->cur_dev_id = 0;
883 hw_dev->mipi_dev_id = 0;
884 hw_dev->isp_ver = match_data->isp_ver;
885 hw_dev->is_unite = match_data->unite;
886 mutex_init(&hw_dev->dev_lock);
887 spin_lock_init(&hw_dev->rdbk_lock);
888 atomic_set(&hw_dev->refcnt, 0);
889 spin_lock_init(&hw_dev->buf_lock);
890 INIT_LIST_HEAD(&hw_dev->list);
891 INIT_LIST_HEAD(&hw_dev->rpt_list);
892 hw_dev->buf_init_cnt = 0;
893 hw_dev->is_idle = true;
894 hw_dev->is_single = true;
895 hw_dev->is_mi_update = false;
896 hw_dev->is_dma_contig = true;
897 hw_dev->is_dma_sg_ops = false;
898 hw_dev->is_buf_init = false;
899 hw_dev->is_shutdown = false;
900 hw_dev->is_mmu = is_iommu_enable(dev);
901 ret = of_reserved_mem_device_init(dev);
902 if (ret) {
903 is_mem_reserved = false;
904
905 if (!hw_dev->is_mmu)
906 dev_info(dev, "No reserved memory region. default cma area!\n");
907 else
908 hw_dev->is_dma_contig = false;
909 }
910 if (is_mem_reserved) {
911 /* reserved memory using rdma_sg */
912 hw_dev->mem_ops = &vb2_rdma_sg_memops;
913 hw_dev->is_dma_sg_ops = true;
914 } else if (hw_dev->is_mmu) {
915 hw_dev->mem_ops = &vb2_dma_sg_memops;
916 hw_dev->is_dma_sg_ops = true;
917 } else {
918 hw_dev->mem_ops = &vb2_dma_contig_memops;
919 }
920
921 pm_runtime_enable(dev);
922
923 return 0;
924 err:
925 return ret;
926 }
927
rkisp_hw_remove(struct platform_device * pdev)928 static int rkisp_hw_remove(struct platform_device *pdev)
929 {
930 struct rkisp_hw_dev *hw_dev = platform_get_drvdata(pdev);
931
932 pm_runtime_disable(&pdev->dev);
933 mutex_destroy(&hw_dev->dev_lock);
934 return 0;
935 }
936
rkisp_hw_shutdown(struct platform_device * pdev)937 static void rkisp_hw_shutdown(struct platform_device *pdev)
938 {
939 struct rkisp_hw_dev *hw_dev = platform_get_drvdata(pdev);
940
941 hw_dev->is_shutdown = true;
942 if (pm_runtime_active(&pdev->dev)) {
943 writel(0xffff, hw_dev->base_addr + CIF_IRCL);
944 if (hw_dev->is_unite)
945 writel(0xffff, hw_dev->base_next_addr + CIF_IRCL);
946 }
947 dev_info(&pdev->dev, "%s\n", __func__);
948 }
949
rkisp_runtime_suspend(struct device * dev)950 static int __maybe_unused rkisp_runtime_suspend(struct device *dev)
951 {
952 struct rkisp_hw_dev *hw_dev = dev_get_drvdata(dev);
953
954 disable_sys_clk(hw_dev);
955 return pinctrl_pm_select_sleep_state(dev);
956 }
957
rkisp_runtime_resume(struct device * dev)958 static int __maybe_unused rkisp_runtime_resume(struct device *dev)
959 {
960 struct rkisp_hw_dev *hw_dev = dev_get_drvdata(dev);
961 void __iomem *base = hw_dev->base_addr;
962 int mult = hw_dev->is_unite ? 2 : 1;
963 int ret, i;
964
965 ret = pinctrl_pm_select_default_state(dev);
966 if (ret < 0)
967 return ret;
968
969 enable_sys_clk(hw_dev);
970
971 for (i = 0; i < hw_dev->dev_num; i++) {
972 void *buf = hw_dev->isp[i]->sw_base_addr;
973
974 memset(buf, 0, RKISP_ISP_SW_MAX_SIZE * mult);
975 memcpy_fromio(buf, base, RKISP_ISP_SW_REG_SIZE);
976 if (hw_dev->is_unite) {
977 buf += RKISP_ISP_SW_MAX_SIZE;
978 base = hw_dev->base_next_addr;
979 memcpy_fromio(buf, base, RKISP_ISP_SW_REG_SIZE);
980 }
981 default_sw_reg_flag(hw_dev->isp[i]);
982 }
983 hw_dev->monitor.is_en = rkisp_monitor;
984 return 0;
985 }
986
987 static const struct dev_pm_ops rkisp_hw_pm_ops = {
988 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
989 pm_runtime_force_resume)
990 SET_RUNTIME_PM_OPS(rkisp_runtime_suspend,
991 rkisp_runtime_resume, NULL)
992 };
993
994 static struct platform_driver rkisp_hw_drv = {
995 .driver = {
996 .name = "rkisp_hw",
997 .of_match_table = of_match_ptr(rkisp_hw_of_match),
998 .pm = &rkisp_hw_pm_ops,
999 },
1000 .probe = rkisp_hw_probe,
1001 .remove = rkisp_hw_remove,
1002 .shutdown = rkisp_hw_shutdown,
1003 };
1004
rkisp_hw_drv_init(void)1005 static int __init rkisp_hw_drv_init(void)
1006 {
1007 int ret;
1008
1009 ret = platform_driver_register(&rkisp_hw_drv);
1010 if (!ret)
1011 ret = platform_driver_register(&rkisp_plat_drv);
1012 #if IS_BUILTIN(CONFIG_VIDEO_ROCKCHIP_ISP) && IS_BUILTIN(CONFIG_VIDEO_ROCKCHIP_ISPP)
1013 if (!ret)
1014 ret = rkispp_hw_drv_init();
1015 #endif
1016 return ret;
1017 }
1018
1019 module_init(rkisp_hw_drv_init);
1020