1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2020 Rockchip Electronics Co., Ltd. */
3
4 #include <linux/delay.h>
5 #include <linux/pm_runtime.h>
6 #include <media/v4l2-common.h>
7 #include <media/v4l2-event.h>
8 #include <media/v4l2-fh.h>
9 #include <media/v4l2-ioctl.h>
10 #include <media/v4l2-subdev.h>
11 #include <media/videobuf2-dma-contig.h>
12 #include <media/videobuf2-dma-sg.h>
13 #include "dev.h"
14 #include "regs.h"
15
16 #define CIF_ISP_REQ_BUFS_MIN 0
17
18 static int mi_frame_end(struct rkisp_stream *stream);
19 static void rkisp_buf_queue(struct vb2_buffer *vb);
20
21 static const struct capture_fmt dmatx_fmts[] = {
22 /* raw */
23 {
24 .fourcc = V4L2_PIX_FMT_SRGGB8,
25 .fmt_type = FMT_BAYER,
26 .bpp = { 8 },
27 .mplanes = 1,
28 }, {
29 .fourcc = V4L2_PIX_FMT_SGRBG8,
30 .fmt_type = FMT_BAYER,
31 .bpp = { 8 },
32 .mplanes = 1,
33 }, {
34 .fourcc = V4L2_PIX_FMT_SGBRG8,
35 .fmt_type = FMT_BAYER,
36 .bpp = { 8 },
37 .mplanes = 1,
38 }, {
39 .fourcc = V4L2_PIX_FMT_SBGGR8,
40 .fmt_type = FMT_BAYER,
41 .bpp = { 8 },
42 .mplanes = 1,
43 }, {
44 .fourcc = V4L2_PIX_FMT_GREY,
45 .fmt_type = FMT_BAYER,
46 .bpp = { 8 },
47 .mplanes = 1,
48 }, {
49 .fourcc = V4L2_PIX_FMT_SRGGB10,
50 .fmt_type = FMT_BAYER,
51 .bpp = { 10 },
52 .mplanes = 1,
53 }, {
54 .fourcc = V4L2_PIX_FMT_SGRBG10,
55 .fmt_type = FMT_BAYER,
56 .bpp = { 10 },
57 .mplanes = 1,
58 }, {
59 .fourcc = V4L2_PIX_FMT_SGBRG10,
60 .fmt_type = FMT_BAYER,
61 .bpp = { 10 },
62 .mplanes = 1,
63 }, {
64 .fourcc = V4L2_PIX_FMT_SBGGR10,
65 .fmt_type = FMT_BAYER,
66 .bpp = { 10 },
67 .mplanes = 1,
68 }, {
69 .fourcc = V4L2_PIX_FMT_Y10,
70 .fmt_type = FMT_BAYER,
71 .bpp = { 10 },
72 .mplanes = 1,
73 }, {
74 .fourcc = V4L2_PIX_FMT_SRGGB12,
75 .fmt_type = FMT_BAYER,
76 .bpp = { 12 },
77 .mplanes = 1,
78 }, {
79 .fourcc = V4L2_PIX_FMT_SGRBG12,
80 .fmt_type = FMT_BAYER,
81 .bpp = { 12 },
82 .mplanes = 1,
83 }, {
84 .fourcc = V4L2_PIX_FMT_SGBRG12,
85 .fmt_type = FMT_BAYER,
86 .bpp = { 12 },
87 .mplanes = 1,
88 }, {
89 .fourcc = V4L2_PIX_FMT_SBGGR12,
90 .fmt_type = FMT_BAYER,
91 .bpp = { 12 },
92 .mplanes = 1,
93 }, {
94 .fourcc = V4L2_PIX_FMT_Y12,
95 .fmt_type = FMT_BAYER,
96 .bpp = { 12 },
97 .mplanes = 1,
98 }, {
99 .fourcc = V4L2_PIX_FMT_YUYV,
100 .fmt_type = FMT_YUV,
101 .bpp = { 16 },
102 .mplanes = 1,
103 }, {
104 .fourcc = V4L2_PIX_FMT_YVYU,
105 .fmt_type = FMT_YUV,
106 .bpp = { 16 },
107 .mplanes = 1,
108 }, {
109 .fourcc = V4L2_PIX_FMT_UYVY,
110 .fmt_type = FMT_YUV,
111 .bpp = { 16 },
112 .mplanes = 1,
113 }, {
114 .fourcc = V4L2_PIX_FMT_VYUY,
115 .fmt_type = FMT_YUV,
116 .bpp = { 16 },
117 .mplanes = 1,
118 }, {
119 .fourcc = V4l2_PIX_FMT_EBD8,
120 .fmt_type = FMT_EBD,
121 .bpp = { 8 },
122 .mplanes = 1,
123 }, {
124 .fourcc = V4l2_PIX_FMT_SPD16,
125 .fmt_type = FMT_SPD,
126 .bpp = { 16 },
127 .mplanes = 1,
128 }
129 };
130
131 static struct stream_config rkisp2_dmatx0_stream_config = {
132 .fmts = dmatx_fmts,
133 .fmt_size = ARRAY_SIZE(dmatx_fmts),
134 .frame_end_id = MI_RAW0_WR_FRAME,
135 .mi = {
136 .y_size_init = MI_RAW0_WR_SIZE,
137 .y_base_ad_init = MI_RAW0_WR_BASE,
138 .y_base_ad_shd = MI_RAW0_WR_BASE_SHD,
139 .length = MI_RAW0_WR_LENGTH,
140 },
141 .dma = {
142 .ctrl = CSI2RX_RAW0_WR_CTRL,
143 .pic_size = CSI2RX_RAW0_WR_PIC_SIZE,
144 .pic_offs = CSI2RX_RAW0_WR_PIC_OFF,
145 },
146 };
147
148 static struct stream_config rkisp2_dmatx1_stream_config = {
149 .fmts = dmatx_fmts,
150 .fmt_size = ARRAY_SIZE(dmatx_fmts),
151 .frame_end_id = MI_RAW1_WR_FRAME,
152 .mi = {
153 .y_size_init = MI_RAW1_WR_SIZE,
154 .y_base_ad_init = MI_RAW1_WR_BASE,
155 .y_base_ad_shd = MI_RAW1_WR_BASE_SHD,
156 .length = MI_RAW1_WR_LENGTH,
157 },
158 .dma = {
159 .ctrl = CSI2RX_RAW1_WR_CTRL,
160 .pic_size = CSI2RX_RAW1_WR_PIC_SIZE,
161 .pic_offs = CSI2RX_RAW1_WR_PIC_OFF,
162 },
163 };
164
165 static struct stream_config rkisp2_dmatx2_stream_config = {
166 .fmts = dmatx_fmts,
167 .fmt_size = ARRAY_SIZE(dmatx_fmts),
168 .frame_end_id = MI_RAW2_WR_FRAME,
169 .mi = {
170 .y_size_init = MI_RAW2_WR_SIZE,
171 .y_base_ad_init = MI_RAW2_WR_BASE,
172 .y_base_ad_shd = MI_RAW2_WR_BASE_SHD,
173 .length = MI_RAW2_WR_LENGTH,
174 },
175 .dma = {
176 .ctrl = CSI2RX_RAW2_WR_CTRL,
177 .pic_size = CSI2RX_RAW2_WR_PIC_SIZE,
178 .pic_offs = CSI2RX_RAW2_WR_PIC_OFF,
179 },
180 };
181
182 static struct stream_config rkisp2_dmatx3_stream_config = {
183 .fmts = dmatx_fmts,
184 .fmt_size = ARRAY_SIZE(dmatx_fmts),
185 .frame_end_id = MI_RAW3_WR_FRAME,
186 .mi = {
187 .y_size_init = MI_RAW3_WR_SIZE,
188 .y_base_ad_init = MI_RAW3_WR_BASE,
189 .y_base_ad_shd = MI_RAW3_WR_BASE_SHD,
190 .length = MI_RAW3_WR_LENGTH,
191 },
192 .dma = {
193 .ctrl = CSI2RX_RAW3_WR_CTRL,
194 .pic_size = CSI2RX_RAW3_WR_PIC_SIZE,
195 .pic_offs = CSI2RX_RAW3_WR_PIC_OFF,
196 },
197 };
198
is_rdbk_stream(struct rkisp_stream * stream)199 static bool is_rdbk_stream(struct rkisp_stream *stream)
200 {
201 struct rkisp_device *dev = stream->ispdev;
202 bool en = false;
203
204 if ((dev->hdr.op_mode == HDR_RDBK_FRAME1 &&
205 stream->id == RKISP_STREAM_DMATX2) ||
206 (dev->hdr.op_mode == HDR_RDBK_FRAME2 &&
207 (stream->id == RKISP_STREAM_DMATX2 ||
208 stream->id == RKISP_STREAM_DMATX0)) ||
209 (dev->hdr.op_mode == HDR_RDBK_FRAME3 &&
210 (stream->id == RKISP_STREAM_DMATX2 ||
211 stream->id == RKISP_STREAM_DMATX1 ||
212 stream->id == RKISP_STREAM_DMATX0)))
213 en = true;
214 return en;
215 }
216
217 /* configure dual-crop unit */
rkisp_stream_config_dcrop(struct rkisp_stream * stream,bool async)218 static int rkisp_stream_config_dcrop(struct rkisp_stream *stream, bool async)
219 {
220 struct rkisp_device *dev = stream->ispdev;
221 struct v4l2_rect *dcrop = &stream->dcrop;
222 struct v4l2_rect *input_win;
223 u32 src_w, src_h;
224
225 /* dual-crop unit get data from isp */
226 input_win = rkisp_get_isp_sd_win(&dev->isp_sdev);
227 src_w = input_win->width;
228 src_h = input_win->height;
229
230 if (dev->isp_ver == ISP_V20 &&
231 dev->rd_mode == HDR_RDBK_FRAME1 &&
232 dev->isp_sdev.in_fmt.fmt_type == FMT_BAYER &&
233 dev->isp_sdev.out_fmt.fmt_type == FMT_YUV)
234 src_h += RKMODULE_EXTEND_LINE;
235
236 if (dcrop->width == src_w &&
237 dcrop->height == src_h &&
238 dcrop->left == 0 && dcrop->top == 0) {
239 rkisp_disable_dcrop(stream, async);
240 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
241 "stream %d crop disabled\n", stream->id);
242 return 0;
243 }
244
245 rkisp_config_dcrop(stream, dcrop, async);
246
247 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
248 "stream %d crop: %dx%d -> %dx%d\n", stream->id,
249 src_w, src_h, dcrop->width, dcrop->height);
250
251 return 0;
252 }
253
254 /* configure scale unit */
rkisp_stream_config_rsz(struct rkisp_stream * stream,bool async)255 static int rkisp_stream_config_rsz(struct rkisp_stream *stream, bool async)
256 {
257 struct rkisp_device *dev = stream->ispdev;
258 struct v4l2_pix_format_mplane output_fmt = stream->out_fmt;
259 struct capture_fmt *output_isp_fmt = &stream->out_isp_fmt;
260 struct ispsd_out_fmt *input_isp_fmt =
261 rkisp_get_ispsd_out_fmt(&dev->isp_sdev);
262 struct v4l2_rect in_y, in_c, out_y, out_c;
263 u32 xsubs_in = 1, ysubs_in = 1;
264 u32 xsubs_out = 1, ysubs_out = 1;
265
266 if (input_isp_fmt->fmt_type == FMT_BAYER ||
267 (dev->br_dev.en && stream->id != RKISP_STREAM_SP))
268 goto disable;
269
270 /* set input and output sizes for scale calculation */
271 in_y.width = stream->dcrop.width;
272 in_y.height = stream->dcrop.height;
273 out_y.width = output_fmt.width;
274 out_y.height = output_fmt.height;
275
276 /* The size of Cb,Cr are related to the format */
277 if (rkisp_mbus_code_xysubs(input_isp_fmt->mbus_code, &xsubs_in, &ysubs_in)) {
278 v4l2_err(&dev->v4l2_dev, "Not xsubs/ysubs found\n");
279 return -EINVAL;
280 }
281 in_c.width = in_y.width / xsubs_in;
282 in_c.height = in_y.height / ysubs_in;
283
284 if (output_isp_fmt->fmt_type == FMT_YUV || output_isp_fmt->fmt_type == FMT_FBCGAIN) {
285 rkisp_fcc_xysubs(output_isp_fmt->fourcc, &xsubs_out, &ysubs_out);
286 if (stream->id == RKISP_STREAM_SP &&
287 stream->out_isp_fmt.fmt_type == FMT_FBCGAIN &&
288 !(dev->br_dev.work_mode & ISP_ISPP_422)) {
289 xsubs_out = 2;
290 ysubs_out = 1;
291 }
292
293 out_c.width = out_y.width / xsubs_out;
294 out_c.height = out_y.height / ysubs_out;
295 } else {
296 out_c.width = out_y.width / xsubs_in;
297 out_c.height = out_y.height / ysubs_in;
298 }
299
300 if (in_c.width == out_c.width && in_c.height == out_c.height)
301 goto disable;
302
303 /* set RSZ input and output */
304 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
305 "stream %d rsz/scale: %dx%d -> %dx%d\n",
306 stream->id, stream->dcrop.width, stream->dcrop.height,
307 output_fmt.width, output_fmt.height);
308 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
309 "chroma scaling %dx%d -> %dx%d\n",
310 in_c.width, in_c.height, out_c.width, out_c.height);
311
312 /* calculate and set scale */
313 rkisp_config_rsz(stream, &in_y, &in_c, &out_y, &out_c, async);
314
315 if (rkisp_debug)
316 rkisp_dump_rsz_regs(stream);
317
318 return 0;
319
320 disable:
321 rkisp_disable_rsz(stream, async);
322
323 return 0;
324 }
325
326 /***************************** stream operations*******************************/
327
328 /*
329 * memory base addresses should be with respect
330 * to the burst alignment restriction for AXI.
331 */
calc_burst_len(struct rkisp_stream * stream)332 static u32 calc_burst_len(struct rkisp_stream *stream)
333 {
334 struct rkisp_device *dev = stream->ispdev;
335 u32 y_size = stream->out_fmt.plane_fmt[0].bytesperline *
336 stream->out_fmt.height;
337 u32 cb_size = stream->out_fmt.plane_fmt[1].sizeimage;
338 u32 cr_size = stream->out_fmt.plane_fmt[2].sizeimage;
339 u32 cb_offs, cr_offs;
340 u32 bus = 16, burst;
341 int i;
342
343 if (stream->out_isp_fmt.fmt_type == FMT_FBCGAIN)
344 y_size = ALIGN(y_size, 64);
345 /* y/c base addr: burstN * bus alignment */
346 cb_offs = y_size;
347 cr_offs = cr_size ? (cb_size + cb_offs) : 0;
348
349 if (!(cb_offs % (bus * 16)) &&
350 !(cr_offs % (bus * 16)))
351 burst = CIF_MI_CTRL_BURST_LEN_LUM_16 |
352 CIF_MI_CTRL_BURST_LEN_CHROM_16;
353 else if (!(cb_offs % (bus * 8)) &&
354 !(cr_offs % (bus * 8)))
355 burst = CIF_MI_CTRL_BURST_LEN_LUM_8 |
356 CIF_MI_CTRL_BURST_LEN_CHROM_8;
357 else
358 burst = CIF_MI_CTRL_BURST_LEN_LUM_4 |
359 CIF_MI_CTRL_BURST_LEN_CHROM_4;
360
361 if (cb_offs % (bus * 4) ||
362 cr_offs % (bus * 4))
363 v4l2_warn(&dev->v4l2_dev,
364 "%dx%d fmt:0x%x not support, should be %d aligned\n",
365 stream->out_fmt.width,
366 stream->out_fmt.height,
367 stream->out_fmt.pixelformat,
368 (cr_offs == 0) ? bus * 4 : bus * 16);
369
370 stream->burst = burst;
371 for (i = 0; i < RKISP_MAX_STREAM; i++)
372 if (burst > dev->cap_dev.stream[i].burst)
373 burst = dev->cap_dev.stream[i].burst;
374
375 if (stream->interlaced) {
376 if (!stream->out_fmt.width % (bus * 16))
377 stream->burst = CIF_MI_CTRL_BURST_LEN_LUM_16 |
378 CIF_MI_CTRL_BURST_LEN_CHROM_16;
379 else if (!stream->out_fmt.width % (bus * 8))
380 stream->burst = CIF_MI_CTRL_BURST_LEN_LUM_8 |
381 CIF_MI_CTRL_BURST_LEN_CHROM_8;
382 else
383 stream->burst = CIF_MI_CTRL_BURST_LEN_LUM_4 |
384 CIF_MI_CTRL_BURST_LEN_CHROM_4;
385 if (stream->out_fmt.width % (bus * 4))
386 v4l2_warn(&dev->v4l2_dev,
387 "interlaced: width should be %d aligned\n",
388 bus * 4);
389 burst = min(stream->burst, burst);
390 stream->burst = burst;
391 }
392
393 return burst;
394 }
395
396 /*
397 * configure memory interface for mainpath
398 * This should only be called when stream-on
399 */
mp_config_mi(struct rkisp_stream * stream)400 static int mp_config_mi(struct rkisp_stream *stream)
401 {
402 void __iomem *base = stream->ispdev->base_addr;
403
404 /*
405 * NOTE: plane_fmt[0].sizeimage is total size of all planes for single
406 * memory plane formats, so calculate the size explicitly.
407 */
408 mi_set_y_size(stream, stream->out_fmt.plane_fmt[0].bytesperline *
409 stream->out_fmt.height);
410 mi_set_cb_size(stream, stream->out_fmt.plane_fmt[1].sizeimage);
411 mi_set_cr_size(stream, stream->out_fmt.plane_fmt[2].sizeimage);
412 mi_frame_end_int_enable(stream);
413 if (stream->out_isp_fmt.uv_swap)
414 mp_set_uv_swap(base);
415
416 config_mi_ctrl(stream, calc_burst_len(stream));
417 mp_mi_ctrl_set_format(base, stream->out_isp_fmt.write_format);
418 mp_mi_ctrl_autoupdate_en(base);
419
420 /* set up first buffer */
421 mi_frame_end(stream);
422 return 0;
423 }
424
mbus_code_sp_in_fmt(u32 in_mbus_code,u32 out_fourcc,u32 * format)425 static int mbus_code_sp_in_fmt(u32 in_mbus_code, u32 out_fourcc, u32 *format)
426 {
427 switch (in_mbus_code) {
428 case MEDIA_BUS_FMT_YUYV8_2X8:
429 *format = MI_CTRL_SP_INPUT_YUV422;
430 break;
431 default:
432 return -EINVAL;
433 }
434
435 /*
436 * Only SP can support output format of YCbCr4:0:0,
437 * and the input format of SP must be YCbCr4:0:0
438 * when outputting YCbCr4:0:0.
439 * The output format of isp is YCbCr4:2:2,
440 * so the CbCr data is discarded here.
441 */
442 if (out_fourcc == V4L2_PIX_FMT_GREY)
443 *format = MI_CTRL_SP_INPUT_YUV400;
444
445 return 0;
446 }
447
448 /*
449 * configure memory interface for selfpath
450 * This should only be called when stream-on
451 */
sp_config_mi(struct rkisp_stream * stream)452 static int sp_config_mi(struct rkisp_stream *stream)
453 {
454 void __iomem *base = stream->ispdev->base_addr;
455 struct rkisp_device *dev = stream->ispdev;
456 struct capture_fmt *output_isp_fmt = &stream->out_isp_fmt;
457 struct ispsd_out_fmt *input_isp_fmt =
458 rkisp_get_ispsd_out_fmt(&dev->isp_sdev);
459 u32 sp_in_fmt, output_format;
460
461 if (mbus_code_sp_in_fmt(input_isp_fmt->mbus_code,
462 output_isp_fmt->fourcc, &sp_in_fmt)) {
463 v4l2_err(&dev->v4l2_dev, "Can't find the input format\n");
464 return -EINVAL;
465 }
466
467 /*
468 * NOTE: plane_fmt[0].sizeimage is total size of all planes for single
469 * memory plane formats, so calculate the size explicitly.
470 */
471 mi_set_y_size(stream, stream->out_fmt.plane_fmt[0].bytesperline *
472 stream->out_fmt.height);
473 mi_set_cb_size(stream, stream->out_fmt.plane_fmt[1].sizeimage);
474 mi_set_cr_size(stream, stream->out_fmt.plane_fmt[2].sizeimage);
475
476 sp_set_y_width(base, stream->out_fmt.width);
477 if (stream->interlaced) {
478 stream->u.sp.vir_offs =
479 stream->out_fmt.plane_fmt[0].bytesperline;
480 sp_set_y_height(base, stream->out_fmt.height / 2);
481 sp_set_y_line_length(base, stream->u.sp.y_stride * 2);
482 } else {
483 sp_set_y_height(base, stream->out_fmt.height);
484 sp_set_y_line_length(base, stream->u.sp.y_stride);
485 }
486
487 mi_frame_end_int_enable(stream);
488 if (output_isp_fmt->uv_swap)
489 sp_set_uv_swap(base);
490
491 config_mi_ctrl(stream, calc_burst_len(stream));
492 output_format = output_isp_fmt->output_format;
493 if (stream->id == RKISP_STREAM_SP &&
494 stream->out_isp_fmt.fmt_type == FMT_FBCGAIN &&
495 !(dev->br_dev.work_mode & ISP_ISPP_422)) {
496 sp_in_fmt = MI_CTRL_SP_INPUT_YUV422;
497 output_format = MI_CTRL_SP_OUTPUT_YUV422;
498 sp_set_y_width(base, 0);
499 sp_set_y_line_length(base, ALIGN(stream->out_fmt.width, 16));
500 }
501
502 sp_mi_ctrl_set_format(base, stream->out_isp_fmt.write_format |
503 sp_in_fmt | output_format);
504
505 sp_mi_ctrl_autoupdate_en(base);
506
507 /* set up first buffer */
508 mi_frame_end(stream);
509 return 0;
510 }
511
dmatx3_config_mi(struct rkisp_stream * stream)512 static int dmatx3_config_mi(struct rkisp_stream *stream)
513 {
514 void __iomem *base = stream->ispdev->base_addr;
515 struct rkisp_device *dev = stream->ispdev;
516 struct rkisp_csi_device *csi = &dev->csi_dev;
517 u32 in_size;
518 u8 vc;
519
520 if (!csi->sink[CSI_SRC_CH4 - 1].linked ||
521 stream->streaming)
522 return -EBUSY;
523
524 if (!dev->active_sensor ||
525 (dev->active_sensor &&
526 dev->active_sensor->mbus.type != V4L2_MBUS_CSI2_DPHY)) {
527 v4l2_err(&dev->v4l2_dev,
528 "only mipi sensor support rawwr3\n");
529 return -EINVAL;
530 }
531 atomic_set(&stream->sequence, 0);
532 in_size = stream->out_fmt.plane_fmt[0].sizeimage;
533 raw_wr_set_pic_size(stream,
534 stream->out_fmt.width,
535 stream->out_fmt.height);
536 raw_wr_set_pic_offs(stream, 0);
537
538 vc = csi->sink[CSI_SRC_CH4 - 1].index;
539 raw_wr_ctrl(stream,
540 SW_CSI_RAW_WR_CH_EN(vc) |
541 stream->memory |
542 SW_CSI_RAW_WR_EN_ORG);
543 mi_set_y_size(stream, in_size);
544 mi_frame_end(stream);
545 mi_frame_end_int_enable(stream);
546 mi_wr_ctrl2(base, SW_RAW3_WR_AUTOUPD);
547 mi_raw_length(stream);
548 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
549 "rawwr3 %dx%d ctrl:0x%x\n",
550 stream->out_fmt.width,
551 stream->out_fmt.height,
552 readl(base + CSI2RX_RAW3_WR_CTRL));
553 return 0;
554 }
555
dmatx2_config_mi(struct rkisp_stream * stream)556 static int dmatx2_config_mi(struct rkisp_stream *stream)
557 {
558 void __iomem *base = stream->ispdev->base_addr;
559 struct rkisp_device *dev = stream->ispdev;
560 struct rkisp_csi_device *csi = &dev->csi_dev;
561 u32 val, in_size;
562 u8 vc;
563
564 if (!csi->sink[CSI_SRC_CH3 - 1].linked ||
565 stream->streaming)
566 return -EBUSY;
567
568 if (!dev->active_sensor ||
569 (dev->active_sensor &&
570 dev->active_sensor->mbus.type != V4L2_MBUS_CSI2_DPHY)) {
571 v4l2_err(&dev->v4l2_dev,
572 "only mipi sensor support rawwr2 path\n");
573 return -EINVAL;
574 }
575
576 if (!stream->u.dmatx.is_config) {
577 atomic_set(&stream->sequence, 0);
578 in_size = stream->out_fmt.plane_fmt[0].sizeimage;
579 raw_wr_set_pic_size(stream,
580 stream->out_fmt.width,
581 stream->out_fmt.height);
582 raw_wr_set_pic_offs(stream, 0);
583 vc = csi->sink[CSI_SRC_CH3 - 1].index;
584 val = SW_CSI_RAW_WR_CH_EN(vc);
585 val |= stream->memory;
586 if (dev->hdr.op_mode != HDR_NORMAL)
587 val |= SW_CSI_RAW_WR_EN_ORG;
588 raw_wr_ctrl(stream, val);
589 mi_set_y_size(stream, in_size);
590 mi_frame_end(stream);
591 mi_frame_end_int_enable(stream);
592 mi_wr_ctrl2(base, SW_RAW2_WR_AUTOUPD);
593 mi_raw_length(stream);
594 stream->u.dmatx.is_config = true;
595 }
596 return 0;
597 }
598
dmatx1_config_mi(struct rkisp_stream * stream)599 static int dmatx1_config_mi(struct rkisp_stream *stream)
600 {
601 void __iomem *base = stream->ispdev->base_addr;
602 struct rkisp_device *dev = stream->ispdev;
603 struct rkisp_csi_device *csi = &dev->csi_dev;
604 u32 val, in_size;
605 u8 vc;
606
607 if (!csi->sink[CSI_SRC_CH2 - 1].linked ||
608 stream->streaming)
609 return -EBUSY;
610
611 if (!dev->active_sensor ||
612 (dev->active_sensor &&
613 dev->active_sensor->mbus.type != V4L2_MBUS_CSI2_DPHY)) {
614 if (stream->id == RKISP_STREAM_DMATX1)
615 v4l2_err(&dev->v4l2_dev,
616 "only mipi sensor support dmatx1 path\n");
617 return -EINVAL;
618 }
619
620 if (!stream->u.dmatx.is_config) {
621 atomic_set(&stream->sequence, 0);
622 in_size = stream->out_fmt.plane_fmt[0].sizeimage;
623 raw_wr_set_pic_size(stream,
624 stream->out_fmt.width,
625 stream->out_fmt.height);
626 raw_wr_set_pic_offs(stream, 0);
627 vc = csi->sink[CSI_SRC_CH2 - 1].index;
628 val = SW_CSI_RAW_WR_CH_EN(vc);
629 val |= stream->memory;
630 if (dev->hdr.op_mode != HDR_NORMAL)
631 val |= SW_CSI_RAW_WR_EN_ORG;
632 raw_wr_ctrl(stream, val);
633 mi_set_y_size(stream, in_size);
634 mi_frame_end(stream);
635 mi_frame_end_int_enable(stream);
636 mi_wr_ctrl2(base, SW_RAW1_WR_AUTOUPD);
637 mi_raw_length(stream);
638 stream->u.dmatx.is_config = true;
639 }
640 return 0;
641 }
642
dmatx0_config_mi(struct rkisp_stream * stream)643 static int dmatx0_config_mi(struct rkisp_stream *stream)
644 {
645 void __iomem *base = stream->ispdev->base_addr;
646 struct rkisp_device *dev = stream->ispdev;
647 struct rkisp_csi_device *csi = &dev->csi_dev;
648 struct rkisp_stream *dmatx =
649 &dev->cap_dev.stream[RKISP_STREAM_DMATX0];
650 u32 val, in_size;
651 u8 vc;
652
653 if (!csi->sink[CSI_SRC_CH1 - 1].linked ||
654 dmatx->streaming)
655 return -EBUSY;
656
657 if (!dev->active_sensor ||
658 (dev->active_sensor &&
659 dev->active_sensor->mbus.type != V4L2_MBUS_CSI2_DPHY)) {
660 if (stream->id == RKISP_STREAM_DMATX0)
661 v4l2_err(&dev->v4l2_dev,
662 "only mipi sensor support rawwr0 path\n");
663 return -EINVAL;
664 }
665
666 in_size = dmatx->out_fmt.plane_fmt[0].sizeimage;
667 if (!stream->u.dmatx.is_config) {
668 if (dmatx->u.dmatx.is_config)
669 return 0;
670 atomic_set(&dmatx->sequence, 0);
671 raw_wr_set_pic_size(dmatx,
672 dmatx->out_fmt.width,
673 dmatx->out_fmt.height);
674 raw_wr_set_pic_offs(dmatx, 0);
675 vc = csi->sink[CSI_SRC_CH1 - 1].index;
676 val = SW_CSI_RAW_WR_CH_EN(vc);
677 val |= stream->memory;
678 if (dev->hdr.op_mode != HDR_NORMAL)
679 val |= SW_CSI_RAW_WR_EN_ORG;
680 raw_wr_ctrl(dmatx, val);
681 mi_set_y_size(dmatx, in_size);
682 mi_frame_end(dmatx);
683 mi_frame_end_int_enable(dmatx);
684 mi_wr_ctrl2(base, SW_RAW0_WR_AUTOUPD);
685 mi_raw_length(stream);
686 dmatx->u.dmatx.is_config = true;
687 }
688
689 return 0;
690 }
691
mp_enable_mi(struct rkisp_stream * stream)692 static void mp_enable_mi(struct rkisp_stream *stream)
693 {
694 void __iomem *base = stream->ispdev->base_addr;
695 struct capture_fmt *isp_fmt = &stream->out_isp_fmt;
696
697 mi_ctrl_mp_disable(base);
698 if (isp_fmt->fmt_type == FMT_BAYER)
699 mi_ctrl_mpraw_enable(base);
700 else if (isp_fmt->fmt_type == FMT_YUV)
701 mi_ctrl_mpyuv_enable(base);
702 }
703
sp_enable_mi(struct rkisp_stream * stream)704 static void sp_enable_mi(struct rkisp_stream *stream)
705 {
706 void __iomem *base = stream->ispdev->base_addr;
707
708 mi_ctrl_spyuv_enable(base);
709 }
710
dmatx_enable_mi(struct rkisp_stream * stream)711 static void dmatx_enable_mi(struct rkisp_stream *stream)
712 {
713 raw_wr_enable(stream);
714 }
715
mp_disable_mi(struct rkisp_stream * stream)716 static void mp_disable_mi(struct rkisp_stream *stream)
717 {
718 struct rkisp_device *dev = stream->ispdev;
719 void __iomem *base = dev->base_addr;
720
721 mi_ctrl_mp_disable(base);
722 hdr_stop_dmatx(dev);
723 }
724
sp_disable_mi(struct rkisp_stream * stream)725 static void sp_disable_mi(struct rkisp_stream *stream)
726 {
727 void __iomem *base = stream->ispdev->base_addr;
728
729 mi_ctrl_spyuv_disable(base);
730 }
731
update_dmatx_v2(struct rkisp_stream * stream)732 static void update_dmatx_v2(struct rkisp_stream *stream)
733 {
734 struct rkisp_device *dev = stream->ispdev;
735 void __iomem *base = dev->base_addr;
736 struct rkisp_dummy_buffer *buf = NULL;
737 u8 index;
738
739 if (stream->next_buf) {
740 mi_set_y_addr(stream,
741 stream->next_buf->buff_addr[RKISP_PLANE_Y]);
742 } else {
743 if (stream->id == RKISP_STREAM_DMATX0)
744 index = dev->hdr.index[HDR_DMA0];
745 else if (stream->id == RKISP_STREAM_DMATX1)
746 index = dev->hdr.index[HDR_DMA1];
747 else if (stream->id == RKISP_STREAM_DMATX2)
748 index = dev->hdr.index[HDR_DMA2];
749
750 if ((stream->id == RKISP_STREAM_DMATX0 ||
751 stream->id == RKISP_STREAM_DMATX1 ||
752 stream->id == RKISP_STREAM_DMATX2)) {
753 buf = hdr_dqbuf(&dev->hdr.q_tx[index]);
754 if (IS_HDR_RDBK(dev->hdr.op_mode) &&
755 !dev->dmarx_dev.trigger)
756 hdr_qbuf(&dev->hdr.q_rx[index], buf);
757 else
758 hdr_qbuf(&dev->hdr.q_tx[index], buf);
759 }
760 if (!buf && dev->hw_dev->dummy_buf.mem_priv) {
761 buf = &dev->hw_dev->dummy_buf;
762 stream->dbg.frameloss++;
763 }
764 if (buf)
765 mi_set_y_addr(stream, buf->dma_addr);
766 }
767 v4l2_dbg(2, rkisp_debug, &dev->v4l2_dev,
768 "%s stream:%d Y:0x%x SHD:0x%x\n",
769 __func__, stream->id,
770 readl(base + stream->config->mi.y_base_ad_init),
771 readl(base + stream->config->mi.y_base_ad_shd));
772 }
773
774 /* Update buffer info to memory interface, it's called in interrupt */
update_mi(struct rkisp_stream * stream)775 static void update_mi(struct rkisp_stream *stream)
776 {
777 struct rkisp_dummy_buffer *dummy_buf = &stream->ispdev->hw_dev->dummy_buf;
778 void __iomem *base = stream->ispdev->base_addr;
779
780 /* The dummy space allocated by dma_alloc_coherent is used, we can
781 * throw data to it if there is no available buffer.
782 */
783 if (stream->next_buf) {
784 mi_set_y_addr(stream,
785 stream->next_buf->buff_addr[RKISP_PLANE_Y]);
786 mi_set_cb_addr(stream,
787 stream->next_buf->buff_addr[RKISP_PLANE_CB]);
788 mi_set_cr_addr(stream,
789 stream->next_buf->buff_addr[RKISP_PLANE_CR]);
790 } else if (dummy_buf->mem_priv) {
791 mi_set_y_addr(stream, dummy_buf->dma_addr);
792 mi_set_cb_addr(stream, dummy_buf->dma_addr);
793 mi_set_cr_addr(stream, dummy_buf->dma_addr);
794 stream->dbg.frameloss++;
795 }
796
797 mi_set_y_offset(stream, 0);
798 mi_set_cb_offset(stream, 0);
799 mi_set_cr_offset(stream, 0);
800 v4l2_dbg(2, rkisp_debug, &stream->ispdev->v4l2_dev,
801 "%s stream:%d Y:0x%x CB:0x%x CR:0x%x\n",
802 __func__, stream->id,
803 readl(base + stream->config->mi.y_base_ad_init),
804 readl(base + stream->config->mi.cb_base_ad_init),
805 readl(base + stream->config->mi.cr_base_ad_init));
806 }
807
mp_stop_mi(struct rkisp_stream * stream)808 static void mp_stop_mi(struct rkisp_stream *stream)
809 {
810 if (!stream->streaming)
811 return;
812 mi_frame_end_int_clear(stream);
813 stream->ops->disable_mi(stream);
814 }
815
sp_stop_mi(struct rkisp_stream * stream)816 static void sp_stop_mi(struct rkisp_stream *stream)
817 {
818 if (!stream->streaming)
819 return;
820 mi_frame_end_int_clear(stream);
821 stream->ops->disable_mi(stream);
822 }
823
dmatx_stop_mi(struct rkisp_stream * stream)824 static void dmatx_stop_mi(struct rkisp_stream *stream)
825 {
826 struct rkisp_hw_dev *hw = stream->ispdev->hw_dev;
827
828 raw_wr_disable(stream);
829 if (hw->dummy_buf.mem_priv)
830 mi_set_y_addr(stream, hw->dummy_buf.dma_addr);
831 stream->u.dmatx.is_config = false;
832 }
833
834 static struct streams_ops rkisp_mp_streams_ops = {
835 .config_mi = mp_config_mi,
836 .enable_mi = mp_enable_mi,
837 .disable_mi = mp_disable_mi,
838 .stop_mi = mp_stop_mi,
839 .set_data_path = stream_data_path,
840 .is_stream_stopped = mp_is_stream_stopped,
841 .update_mi = update_mi,
842 .frame_end = mi_frame_end,
843 };
844
845 static struct streams_ops rkisp_sp_streams_ops = {
846 .config_mi = sp_config_mi,
847 .enable_mi = sp_enable_mi,
848 .disable_mi = sp_disable_mi,
849 .stop_mi = sp_stop_mi,
850 .set_data_path = stream_data_path,
851 .is_stream_stopped = sp_is_stream_stopped,
852 .update_mi = update_mi,
853 .frame_end = mi_frame_end,
854 };
855
856 static struct streams_ops rkisp2_dmatx0_streams_ops = {
857 .config_mi = dmatx0_config_mi,
858 .enable_mi = dmatx_enable_mi,
859 .stop_mi = dmatx_stop_mi,
860 .is_stream_stopped = dmatx0_is_stream_stopped,
861 .update_mi = update_dmatx_v2,
862 .frame_end = mi_frame_end,
863 };
864
865 static struct streams_ops rkisp2_dmatx1_streams_ops = {
866 .config_mi = dmatx1_config_mi,
867 .enable_mi = dmatx_enable_mi,
868 .stop_mi = dmatx_stop_mi,
869 .is_stream_stopped = dmatx1_is_stream_stopped,
870 .update_mi = update_dmatx_v2,
871 .frame_end = mi_frame_end,
872 };
873
874 static struct streams_ops rkisp2_dmatx2_streams_ops = {
875 .config_mi = dmatx2_config_mi,
876 .enable_mi = dmatx_enable_mi,
877 .stop_mi = dmatx_stop_mi,
878 .is_stream_stopped = dmatx2_is_stream_stopped,
879 .update_mi = update_dmatx_v2,
880 .frame_end = mi_frame_end,
881 };
882
883 static struct streams_ops rkisp2_dmatx3_streams_ops = {
884 .config_mi = dmatx3_config_mi,
885 .enable_mi = dmatx_enable_mi,
886 .stop_mi = dmatx_stop_mi,
887 .is_stream_stopped = dmatx3_is_stream_stopped,
888 .update_mi = update_dmatx_v2,
889 .frame_end = mi_frame_end,
890 };
891
rdbk_frame_end(struct rkisp_stream * stream)892 static void rdbk_frame_end(struct rkisp_stream *stream)
893 {
894 struct rkisp_device *isp_dev = stream->ispdev;
895 struct rkisp_capture_device *cap = &isp_dev->cap_dev;
896 struct rkisp_sensor_info *sensor = isp_dev->active_sensor;
897 u32 denominator = sensor->fi.interval.denominator;
898 u32 numerator = sensor->fi.interval.numerator;
899 u64 l_ts, m_ts, s_ts;
900 int ret, fps = -1, time = 30000000;
901
902 if (stream->id != RKISP_STREAM_DMATX2)
903 return;
904
905 if (denominator && numerator)
906 time = numerator * 1000 / denominator * 1000 * 1000;
907
908 if (isp_dev->hdr.op_mode == HDR_RDBK_FRAME3) {
909 if (cap->rdbk_buf[RDBK_L] && cap->rdbk_buf[RDBK_M] &&
910 cap->rdbk_buf[RDBK_S]) {
911 l_ts = cap->rdbk_buf[RDBK_L]->vb.vb2_buf.timestamp;
912 m_ts = cap->rdbk_buf[RDBK_M]->vb.vb2_buf.timestamp;
913 s_ts = cap->rdbk_buf[RDBK_S]->vb.vb2_buf.timestamp;
914
915 if ((m_ts - l_ts) > time || (s_ts - m_ts) > time) {
916 ret = v4l2_subdev_call(sensor->sd,
917 video, g_frame_interval, &sensor->fi);
918 if (!ret) {
919 denominator = sensor->fi.interval.denominator;
920 numerator = sensor->fi.interval.numerator;
921 time = numerator * 1000 / denominator * 1000 * 1000;
922 if (numerator)
923 fps = denominator / numerator;
924 }
925 if ((m_ts - l_ts) > time || (s_ts - m_ts) > time) {
926 v4l2_err(&isp_dev->v4l2_dev,
927 "timestamp no match, s:%lld m:%lld l:%lld, fps:%d\n",
928 s_ts, m_ts, l_ts, fps);
929 goto RDBK_FRM_UNMATCH;
930 }
931 }
932
933 if (m_ts < l_ts || s_ts < m_ts) {
934 v4l2_err(&isp_dev->v4l2_dev,
935 "s/m/l frame err, timestamp s:%lld m:%lld l:%lld\n",
936 s_ts, m_ts, l_ts);
937 goto RDBK_FRM_UNMATCH;
938 }
939
940 cap->rdbk_buf[RDBK_S]->vb.sequence =
941 cap->rdbk_buf[RDBK_L]->vb.sequence;
942 cap->rdbk_buf[RDBK_M]->vb.sequence =
943 cap->rdbk_buf[RDBK_L]->vb.sequence;
944 vb2_buffer_done(&cap->rdbk_buf[RDBK_L]->vb.vb2_buf,
945 VB2_BUF_STATE_DONE);
946 vb2_buffer_done(&cap->rdbk_buf[RDBK_M]->vb.vb2_buf,
947 VB2_BUF_STATE_DONE);
948 vb2_buffer_done(&cap->rdbk_buf[RDBK_S]->vb.vb2_buf,
949 VB2_BUF_STATE_DONE);
950 } else {
951 v4l2_err(&isp_dev->v4l2_dev, "lost long or middle frames\n");
952 goto RDBK_FRM_UNMATCH;
953 }
954 } else if (isp_dev->hdr.op_mode == HDR_RDBK_FRAME2) {
955 if (cap->rdbk_buf[RDBK_L] && cap->rdbk_buf[RDBK_S]) {
956 l_ts = cap->rdbk_buf[RDBK_L]->vb.vb2_buf.timestamp;
957 s_ts = cap->rdbk_buf[RDBK_S]->vb.vb2_buf.timestamp;
958
959 if ((s_ts - l_ts) > time) {
960 ret = v4l2_subdev_call(sensor->sd,
961 video, g_frame_interval, &sensor->fi);
962 if (!ret) {
963 denominator = sensor->fi.interval.denominator;
964 numerator = sensor->fi.interval.numerator;
965 time = numerator * 1000 / denominator * 1000 * 1000;
966 if (numerator)
967 fps = denominator / numerator;
968 }
969 if ((s_ts - l_ts) > time) {
970 v4l2_err(&isp_dev->v4l2_dev,
971 "timestamp no match, s:%lld l:%lld, fps:%d\n",
972 s_ts, l_ts, fps);
973 goto RDBK_FRM_UNMATCH;
974 }
975 }
976
977 if (s_ts < l_ts) {
978 v4l2_err(&isp_dev->v4l2_dev,
979 "s/l frame err, timestamp s:%lld l:%lld\n",
980 s_ts, l_ts);
981 goto RDBK_FRM_UNMATCH;
982 }
983
984 cap->rdbk_buf[RDBK_S]->vb.sequence =
985 cap->rdbk_buf[RDBK_L]->vb.sequence;
986 vb2_buffer_done(&cap->rdbk_buf[RDBK_L]->vb.vb2_buf,
987 VB2_BUF_STATE_DONE);
988 vb2_buffer_done(&cap->rdbk_buf[RDBK_S]->vb.vb2_buf,
989 VB2_BUF_STATE_DONE);
990 } else {
991 v4l2_err(&isp_dev->v4l2_dev, "lost long frames\n");
992 goto RDBK_FRM_UNMATCH;
993 }
994 } else {
995 vb2_buffer_done(&cap->rdbk_buf[RDBK_S]->vb.vb2_buf, VB2_BUF_STATE_DONE);
996 }
997
998 cap->rdbk_buf[RDBK_L] = NULL;
999 cap->rdbk_buf[RDBK_M] = NULL;
1000 cap->rdbk_buf[RDBK_S] = NULL;
1001 return;
1002
1003 RDBK_FRM_UNMATCH:
1004 if (cap->rdbk_buf[RDBK_L])
1005 rkisp_buf_queue(&cap->rdbk_buf[RDBK_L]->vb.vb2_buf);
1006 if (cap->rdbk_buf[RDBK_M])
1007 rkisp_buf_queue(&cap->rdbk_buf[RDBK_M]->vb.vb2_buf);
1008 if (cap->rdbk_buf[RDBK_S])
1009 rkisp_buf_queue(&cap->rdbk_buf[RDBK_S]->vb.vb2_buf);
1010
1011 cap->rdbk_buf[RDBK_L] = NULL;
1012 cap->rdbk_buf[RDBK_M] = NULL;
1013 cap->rdbk_buf[RDBK_S] = NULL;
1014 }
1015
1016 /*
1017 * This function is called when a frame end come. The next frame
1018 * is processing and we should set up buffer for next-next frame,
1019 * otherwise it will overflow.
1020 */
mi_frame_end(struct rkisp_stream * stream)1021 static int mi_frame_end(struct rkisp_stream *stream)
1022 {
1023 struct rkisp_device *dev = stream->ispdev;
1024 struct rkisp_capture_device *cap = &dev->cap_dev;
1025 struct capture_fmt *isp_fmt = &stream->out_isp_fmt;
1026 bool interlaced = stream->interlaced;
1027 unsigned long lock_flags = 0;
1028 u64 ns = 0;
1029 int i = 0;
1030
1031 if (!stream->next_buf && stream->streaming &&
1032 dev->dmarx_dev.trigger == T_MANUAL &&
1033 is_rdbk_stream(stream))
1034 v4l2_info(&dev->v4l2_dev,
1035 "tx stream:%d lose frame:%d, isp state:0x%x frame:%d\n",
1036 stream->id, atomic_read(&stream->sequence) - 1,
1037 dev->isp_state, dev->dmarx_dev.cur_frame.id);
1038
1039 if (stream->curr_buf &&
1040 (!interlaced ||
1041 (stream->u.sp.field_rec == RKISP_FIELD_ODD &&
1042 stream->u.sp.field == RKISP_FIELD_EVEN))) {
1043 struct vb2_buffer *vb2_buf = &stream->curr_buf->vb.vb2_buf;
1044
1045 /* Dequeue a filled buffer */
1046 for (i = 0; i < isp_fmt->mplanes; i++) {
1047 u32 payload_size =
1048 stream->out_fmt.plane_fmt[i].sizeimage;
1049 vb2_set_plane_payload(vb2_buf, i, payload_size);
1050 }
1051 if (stream->id == RKISP_STREAM_MP ||
1052 stream->id == RKISP_STREAM_SP)
1053 rkisp_dmarx_get_frame(dev,
1054 &stream->curr_buf->vb.sequence,
1055 NULL, &ns, true);
1056 else
1057 stream->curr_buf->vb.sequence =
1058 atomic_read(&stream->sequence) - 1;
1059 if (!ns)
1060 ns = ktime_get_ns();
1061 vb2_buf->timestamp = ns;
1062
1063 ns = ktime_get_ns();
1064 stream->dbg.interval = ns - stream->dbg.timestamp;
1065 stream->dbg.timestamp = ns;
1066 stream->dbg.id = stream->curr_buf->vb.sequence;
1067 if (stream->id == RKISP_STREAM_MP || stream->id == RKISP_STREAM_SP)
1068 stream->dbg.delay = ns - dev->isp_sdev.frm_timestamp;
1069
1070 if (!stream->streaming) {
1071 vb2_buffer_done(vb2_buf, VB2_BUF_STATE_ERROR);
1072 } else if (is_rdbk_stream(stream) &&
1073 dev->dmarx_dev.trigger == T_MANUAL) {
1074 if (stream->id == RKISP_STREAM_DMATX0) {
1075 if (cap->rdbk_buf[RDBK_L]) {
1076 v4l2_err(&dev->v4l2_dev,
1077 "multiple long data in hdr frame\n");
1078 rkisp_buf_queue(&cap->rdbk_buf[RDBK_L]->vb.vb2_buf);
1079 stream->dbg.frameloss++;
1080 }
1081 cap->rdbk_buf[RDBK_L] = stream->curr_buf;
1082 } else if (stream->id == RKISP_STREAM_DMATX1) {
1083 if (cap->rdbk_buf[RDBK_M]) {
1084 v4l2_err(&dev->v4l2_dev,
1085 "multiple middle data in hdr frame\n");
1086 rkisp_buf_queue(&cap->rdbk_buf[RDBK_M]->vb.vb2_buf);
1087 stream->dbg.frameloss++;
1088 }
1089 cap->rdbk_buf[RDBK_M] = stream->curr_buf;
1090 } else if (stream->id == RKISP_STREAM_DMATX2) {
1091 if (cap->rdbk_buf[RDBK_S]) {
1092 v4l2_err(&dev->v4l2_dev,
1093 "multiple short data in hdr frame\n");
1094 rkisp_buf_queue(&cap->rdbk_buf[RDBK_S]->vb.vb2_buf);
1095 stream->dbg.frameloss++;
1096 }
1097 cap->rdbk_buf[RDBK_S] = stream->curr_buf;
1098 rdbk_frame_end(stream);
1099 } else {
1100 vb2_buffer_done(vb2_buf, VB2_BUF_STATE_DONE);
1101 }
1102 } else {
1103 if (stream->id == RKISP_STREAM_SP && isp_fmt->fmt_type == FMT_FBCGAIN) {
1104 u32 sizeimage = vb2_plane_size(&stream->curr_buf->vb.vb2_buf, 0);
1105 u32 *buf = (u32 *)vb2_plane_vaddr(&stream->curr_buf->vb.vb2_buf, 0);
1106
1107 *(u64 *)(buf + sizeimage / 4 - 2) = ktime_get_ns();
1108 stream->curr_buf->dev_id = dev->dev_id;
1109 rkisp_bridge_save_spbuf(dev, stream->curr_buf);
1110 } else {
1111 vb2_buffer_done(vb2_buf, VB2_BUF_STATE_DONE);
1112 }
1113 }
1114
1115 stream->curr_buf = NULL;
1116 } else if (stream->id == RKISP_STREAM_SP && isp_fmt->fmt_type == FMT_FBCGAIN) {
1117 u32 frm_id;
1118
1119 rkisp_dmarx_get_frame(dev, &frm_id, NULL, &ns, true);
1120 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
1121 "use dummy buffer, lost sp data, frm_id %d\n", frm_id);
1122 }
1123
1124 if (!interlaced ||
1125 (stream->curr_buf == stream->next_buf &&
1126 stream->u.sp.field == RKISP_FIELD_ODD)) {
1127 /* Next frame is writing to it
1128 * Interlaced: odd field next buffer address
1129 */
1130 stream->curr_buf = stream->next_buf;
1131 stream->next_buf = NULL;
1132
1133 /* Set up an empty buffer for the next-next frame */
1134 spin_lock_irqsave(&stream->vbq_lock, lock_flags);
1135 if (!list_empty(&stream->buf_queue)) {
1136 stream->next_buf =
1137 list_first_entry(&stream->buf_queue,
1138 struct rkisp_buffer,
1139 queue);
1140 list_del(&stream->next_buf->queue);
1141 }
1142 spin_unlock_irqrestore(&stream->vbq_lock, lock_flags);
1143 } else if (stream->u.sp.field_rec == RKISP_FIELD_ODD &&
1144 stream->u.sp.field == RKISP_FIELD_EVEN) {
1145 /* Interlaced: event field next buffer address */
1146 if (stream->next_buf) {
1147 stream->next_buf->buff_addr[RKISP_PLANE_Y] +=
1148 stream->u.sp.vir_offs;
1149 stream->next_buf->buff_addr[RKISP_PLANE_CB] +=
1150 stream->u.sp.vir_offs;
1151 stream->next_buf->buff_addr[RKISP_PLANE_CR] +=
1152 stream->u.sp.vir_offs;
1153 }
1154 stream->curr_buf = stream->next_buf;
1155 }
1156
1157 stream->ops->update_mi(stream);
1158
1159 if (interlaced)
1160 stream->u.sp.field_rec = stream->u.sp.field;
1161
1162 return 0;
1163 }
1164
1165 /***************************** vb2 operations*******************************/
1166
1167 /*
1168 * Set flags and wait, it should stop in interrupt.
1169 * If it didn't, stop it by force.
1170 */
rkisp_stream_stop(struct rkisp_stream * stream)1171 static void rkisp_stream_stop(struct rkisp_stream *stream)
1172 {
1173 struct rkisp_device *dev = stream->ispdev;
1174 struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
1175 int ret = 0;
1176
1177 if (!dev->dmarx_dev.trigger &&
1178 is_rdbk_stream(stream)) {
1179 stream->streaming = false;
1180 return;
1181 }
1182
1183 stream->stopping = true;
1184 stream->ops->stop_mi(stream);
1185 if ((dev->isp_state & ISP_START) &&
1186 dev->isp_inp != INP_DMARX_ISP) {
1187 ret = wait_event_timeout(stream->done,
1188 !stream->streaming,
1189 msecs_to_jiffies(1000));
1190 if (!ret)
1191 v4l2_warn(v4l2_dev, "%s id:%d timeout\n",
1192 __func__, stream->id);
1193 }
1194
1195 stream->stopping = false;
1196 stream->streaming = false;
1197
1198 if (stream->id == RKISP_STREAM_MP ||
1199 stream->id == RKISP_STREAM_SP) {
1200 rkisp_disable_dcrop(stream, true);
1201 rkisp_disable_rsz(stream, true);
1202 }
1203
1204 stream->burst =
1205 CIF_MI_CTRL_BURST_LEN_LUM_16 |
1206 CIF_MI_CTRL_BURST_LEN_CHROM_16;
1207 stream->interlaced = false;
1208 }
1209
1210 /*
1211 * Most of registers inside rockchip isp1 have shadow register since
1212 * they must be not changed during processing a frame.
1213 * Usually, each sub-module updates its shadow register after
1214 * processing the last pixel of a frame.
1215 */
rkisp_start(struct rkisp_stream * stream)1216 static int rkisp_start(struct rkisp_stream *stream)
1217 {
1218 struct rkisp_device *dev = stream->ispdev;
1219 int ret;
1220
1221 /* only MP support HDR mode, SP want to with HDR need
1222 * to start after MP.
1223 */
1224 if (stream->id == RKISP_STREAM_MP)
1225 hdr_config_dmatx(dev);
1226
1227 if (stream->ops->set_data_path)
1228 stream->ops->set_data_path(stream);
1229 ret = stream->ops->config_mi(stream);
1230 if (ret)
1231 return ret;
1232
1233 stream->ops->enable_mi(stream);
1234 stream->streaming = true;
1235
1236 return 0;
1237 }
1238
rkisp_queue_setup(struct vb2_queue * queue,unsigned int * num_buffers,unsigned int * num_planes,unsigned int sizes[],struct device * alloc_ctxs[])1239 static int rkisp_queue_setup(struct vb2_queue *queue,
1240 unsigned int *num_buffers,
1241 unsigned int *num_planes,
1242 unsigned int sizes[],
1243 struct device *alloc_ctxs[])
1244 {
1245 struct rkisp_stream *stream = queue->drv_priv;
1246 struct rkisp_device *dev = stream->ispdev;
1247 const struct v4l2_pix_format_mplane *pixm = NULL;
1248 const struct capture_fmt *isp_fmt = NULL;
1249 u32 i;
1250
1251 pixm = &stream->out_fmt;
1252 isp_fmt = &stream->out_isp_fmt;
1253 *num_planes = isp_fmt->mplanes;
1254
1255 for (i = 0; i < isp_fmt->mplanes; i++) {
1256 const struct v4l2_plane_pix_format *plane_fmt;
1257
1258 plane_fmt = &pixm->plane_fmt[i];
1259 /* height to align with 16 when allocating memory
1260 * so that Rockchip encoder can use DMA buffer directly
1261 */
1262 switch (isp_fmt->fmt_type) {
1263 case FMT_YUV:
1264 sizes[i] = plane_fmt->sizeimage / pixm->height * ALIGN(pixm->height, 16);
1265 break;
1266 case FMT_FBCGAIN:
1267 if (i == isp_fmt->mplanes - 1)
1268 sizes[i] = sizeof(struct isp2x_ispgain_buf);
1269 else
1270 sizes[i] = plane_fmt->sizeimage / pixm->height *
1271 ALIGN(pixm->height, 16) + RKISP_MOTION_DECT_TS_SIZE;
1272 break;
1273 default:
1274 sizes[i] = plane_fmt->sizeimage;
1275 }
1276
1277 }
1278
1279 rkisp_chk_tb_over(dev);
1280 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev, "%s count %d, size %d\n",
1281 v4l2_type_names[queue->type], *num_buffers, sizes[0]);
1282
1283 return 0;
1284 }
1285
1286 /*
1287 * The vb2_buffer are stored in rkisp_buffer, in order to unify
1288 * mplane buffer and none-mplane buffer.
1289 */
rkisp_buf_queue(struct vb2_buffer * vb)1290 static void rkisp_buf_queue(struct vb2_buffer *vb)
1291 {
1292 struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
1293 struct rkisp_buffer *ispbuf = to_rkisp_buffer(vbuf);
1294 struct vb2_queue *queue = vb->vb2_queue;
1295 struct rkisp_stream *stream = queue->drv_priv;
1296 unsigned long lock_flags = 0;
1297 struct v4l2_pix_format_mplane *pixm = &stream->out_fmt;
1298 struct capture_fmt *isp_fmt = &stream->out_isp_fmt;
1299 struct sg_table *sgt;
1300 u32 val;
1301 int i;
1302
1303 memset(ispbuf->buff_addr, 0, sizeof(ispbuf->buff_addr));
1304 for (i = 0; i < isp_fmt->mplanes; i++) {
1305 vb2_plane_vaddr(vb, i);
1306 if (stream->ispdev->hw_dev->is_dma_sg_ops) {
1307 sgt = vb2_dma_sg_plane_desc(vb, i);
1308 ispbuf->buff_addr[i] = sg_dma_address(sgt->sgl);
1309 } else {
1310 ispbuf->buff_addr[i] = vb2_dma_contig_plane_dma_addr(vb, i);
1311 }
1312 }
1313 /*
1314 * NOTE: plane_fmt[0].sizeimage is total size of all planes for single
1315 * memory plane formats, so calculate the size explicitly.
1316 */
1317 if (isp_fmt->mplanes == 1 || isp_fmt->fmt_type == FMT_FBCGAIN) {
1318 for (i = 0; i < isp_fmt->cplanes - 1; i++) {
1319 val = pixm->plane_fmt[i].bytesperline * pixm->height;
1320 if (isp_fmt->fmt_type == FMT_FBCGAIN)
1321 val = ALIGN(val, 64);
1322 ispbuf->buff_addr[i + 1] = (i == 0) ?
1323 ispbuf->buff_addr[i] + val :
1324 ispbuf->buff_addr[i] +
1325 pixm->plane_fmt[i].sizeimage;
1326 }
1327 }
1328
1329 v4l2_dbg(2, rkisp_debug, &stream->ispdev->v4l2_dev,
1330 "stream:%d queue buf:0x%x\n",
1331 stream->id, ispbuf->buff_addr[0]);
1332
1333 if (stream->ispdev->send_fbcgain && stream->id == RKISP_STREAM_SP)
1334 rkisp_bridge_sendtopp_buffer(stream->ispdev, stream->ispdev->dev_id, vb->index);
1335
1336 spin_lock_irqsave(&stream->vbq_lock, lock_flags);
1337 list_add_tail(&ispbuf->queue, &stream->buf_queue);
1338 spin_unlock_irqrestore(&stream->vbq_lock, lock_flags);
1339 }
1340
rkisp_create_dummy_buf(struct rkisp_stream * stream)1341 static int rkisp_create_dummy_buf(struct rkisp_stream *stream)
1342 {
1343 return rkisp_alloc_common_dummy_buf(stream->ispdev);
1344 }
1345
rkisp_destroy_dummy_buf(struct rkisp_stream * stream)1346 static void rkisp_destroy_dummy_buf(struct rkisp_stream *stream)
1347 {
1348 struct rkisp_device *dev = stream->ispdev;
1349
1350 hdr_destroy_buf(dev);
1351 rkisp_free_common_dummy_buf(dev);
1352 }
1353
destroy_buf_queue(struct rkisp_stream * stream,enum vb2_buffer_state state)1354 static void destroy_buf_queue(struct rkisp_stream *stream,
1355 enum vb2_buffer_state state)
1356 {
1357 struct rkisp_device *isp_dev = stream->ispdev;
1358 struct rkisp_capture_device *cap = &isp_dev->cap_dev;
1359 unsigned long lock_flags = 0;
1360 struct rkisp_buffer *buf;
1361
1362 spin_lock_irqsave(&stream->vbq_lock, lock_flags);
1363 if (cap->rdbk_buf[RDBK_L] && stream->id == RKISP_STREAM_DMATX0) {
1364 list_add_tail(&cap->rdbk_buf[RDBK_L]->queue,
1365 &stream->buf_queue);
1366 if (cap->rdbk_buf[RDBK_L] == stream->curr_buf)
1367 stream->curr_buf = NULL;
1368 if (cap->rdbk_buf[RDBK_L] == stream->next_buf)
1369 stream->next_buf = NULL;
1370 cap->rdbk_buf[RDBK_L] = NULL;
1371 }
1372 if (cap->rdbk_buf[RDBK_M] && stream->id == RKISP_STREAM_DMATX1) {
1373 list_add_tail(&cap->rdbk_buf[RDBK_M]->queue,
1374 &stream->buf_queue);
1375 if (cap->rdbk_buf[RDBK_M] == stream->curr_buf)
1376 stream->curr_buf = NULL;
1377 if (cap->rdbk_buf[RDBK_M] == stream->next_buf)
1378 stream->next_buf = NULL;
1379 cap->rdbk_buf[RDBK_M] = NULL;
1380 }
1381 if (cap->rdbk_buf[RDBK_S] && stream->id == RKISP_STREAM_DMATX2) {
1382 list_add_tail(&cap->rdbk_buf[RDBK_S]->queue,
1383 &stream->buf_queue);
1384 if (cap->rdbk_buf[RDBK_S] == stream->curr_buf)
1385 stream->curr_buf = NULL;
1386 if (cap->rdbk_buf[RDBK_S] == stream->next_buf)
1387 stream->next_buf = NULL;
1388 cap->rdbk_buf[RDBK_S] = NULL;
1389 }
1390 if (stream->curr_buf) {
1391 list_add_tail(&stream->curr_buf->queue, &stream->buf_queue);
1392 if (stream->curr_buf == stream->next_buf)
1393 stream->next_buf = NULL;
1394 stream->curr_buf = NULL;
1395 }
1396 if (stream->next_buf) {
1397 list_add_tail(&stream->next_buf->queue, &stream->buf_queue);
1398 stream->next_buf = NULL;
1399 }
1400 while (!list_empty(&stream->buf_queue)) {
1401 buf = list_first_entry(&stream->buf_queue,
1402 struct rkisp_buffer, queue);
1403 list_del(&buf->queue);
1404 vb2_buffer_done(&buf->vb.vb2_buf, state);
1405 }
1406 spin_unlock_irqrestore(&stream->vbq_lock, lock_flags);
1407 }
1408
rkisp_stop_streaming(struct vb2_queue * queue)1409 static void rkisp_stop_streaming(struct vb2_queue *queue)
1410 {
1411 struct rkisp_stream *stream = queue->drv_priv;
1412 struct rkisp_vdev_node *node = &stream->vnode;
1413 struct rkisp_device *dev = stream->ispdev;
1414 struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
1415 int ret;
1416
1417 mutex_lock(&dev->hw_dev->dev_lock);
1418
1419 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
1420 "%s %d\n", __func__, stream->id);
1421
1422 if (stream->id != RKISP_STREAM_SP || stream->out_isp_fmt.fmt_type != FMT_FBCGAIN) {
1423 if (!stream->streaming)
1424 goto end;
1425 rkisp_stream_stop(stream);
1426 }
1427
1428 if (stream->id == RKISP_STREAM_MP ||
1429 (stream->id == RKISP_STREAM_SP && stream->out_isp_fmt.fmt_type != FMT_FBCGAIN)) {
1430 /* call to the other devices */
1431 media_pipeline_stop(&node->vdev.entity);
1432 ret = dev->pipe.set_stream(&dev->pipe, false);
1433 if (ret < 0)
1434 v4l2_err(v4l2_dev,
1435 "pipeline stream-off failed:%d\n", ret);
1436 }
1437
1438 rkisp_stop_spstream(stream);
1439
1440 /* release buffers */
1441 destroy_buf_queue(stream, VB2_BUF_STATE_ERROR);
1442
1443 ret = dev->pipe.close(&dev->pipe);
1444 if (ret < 0)
1445 v4l2_err(v4l2_dev, "pipeline close failed error:%d\n", ret);
1446 rkisp_destroy_dummy_buf(stream);
1447 atomic_dec(&dev->cap_dev.refcnt);
1448 stream->start_stream = false;
1449 end:
1450 mutex_unlock(&dev->hw_dev->dev_lock);
1451 }
1452
rkisp_stream_start(struct rkisp_stream * stream)1453 static int rkisp_stream_start(struct rkisp_stream *stream)
1454 {
1455 struct v4l2_device *v4l2_dev = &stream->ispdev->v4l2_dev;
1456 struct rkisp_device *dev = stream->ispdev;
1457 struct rkisp_stream *other = &dev->cap_dev.stream[stream->id ^ 1];
1458 bool async = false;
1459 int ret;
1460
1461 /* STREAM DMATX don't have rsz and dcrop */
1462 if (stream->id == RKISP_STREAM_DMATX0 ||
1463 stream->id == RKISP_STREAM_DMATX1 ||
1464 stream->id == RKISP_STREAM_DMATX2 ||
1465 stream->id == RKISP_STREAM_DMATX3)
1466 goto end;
1467
1468 if (other->streaming)
1469 async = true;
1470
1471 ret = rkisp_stream_config_rsz(stream, async);
1472 if (ret < 0) {
1473 v4l2_err(v4l2_dev, "config rsz failed with error %d\n", ret);
1474 return ret;
1475 }
1476
1477 /*
1478 * can't be async now, otherwise the latter started stream fails to
1479 * produce mi interrupt.
1480 */
1481 ret = rkisp_stream_config_dcrop(stream, false);
1482 if (ret < 0) {
1483 v4l2_err(v4l2_dev, "config dcrop failed with error %d\n", ret);
1484 return ret;
1485 }
1486
1487 end:
1488 return rkisp_start(stream);
1489 }
1490
1491 static int
rkisp_start_streaming(struct vb2_queue * queue,unsigned int count)1492 rkisp_start_streaming(struct vb2_queue *queue, unsigned int count)
1493 {
1494 struct rkisp_stream *stream = queue->drv_priv;
1495 struct rkisp_vdev_node *node = &stream->vnode;
1496 struct rkisp_device *dev = stream->ispdev;
1497 struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
1498 int ret = -1;
1499
1500 mutex_lock(&dev->hw_dev->dev_lock);
1501
1502 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
1503 "%s %d\n", __func__, stream->id);
1504
1505 if (stream->id != RKISP_STREAM_SP || stream->out_isp_fmt.fmt_type != FMT_FBCGAIN) {
1506 if (WARN_ON(stream->streaming)) {
1507 mutex_unlock(&dev->hw_dev->dev_lock);
1508 return -EBUSY;
1509 }
1510 }
1511
1512 memset(&stream->dbg, 0, sizeof(stream->dbg));
1513 atomic_inc(&dev->cap_dev.refcnt);
1514 if (!dev->isp_inp || !stream->linked) {
1515 v4l2_err(v4l2_dev, "check video link or isp input\n");
1516 goto buffer_done;
1517 }
1518
1519 if (atomic_read(&dev->cap_dev.refcnt) == 1 &&
1520 (dev->isp_inp & INP_CSI || dev->isp_inp & INP_DVP)) {
1521 /* update sensor info when first streaming */
1522 ret = rkisp_update_sensor_info(dev);
1523 if (ret < 0) {
1524 v4l2_err(v4l2_dev,
1525 "update sensor info failed %d\n",
1526 ret);
1527 goto buffer_done;
1528 }
1529 }
1530
1531 if (dev->active_sensor &&
1532 dev->active_sensor->fmt[0].format.field ==
1533 V4L2_FIELD_INTERLACED) {
1534 if (stream->id != RKISP_STREAM_SP) {
1535 v4l2_err(v4l2_dev,
1536 "only selfpath support interlaced\n");
1537 ret = -EINVAL;
1538 goto buffer_done;
1539 }
1540 stream->interlaced = true;
1541 stream->u.sp.field = RKISP_FIELD_INVAL;
1542 stream->u.sp.field_rec = RKISP_FIELD_INVAL;
1543 }
1544
1545 ret = rkisp_create_dummy_buf(stream);
1546 if (ret < 0)
1547 goto buffer_done;
1548
1549 /* enable clocks/power-domains */
1550 ret = dev->pipe.open(&dev->pipe, &node->vdev.entity, true);
1551 if (ret < 0) {
1552 v4l2_err(v4l2_dev, "open cif pipeline failed %d\n", ret);
1553 goto destroy_dummy_buf;
1554 }
1555
1556 if (stream->id != RKISP_STREAM_SP ||
1557 (stream->id == RKISP_STREAM_SP && stream->out_isp_fmt.fmt_type != FMT_FBCGAIN)) {
1558 /* configure stream hardware to start */
1559 ret = rkisp_stream_start(stream);
1560 if (ret < 0) {
1561 v4l2_err(v4l2_dev, "start streaming failed\n");
1562 goto close_pipe;
1563 }
1564 }
1565
1566 if (stream->id == RKISP_STREAM_MP ||
1567 (stream->id == RKISP_STREAM_SP && stream->out_isp_fmt.fmt_type != FMT_FBCGAIN)) {
1568 /* start sub-devices */
1569 ret = dev->pipe.set_stream(&dev->pipe, true);
1570 if (ret < 0)
1571 goto stop_stream;
1572
1573 ret = media_pipeline_start(&node->vdev.entity, &dev->pipe.pipe);
1574 if (ret < 0) {
1575 v4l2_err(&dev->v4l2_dev,
1576 "start pipeline failed %d\n", ret);
1577 goto pipe_stream_off;
1578 }
1579 }
1580
1581 stream->start_stream = true;
1582
1583 mutex_unlock(&dev->hw_dev->dev_lock);
1584 return 0;
1585
1586 pipe_stream_off:
1587 dev->pipe.set_stream(&dev->pipe, false);
1588 stop_stream:
1589 rkisp_stream_stop(stream);
1590 close_pipe:
1591 dev->pipe.close(&dev->pipe);
1592 destroy_dummy_buf:
1593 rkisp_destroy_dummy_buf(stream);
1594 buffer_done:
1595 destroy_buf_queue(stream, VB2_BUF_STATE_QUEUED);
1596 atomic_dec(&dev->cap_dev.refcnt);
1597 stream->streaming = false;
1598 mutex_unlock(&dev->hw_dev->dev_lock);
1599 return ret;
1600 }
1601
1602 static struct vb2_ops rkisp_vb2_ops = {
1603 .queue_setup = rkisp_queue_setup,
1604 .buf_queue = rkisp_buf_queue,
1605 .wait_prepare = vb2_ops_wait_prepare,
1606 .wait_finish = vb2_ops_wait_finish,
1607 .stop_streaming = rkisp_stop_streaming,
1608 .start_streaming = rkisp_start_streaming,
1609 };
1610
rkisp_init_vb2_queue(struct vb2_queue * q,struct rkisp_stream * stream,enum v4l2_buf_type buf_type)1611 static int rkisp_init_vb2_queue(struct vb2_queue *q,
1612 struct rkisp_stream *stream,
1613 enum v4l2_buf_type buf_type)
1614 {
1615 q->type = buf_type;
1616 q->io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF;
1617 q->drv_priv = stream;
1618 q->ops = &rkisp_vb2_ops;
1619 q->mem_ops = stream->ispdev->hw_dev->mem_ops;
1620 q->buf_struct_size = sizeof(struct rkisp_buffer);
1621 q->min_buffers_needed = CIF_ISP_REQ_BUFS_MIN;
1622 q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
1623 q->lock = &stream->apilock;
1624 q->dev = stream->ispdev->hw_dev->dev;
1625 q->allow_cache_hints = 1;
1626 q->bidirectional = 1;
1627 if (stream->ispdev->hw_dev->is_dma_contig)
1628 q->dma_attrs = DMA_ATTR_FORCE_CONTIGUOUS;
1629 q->gfp_flags = GFP_DMA32;
1630 return vb2_queue_init(q);
1631 }
1632
rkisp_stream_init(struct rkisp_device * dev,u32 id)1633 static int rkisp_stream_init(struct rkisp_device *dev, u32 id)
1634 {
1635 struct rkisp_capture_device *cap_dev = &dev->cap_dev;
1636 struct rkisp_stream *stream;
1637 struct video_device *vdev;
1638 struct rkisp_vdev_node *node;
1639 int ret = 0;
1640
1641 stream = &cap_dev->stream[id];
1642 stream->id = id;
1643 stream->ispdev = dev;
1644 vdev = &stream->vnode.vdev;
1645
1646 INIT_LIST_HEAD(&stream->buf_queue);
1647 init_waitqueue_head(&stream->done);
1648 spin_lock_init(&stream->vbq_lock);
1649
1650 stream->linked = true;
1651 /* isp2 disable MP/SP, enable BRIDGE default */
1652 if (id == RKISP_STREAM_MP)
1653 stream->linked = false;
1654
1655 switch (id) {
1656 case RKISP_STREAM_SP:
1657 strlcpy(vdev->name, SP_VDEV_NAME,
1658 sizeof(vdev->name));
1659 stream->ops = &rkisp_sp_streams_ops;
1660 stream->config = &rkisp_sp_stream_config;
1661 break;
1662 case RKISP_STREAM_DMATX0:
1663 strlcpy(vdev->name, DMATX0_VDEV_NAME,
1664 sizeof(vdev->name));
1665 stream->ops = &rkisp2_dmatx0_streams_ops;
1666 stream->config = &rkisp2_dmatx0_stream_config;
1667 break;
1668 case RKISP_STREAM_DMATX1:
1669 strlcpy(vdev->name, DMATX1_VDEV_NAME,
1670 sizeof(vdev->name));
1671 stream->ops = &rkisp2_dmatx1_streams_ops;
1672 stream->config = &rkisp2_dmatx1_stream_config;
1673 break;
1674 case RKISP_STREAM_DMATX2:
1675 strlcpy(vdev->name, DMATX2_VDEV_NAME,
1676 sizeof(vdev->name));
1677 stream->ops = &rkisp2_dmatx2_streams_ops;
1678 stream->config = &rkisp2_dmatx2_stream_config;
1679 break;
1680 case RKISP_STREAM_DMATX3:
1681 strlcpy(vdev->name, DMATX3_VDEV_NAME,
1682 sizeof(vdev->name));
1683 stream->ops = &rkisp2_dmatx3_streams_ops;
1684 stream->config = &rkisp2_dmatx3_stream_config;
1685 break;
1686 default:
1687 strlcpy(vdev->name, MP_VDEV_NAME,
1688 sizeof(vdev->name));
1689 stream->ops = &rkisp_mp_streams_ops;
1690 stream->config = &rkisp_mp_stream_config;
1691 }
1692
1693 node = vdev_to_node(vdev);
1694 rkisp_init_vb2_queue(&node->buf_queue, stream,
1695 V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
1696 ret = rkisp_register_stream_vdev(stream);
1697 if (ret < 0)
1698 return ret;
1699
1700 stream->streaming = false;
1701 stream->interlaced = false;
1702 stream->burst =
1703 CIF_MI_CTRL_BURST_LEN_LUM_16 |
1704 CIF_MI_CTRL_BURST_LEN_CHROM_16;
1705 atomic_set(&stream->sequence, 0);
1706 return 0;
1707 }
1708
rkisp_register_stream_v20(struct rkisp_device * dev)1709 int rkisp_register_stream_v20(struct rkisp_device *dev)
1710 {
1711 struct rkisp_capture_device *cap_dev = &dev->cap_dev;
1712 int ret;
1713
1714 ret = rkisp_stream_init(dev, RKISP_STREAM_MP);
1715 if (ret < 0)
1716 goto err;
1717 ret = rkisp_stream_init(dev, RKISP_STREAM_SP);
1718 if (ret < 0)
1719 goto err_free_mp;
1720 ret = rkisp_stream_init(dev, RKISP_STREAM_DMATX0);
1721 if (ret < 0)
1722 goto err_free_sp;
1723 ret = rkisp_stream_init(dev, RKISP_STREAM_DMATX1);
1724 if (ret < 0)
1725 goto err_free_tx0;
1726 ret = rkisp_stream_init(dev, RKISP_STREAM_DMATX2);
1727 if (ret < 0)
1728 goto err_free_tx1;
1729 ret = rkisp_stream_init(dev, RKISP_STREAM_DMATX3);
1730 if (ret < 0)
1731 goto err_free_tx2;
1732
1733 return 0;
1734 err_free_tx2:
1735 rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_DMATX2]);
1736 err_free_tx1:
1737 rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_DMATX1]);
1738 err_free_tx0:
1739 rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_DMATX0]);
1740 err_free_sp:
1741 rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_SP]);
1742 err_free_mp:
1743 rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_MP]);
1744 err:
1745 return ret;
1746 }
1747
rkisp_unregister_stream_v20(struct rkisp_device * dev)1748 void rkisp_unregister_stream_v20(struct rkisp_device *dev)
1749 {
1750 struct rkisp_capture_device *cap_dev = &dev->cap_dev;
1751 struct rkisp_stream *stream;
1752
1753 stream = &cap_dev->stream[RKISP_STREAM_MP];
1754 rkisp_unregister_stream_vdev(stream);
1755 stream = &cap_dev->stream[RKISP_STREAM_SP];
1756 rkisp_unregister_stream_vdev(stream);
1757 stream = &cap_dev->stream[RKISP_STREAM_DMATX0];
1758 rkisp_unregister_stream_vdev(stream);
1759 stream = &cap_dev->stream[RKISP_STREAM_DMATX1];
1760 rkisp_unregister_stream_vdev(stream);
1761 stream = &cap_dev->stream[RKISP_STREAM_DMATX2];
1762 rkisp_unregister_stream_vdev(stream);
1763 stream = &cap_dev->stream[RKISP_STREAM_DMATX3];
1764 rkisp_unregister_stream_vdev(stream);
1765 }
1766
rkisp_spbuf_queue(struct rkisp_stream * stream,struct rkisp_buffer * sp_buf)1767 void rkisp_spbuf_queue(struct rkisp_stream *stream, struct rkisp_buffer *sp_buf)
1768 {
1769 unsigned long lock_flags = 0;
1770
1771 spin_lock_irqsave(&stream->vbq_lock, lock_flags);
1772 list_add_tail(&sp_buf->queue, &stream->buf_queue);
1773 spin_unlock_irqrestore(&stream->vbq_lock, lock_flags);
1774 }
1775
rkisp_start_spstream(struct rkisp_stream * stream)1776 int rkisp_start_spstream(struct rkisp_stream *stream)
1777 {
1778 struct rkisp_device *dev = stream->ispdev;
1779 struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
1780 int ret = -1;
1781
1782 dev->send_fbcgain = false;
1783 if (stream->id == RKISP_STREAM_SP && stream->out_isp_fmt.fmt_type == FMT_FBCGAIN) {
1784 /* configure stream hardware to start */
1785 ret = rkisp_stream_start(stream);
1786 if (ret < 0) {
1787 v4l2_err(v4l2_dev, "start streaming failed\n");
1788 return ret;
1789 }
1790 dev->send_fbcgain = true;
1791 }
1792
1793 return 0;
1794 }
1795
rkisp_stop_spstream(struct rkisp_stream * stream)1796 void rkisp_stop_spstream(struct rkisp_stream *stream)
1797 {
1798 struct rkisp_device *dev = stream->ispdev;
1799 unsigned long lock_flags = 0;
1800
1801 if (stream->id == RKISP_STREAM_SP && stream->out_isp_fmt.fmt_type == FMT_FBCGAIN) {
1802 rkisp_stream_stop(stream);
1803 rkisp_bridge_stop_spstream(dev);
1804 dev->send_fbcgain = false;
1805
1806 spin_lock_irqsave(&stream->vbq_lock, lock_flags);
1807 if (stream->curr_buf) {
1808 list_add_tail(&stream->curr_buf->queue, &stream->buf_queue);
1809 if (stream->curr_buf == stream->next_buf)
1810 stream->next_buf = NULL;
1811 stream->curr_buf = NULL;
1812 }
1813 if (stream->next_buf) {
1814 list_add_tail(&stream->next_buf->queue, &stream->buf_queue);
1815 stream->next_buf = NULL;
1816 }
1817 spin_unlock_irqrestore(&stream->vbq_lock, lock_flags);
1818 }
1819 }
1820
rkisp_update_spstream_buf(struct rkisp_stream * stream)1821 void rkisp_update_spstream_buf(struct rkisp_stream *stream)
1822 {
1823 if (stream->id == RKISP_STREAM_SP && stream->out_isp_fmt.fmt_type == FMT_FBCGAIN)
1824 mi_frame_end(stream);
1825 }
1826
1827 /**************** Interrupter Handler ****************/
1828
rkisp_mi_v20_isr(u32 mis_val,struct rkisp_device * dev)1829 void rkisp_mi_v20_isr(u32 mis_val, struct rkisp_device *dev)
1830 {
1831 unsigned int i;
1832 static u8 end_tx0, end_tx1, end_tx2;
1833
1834 v4l2_dbg(3, rkisp_debug, &dev->v4l2_dev,
1835 "mi isr:0x%x\n", mis_val);
1836
1837 if (mis_val & CIF_MI_DMA_READY)
1838 rkisp_dmarx_isr(mis_val, dev);
1839
1840 for (i = 0; i < RKISP_MAX_STREAM; ++i) {
1841 struct rkisp_stream *stream = &dev->cap_dev.stream[i];
1842
1843 if (!(mis_val & CIF_MI_FRAME(stream)))
1844 continue;
1845
1846 if (i == RKISP_STREAM_DMATX0)
1847 end_tx0 = true;
1848 if (i == RKISP_STREAM_DMATX1)
1849 end_tx1 = true;
1850 if (i == RKISP_STREAM_DMATX2)
1851 end_tx2 = true;
1852 /* DMATX3 no csi frame end isr, mi isr instead */
1853 if (i == RKISP_STREAM_DMATX3)
1854 atomic_inc(&stream->sequence);
1855
1856 mi_frame_end_int_clear(stream);
1857
1858 if (stream->stopping) {
1859 /*
1860 * Make sure stream is actually stopped, whose state
1861 * can be read from the shadow register, before
1862 * wake_up() thread which would immediately free all
1863 * frame buffers. stop_mi() takes effect at the next
1864 * frame end that sync the configurations to shadow
1865 * regs.
1866 */
1867 if (stream->ops->is_stream_stopped(dev->base_addr)) {
1868 stream->stopping = false;
1869 stream->streaming = false;
1870 wake_up(&stream->done);
1871 }
1872 if (i == RKISP_STREAM_MP) {
1873 end_tx0 = false;
1874 end_tx1 = false;
1875 end_tx2 = false;
1876 }
1877 } else {
1878 mi_frame_end(stream);
1879 if (dev->dmarx_dev.trigger == T_AUTO &&
1880 ((dev->hdr.op_mode == HDR_RDBK_FRAME1 && end_tx2) ||
1881 (dev->hdr.op_mode == HDR_RDBK_FRAME2 && end_tx2 && end_tx0) ||
1882 (dev->hdr.op_mode == HDR_RDBK_FRAME3 && end_tx2 && end_tx1 && end_tx0))) {
1883 end_tx0 = false;
1884 end_tx1 = false;
1885 end_tx2 = false;
1886 rkisp_trigger_read_back(dev, false, false, false);
1887 }
1888 }
1889 }
1890
1891 rkisp_bridge_isr(&mis_val, dev);
1892 }
1893
rkisp_mipi_v20_isr(unsigned int phy,unsigned int packet,unsigned int overflow,unsigned int state,struct rkisp_device * dev)1894 void rkisp_mipi_v20_isr(unsigned int phy, unsigned int packet,
1895 unsigned int overflow, unsigned int state,
1896 struct rkisp_device *dev)
1897 {
1898 struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
1899 struct rkisp_stream *stream;
1900 u32 packet_err = PACKET_ERR_F_BNDRY_MATCG | PACKET_ERR_F_SEQ |
1901 PACKET_ERR_FRAME_DATA | PACKET_ERR_ECC_1BIT |
1902 PACKET_ERR_ECC_2BIT | PACKET_ERR_CHECKSUM;
1903 u32 state_err = RAW_WR_SIZE_ERR | RAW_RD_SIZE_ERR;
1904 int i;
1905
1906 v4l2_dbg(3, rkisp_debug, &dev->v4l2_dev,
1907 "csi state:0x%x\n", state);
1908 dev->csi_dev.irq_cnt++;
1909 if (phy && (dev->isp_inp & INP_CSI) &&
1910 dev->csi_dev.err_cnt++ < RKISP_CONTI_ERR_MAX)
1911 v4l2_warn(v4l2_dev, "MIPI error: phy: 0x%08x\n", phy);
1912 if ((packet & packet_err) && (dev->isp_inp & INP_CSI) &&
1913 dev->csi_dev.err_cnt < RKISP_CONTI_ERR_MAX) {
1914 if (packet & 0xfff)
1915 dev->csi_dev.err_cnt++;
1916 v4l2_warn(v4l2_dev, "MIPI error: packet: 0x%08x\n", packet);
1917 }
1918 if (overflow &&
1919 dev->csi_dev.err_cnt++ < RKISP_CONTI_ERR_MAX)
1920 v4l2_warn(v4l2_dev, "MIPI error: overflow: 0x%08x\n", overflow);
1921 if (state & state_err)
1922 v4l2_warn(v4l2_dev, "MIPI error: size: 0x%08x\n", state);
1923 if (state & MIPI_DROP_FRM)
1924 v4l2_warn(v4l2_dev, "MIPI drop frame\n");
1925
1926 /* first Y_STATE irq as csi sof event */
1927 if (state & (RAW0_Y_STATE | RAW1_Y_STATE | RAW2_Y_STATE)) {
1928 for (i = 0; i < HDR_DMA_MAX; i++) {
1929 if (!((RAW0_Y_STATE << i) & state) ||
1930 dev->csi_dev.tx_first[i])
1931 continue;
1932 dev->csi_dev.tx_first[i] = true;
1933 rkisp_csi_sof(dev, i);
1934 stream = &dev->cap_dev.stream[i + RKISP_STREAM_DMATX0];
1935 atomic_inc(&stream->sequence);
1936 }
1937 }
1938 if (state & (RAW0_WR_FRAME | RAW1_WR_FRAME | RAW2_WR_FRAME)) {
1939 dev->csi_dev.err_cnt = 0;
1940 for (i = 0; i < HDR_DMA_MAX; i++) {
1941 if (!((RAW0_WR_FRAME << i) & state))
1942 continue;
1943 dev->csi_dev.tx_first[i] = false;
1944
1945 if (!IS_HDR_RDBK(dev->hdr.op_mode)) {
1946 stream = &dev->cap_dev.stream[i + RKISP_STREAM_DMATX0];
1947 atomic_inc(&stream->sequence);
1948 }
1949 }
1950 }
1951
1952 if (state & (RAW0_Y_STATE | RAW1_Y_STATE | RAW2_Y_STATE |
1953 RAW0_WR_FRAME | RAW1_WR_FRAME | RAW2_WR_FRAME))
1954 rkisp_luma_isr(&dev->luma_vdev, state);
1955
1956 if (dev->csi_dev.err_cnt > RKISP_CONTI_ERR_MAX) {
1957 if (!(dev->isp_state & ISP_MIPI_ERROR)) {
1958 dev->isp_state |= ISP_MIPI_ERROR;
1959 rkisp_write(dev, CSI2RX_MASK_PHY, 0, true);
1960 rkisp_write(dev, CSI2RX_MASK_PACKET, 0, true);
1961 rkisp_write(dev, CSI2RX_MASK_OVERFLOW, 0, true);
1962 if (dev->hw_dev->monitor.is_en) {
1963 if (!completion_done(&dev->hw_dev->monitor.cmpl))
1964 complete(&dev->hw_dev->monitor.cmpl);
1965 dev->hw_dev->monitor.state |= ISP_MIPI_ERROR;
1966 }
1967 }
1968 }
1969 }
1970