1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2021 Rockchip Electronics Co., Ltd. */
3
4 #include <linux/delay.h>
5 #include <linux/pm_runtime.h>
6 #include <media/v4l2-common.h>
7 #include <media/v4l2-event.h>
8 #include <media/v4l2-fh.h>
9 #include <media/v4l2-ioctl.h>
10 #include <media/v4l2-subdev.h>
11 #include <media/videobuf2-dma-contig.h>
12 #include <media/videobuf2-dma-sg.h>
13 #include "dev.h"
14 #include "regs.h"
15
16 #define CIF_ISP_REQ_BUFS_MIN 0
17
18 static int mi_frame_end(struct rkisp_stream *stream);
19
20 static const struct capture_fmt fbc_fmts[] = {
21 {
22 .fourcc = V4L2_PIX_FMT_FBC0,
23 .fmt_type = FMT_FBC,
24 .bpp = { 8, 16 },
25 .cplanes = 2,
26 .mplanes = 1,
27 .write_format = ISP3X_MPFBC_YUV420,
28 }, {
29 .fourcc = V4L2_PIX_FMT_FBC2,
30 .fmt_type = FMT_FBC,
31 .bpp = { 8, 16 },
32 .cplanes = 2,
33 .mplanes = 1,
34 .write_format = ISP3X_MPFBC_YUV422,
35 }
36 };
37
38 static const struct capture_fmt bp_fmts[] = {
39 {
40 .fourcc = V4L2_PIX_FMT_UYVY,
41 .fmt_type = FMT_YUV,
42 .bpp = { 16 },
43 .cplanes = 1,
44 .mplanes = 1,
45 .write_format = ISP3X_BP_FORMAT_INT,
46 }, {
47 .fourcc = V4L2_PIX_FMT_NV16,
48 .fmt_type = FMT_YUV,
49 .bpp = { 8, 16 },
50 .cplanes = 2,
51 .mplanes = 1,
52 .write_format = ISP3X_BP_FORMAT_SPLA,
53 }, {
54 .fourcc = V4L2_PIX_FMT_NV12,
55 .fmt_type = FMT_YUV,
56 .bpp = { 8, 16 },
57 .cplanes = 2,
58 .mplanes = 1,
59 .write_format = ISP3X_BP_FORMAT_SPLA,
60 }, {
61 .fourcc = V4L2_PIX_FMT_NV12M,
62 .fmt_type = FMT_YUV,
63 .bpp = { 8, 16 },
64 .cplanes = 2,
65 .mplanes = 2,
66 .write_format = ISP3X_BP_FORMAT_SPLA,
67 }
68 };
69
70 struct stream_config rkisp_fbc_stream_config = {
71 .fmts = fbc_fmts,
72 .fmt_size = ARRAY_SIZE(fbc_fmts),
73 .frame_end_id = ISP3X_MI_MPFBC_FRAME,
74 .dual_crop = {
75 .ctrl = ISP3X_DUAL_CROP_CTRL,
76 .yuvmode_mask = ISP3X_DUAL_CROP_FBC_MODE,
77 .h_offset = ISP3X_DUAL_CROP_FBC_H_OFFS,
78 .v_offset = ISP3X_DUAL_CROP_FBC_V_OFFS,
79 .h_size = ISP3X_DUAL_CROP_FBC_H_SIZE,
80 .v_size = ISP3X_DUAL_CROP_FBC_V_SIZE,
81 },
82 .mi = {
83 .y_base_ad_init = ISP3X_MPFBC_HEAD_PTR,
84 .cb_base_ad_init = ISP3X_MPFBC_PAYL_PTR,
85 .y_base_ad_shd = ISP3X_MPFBC_HEAD_PTR,
86 },
87 };
88
89 struct stream_config rkisp_bp_stream_config = {
90 .fmts = bp_fmts,
91 .fmt_size = ARRAY_SIZE(bp_fmts),
92 .frame_end_id = ISP3X_MI_BP_FRAME,
93 .dual_crop = {
94 .ctrl = ISP3X_DUAL_CROP_CTRL,
95 .yuvmode_mask = ISP3X_DUAL_CROP_FBC_MODE,
96 .h_offset = ISP3X_DUAL_CROP_FBC_H_OFFS,
97 .v_offset = ISP3X_DUAL_CROP_FBC_V_OFFS,
98 .h_size = ISP3X_DUAL_CROP_FBC_H_SIZE,
99 .v_size = ISP3X_DUAL_CROP_FBC_V_SIZE,
100 },
101 .mi = {
102 .y_size_init = ISP3X_MI_BP_WR_Y_SIZE,
103 .cb_size_init = ISP3X_MI_BP_WR_CB_SIZE,
104 .y_base_ad_init = ISP3X_MI_BP_WR_Y_BASE,
105 .cb_base_ad_init = ISP3X_MI_BP_WR_CB_BASE,
106 .y_offs_cnt_init = ISP3X_MI_BP_WR_Y_OFFS_CNT,
107 .cb_offs_cnt_init = ISP3X_MI_BP_WR_CB_OFFS_CNT,
108 .y_base_ad_shd = ISP3X_MI_BP_WR_Y_BASE_SHD,
109 },
110 };
111
is_fbc_stream_stopped(void __iomem * base)112 static bool is_fbc_stream_stopped(void __iomem *base)
113 {
114 u32 ret = readl(base + ISP3X_MPFBC_CTRL);
115
116 return !(ret & ISP3X_MPFBC_EN_SHD);
117 }
118
get_stream_irq_mask(struct rkisp_stream * stream)119 static int get_stream_irq_mask(struct rkisp_stream *stream)
120 {
121 int ret;
122
123 switch (stream->id) {
124 case RKISP_STREAM_SP:
125 ret = ISP_FRAME_SP;
126 break;
127 case RKISP_STREAM_FBC:
128 ret = ISP_FRAME_MPFBC;
129 break;
130 case RKISP_STREAM_BP:
131 ret = ISP_FRAME_BP;
132 break;
133 case RKISP_STREAM_MP:
134 default:
135 ret = ISP_FRAME_MP;
136 }
137
138 return ret;
139 }
140
141 /* configure dual-crop unit */
rkisp_stream_config_dcrop(struct rkisp_stream * stream,bool async)142 static int rkisp_stream_config_dcrop(struct rkisp_stream *stream, bool async)
143 {
144 struct rkisp_device *dev = stream->ispdev;
145 struct v4l2_rect *dcrop = &stream->dcrop;
146 struct v4l2_rect *input_win;
147
148 /* dual-crop unit get data from isp */
149 input_win = rkisp_get_isp_sd_win(&dev->isp_sdev);
150
151 if (dcrop->width == input_win->width &&
152 dcrop->height == input_win->height &&
153 dcrop->left == 0 && dcrop->top == 0 &&
154 !dev->hw_dev->is_unite) {
155 rkisp_disable_dcrop(stream, async);
156 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
157 "stream %d crop disabled\n", stream->id);
158 return 0;
159 }
160
161 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
162 "stream %d crop: %dx%d -> %dx%d\n", stream->id,
163 input_win->width, input_win->height,
164 dcrop->width, dcrop->height);
165
166 rkisp_config_dcrop(stream, dcrop, async);
167
168 return 0;
169 }
170
171 /* configure scale unit */
rkisp_stream_config_rsz(struct rkisp_stream * stream,bool async)172 static int rkisp_stream_config_rsz(struct rkisp_stream *stream, bool async)
173 {
174 struct rkisp_device *dev = stream->ispdev;
175 struct v4l2_pix_format_mplane output_fmt = stream->out_fmt;
176 struct ispsd_out_fmt *input_isp_fmt =
177 rkisp_get_ispsd_out_fmt(&dev->isp_sdev);
178 struct v4l2_rect in_y, in_c, out_y, out_c;
179
180 if (input_isp_fmt->fmt_type == FMT_BAYER ||
181 stream->id == RKISP_STREAM_FBC ||
182 stream->id == RKISP_STREAM_BP)
183 goto disable;
184
185 /* set input and output sizes for scale calculation
186 * input/output yuv422
187 */
188 in_y.width = stream->dcrop.width;
189 in_y.height = stream->dcrop.height;
190 in_c.width = in_y.width / 2;
191 in_c.height = in_y.height;
192
193 out_y.width = output_fmt.width;
194 out_y.height = output_fmt.height;
195 out_c.width = out_y.width / 2;
196 out_c.height = out_y.height;
197
198 if (in_c.width == out_c.width && in_c.height == out_c.height)
199 goto disable;
200
201 /* set RSZ input and output */
202 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
203 "stream %d rsz/scale: %dx%d -> %dx%d\n",
204 stream->id, stream->dcrop.width, stream->dcrop.height,
205 output_fmt.width, output_fmt.height);
206 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
207 "chroma scaling %dx%d -> %dx%d\n",
208 in_c.width, in_c.height, out_c.width, out_c.height);
209
210 /* calculate and set scale */
211 rkisp_config_rsz(stream, &in_y, &in_c, &out_y, &out_c, async);
212
213 return 0;
214
215 disable:
216 rkisp_disable_rsz(stream, async);
217
218 return 0;
219 }
220
221 /***************************** stream operations*******************************/
222
223 /*
224 * memory base addresses should be with respect
225 * to the burst alignment restriction for AXI.
226 */
calc_burst_len(struct rkisp_stream * stream)227 static u32 calc_burst_len(struct rkisp_stream *stream)
228 {
229 struct rkisp_device *dev = stream->ispdev;
230 u32 y_size = stream->out_fmt.plane_fmt[0].bytesperline *
231 stream->out_fmt.height;
232 u32 cb_size = stream->out_fmt.plane_fmt[1].sizeimage;
233 u32 cr_size = stream->out_fmt.plane_fmt[2].sizeimage;
234 u32 cb_offs, cr_offs;
235 u32 bus = 16, burst;
236 int i;
237
238 /* y/c base addr: burstN * bus alignment */
239 cb_offs = y_size;
240 cr_offs = cr_size ? (cb_size + cb_offs) : 0;
241
242 if (!(cb_offs % (bus * 16)) && !(cr_offs % (bus * 16)))
243 burst = CIF_MI_CTRL_BURST_LEN_LUM_16 |
244 CIF_MI_CTRL_BURST_LEN_CHROM_16;
245 else if (!(cb_offs % (bus * 8)) && !(cr_offs % (bus * 8)))
246 burst = CIF_MI_CTRL_BURST_LEN_LUM_8 |
247 CIF_MI_CTRL_BURST_LEN_CHROM_8;
248 else
249 burst = CIF_MI_CTRL_BURST_LEN_LUM_4 |
250 CIF_MI_CTRL_BURST_LEN_CHROM_4;
251
252 if (cb_offs % (bus * 4) || cr_offs % (bus * 4))
253 v4l2_warn(&dev->v4l2_dev,
254 "%dx%d fmt:0x%x not support, should be %d aligned\n",
255 stream->out_fmt.width,
256 stream->out_fmt.height,
257 stream->out_fmt.pixelformat,
258 (cr_offs == 0) ? bus * 4 : bus * 16);
259
260 stream->burst = burst;
261 for (i = 0; i <= RKISP_STREAM_SP; i++)
262 if (burst > dev->cap_dev.stream[i].burst)
263 burst = dev->cap_dev.stream[i].burst;
264
265 if (stream->interlaced) {
266 if (!stream->out_fmt.width % (bus * 16))
267 stream->burst = CIF_MI_CTRL_BURST_LEN_LUM_16 |
268 CIF_MI_CTRL_BURST_LEN_CHROM_16;
269 else if (!stream->out_fmt.width % (bus * 8))
270 stream->burst = CIF_MI_CTRL_BURST_LEN_LUM_8 |
271 CIF_MI_CTRL_BURST_LEN_CHROM_8;
272 else
273 stream->burst = CIF_MI_CTRL_BURST_LEN_LUM_4 |
274 CIF_MI_CTRL_BURST_LEN_CHROM_4;
275 if (stream->out_fmt.width % (bus * 4))
276 v4l2_warn(&dev->v4l2_dev,
277 "interlaced: width should be %d aligned\n",
278 bus * 4);
279 burst = min(stream->burst, burst);
280 stream->burst = burst;
281 }
282
283 return burst;
284 }
285
286 /*
287 * configure memory interface for mainpath
288 * This should only be called when stream-on
289 */
mp_config_mi(struct rkisp_stream * stream)290 static int mp_config_mi(struct rkisp_stream *stream)
291 {
292 struct rkisp_device *dev = stream->ispdev;
293 struct v4l2_pix_format_mplane *out_fmt = &stream->out_fmt;
294 bool is_unite = dev->hw_dev->is_unite;
295 u32 val, mask;
296
297 /*
298 * NOTE: plane_fmt[0].sizeimage is total size of all planes for single
299 * memory plane formats, so calculate the size explicitly.
300 */
301 val = out_fmt->plane_fmt[0].bytesperline * out_fmt->height;
302 rkisp_unite_write(dev, stream->config->mi.y_size_init, val, false, is_unite);
303
304 val = out_fmt->plane_fmt[1].sizeimage;
305 rkisp_unite_write(dev, stream->config->mi.cb_size_init, val, false, is_unite);
306
307 val = out_fmt->plane_fmt[2].sizeimage;
308 rkisp_unite_write(dev, stream->config->mi.cr_size_init, val, false, is_unite);
309
310 val = is_unite ? out_fmt->width / 2 : out_fmt->width;
311 rkisp_unite_write(dev, ISP3X_MI_MP_WR_Y_PIC_WIDTH, val, false, is_unite);
312
313 val = out_fmt->height;
314 rkisp_unite_write(dev, ISP3X_MI_MP_WR_Y_PIC_HEIGHT, val, false, is_unite);
315
316 val = ALIGN(out_fmt->plane_fmt[0].bytesperline, 16);
317 rkisp_unite_write(dev, ISP3X_MI_MP_WR_Y_LLENGTH, val, false, is_unite);
318
319 val = stream->out_isp_fmt.uv_swap ? ISP3X_MI_XTD_FORMAT_MP_UV_SWAP : 0;
320 mask = ISP3X_MI_XTD_FORMAT_MP_UV_SWAP;
321 rkisp_unite_set_bits(dev, ISP3X_MI_WR_XTD_FORMAT_CTRL, mask, val, false, is_unite);
322
323 mask = ISP3X_MPFBC_FORCE_UPD | ISP3X_MP_YUV_MODE;
324 val = rkisp_read_reg_cache(dev, ISP3X_MPFBC_CTRL) & ~mask;
325 if (stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV21 ||
326 stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV12 ||
327 stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV21M ||
328 stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV12M ||
329 stream->out_fmt.pixelformat == V4L2_PIX_FMT_YUV420)
330 val |= ISP3X_SEPERATE_YUV_CFG;
331 else
332 val |= ISP3X_SEPERATE_YUV_CFG | ISP3X_MP_YUV_MODE;
333 rkisp_unite_write(dev, ISP3X_MPFBC_CTRL, val, false, is_unite);
334
335 val = calc_burst_len(stream) | CIF_MI_CTRL_INIT_BASE_EN |
336 CIF_MI_CTRL_INIT_OFFSET_EN | CIF_MI_MP_AUTOUPDATE_ENABLE |
337 stream->out_isp_fmt.write_format;
338 mask = GENMASK(19, 16) | MI_CTRL_MP_FMT_MASK;
339 rkisp_unite_set_bits(dev, ISP3X_MI_WR_CTRL, mask, val, false, is_unite);
340
341 mi_frame_end_int_enable(stream);
342 /* set up first buffer */
343 mi_frame_end(stream);
344 return 0;
345 }
346
mbus_code_sp_in_fmt(u32 in_mbus_code,u32 out_fourcc,u32 * format)347 static int mbus_code_sp_in_fmt(u32 in_mbus_code, u32 out_fourcc, u32 *format)
348 {
349 switch (in_mbus_code) {
350 case MEDIA_BUS_FMT_YUYV8_2X8:
351 *format = MI_CTRL_SP_INPUT_YUV422;
352 break;
353 default:
354 return -EINVAL;
355 }
356
357 /*
358 * Only SP can support output format of YCbCr4:0:0,
359 * and the input format of SP must be YCbCr4:0:0
360 * when outputting YCbCr4:0:0.
361 * The output format of isp is YCbCr4:2:2,
362 * so the CbCr data is discarded here.
363 */
364 if (out_fourcc == V4L2_PIX_FMT_GREY)
365 *format = MI_CTRL_SP_INPUT_YUV400;
366
367 return 0;
368 }
369
370 /*
371 * configure memory interface for selfpath
372 * This should only be called when stream-on
373 */
sp_config_mi(struct rkisp_stream * stream)374 static int sp_config_mi(struct rkisp_stream *stream)
375 {
376 struct rkisp_device *dev = stream->ispdev;
377 struct v4l2_pix_format_mplane *out_fmt = &stream->out_fmt;
378 struct ispsd_out_fmt *input_isp_fmt =
379 rkisp_get_ispsd_out_fmt(&dev->isp_sdev);
380 bool is_unite = dev->hw_dev->is_unite;
381 u32 sp_in_fmt, val, mask;
382
383 if (mbus_code_sp_in_fmt(input_isp_fmt->mbus_code,
384 out_fmt->pixelformat, &sp_in_fmt)) {
385 v4l2_err(&dev->v4l2_dev, "Can't find the input format\n");
386 return -EINVAL;
387 }
388
389 /*
390 * NOTE: plane_fmt[0].sizeimage is total size of all planes for single
391 * memory plane formats, so calculate the size explicitly.
392 */
393 val = out_fmt->plane_fmt[0].bytesperline * out_fmt->height;
394 rkisp_unite_write(dev, stream->config->mi.y_size_init, val, false, is_unite);
395
396 val = out_fmt->plane_fmt[1].sizeimage;
397 rkisp_unite_write(dev, stream->config->mi.cb_size_init, val, false, is_unite);
398
399 val = out_fmt->plane_fmt[2].sizeimage;
400 rkisp_unite_write(dev, stream->config->mi.cr_size_init, val, false, is_unite);
401
402 val = is_unite ? out_fmt->width / 2 : out_fmt->width;
403 rkisp_unite_write(dev, ISP3X_MI_SP_WR_Y_PIC_WIDTH, val, false, is_unite);
404
405 val = out_fmt->height;
406 rkisp_unite_write(dev, ISP3X_MI_SP_WR_Y_PIC_HEIGHT, val, false, is_unite);
407
408 val = ALIGN(out_fmt->plane_fmt[0].bytesperline, 16);
409 rkisp_unite_write(dev, ISP3X_MI_SP_WR_Y_LLENGTH, val, false, is_unite);
410
411 val = stream->out_isp_fmt.uv_swap ? ISP3X_MI_XTD_FORMAT_SP_UV_SWAP : 0;
412 mask = ISP3X_MI_XTD_FORMAT_SP_UV_SWAP;
413 rkisp_unite_set_bits(dev, ISP3X_MI_WR_XTD_FORMAT_CTRL, mask, val, false, is_unite);
414
415 mask = ISP3X_MPFBC_FORCE_UPD | ISP3X_SP_YUV_MODE;
416 val = rkisp_read_reg_cache(dev, ISP3X_MPFBC_CTRL) & ~mask;
417 if (stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV21 ||
418 stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV12 ||
419 stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV21M ||
420 stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV12M ||
421 stream->out_fmt.pixelformat == V4L2_PIX_FMT_YUV420)
422 val |= ISP3X_SEPERATE_YUV_CFG;
423 else
424 val |= ISP3X_SEPERATE_YUV_CFG | ISP3X_SP_YUV_MODE;
425 rkisp_unite_write(dev, ISP3X_MPFBC_CTRL, val, false, is_unite);
426
427 val = calc_burst_len(stream) | CIF_MI_CTRL_INIT_BASE_EN |
428 CIF_MI_CTRL_INIT_OFFSET_EN | stream->out_isp_fmt.write_format |
429 sp_in_fmt | stream->out_isp_fmt.output_format |
430 CIF_MI_SP_AUTOUPDATE_ENABLE;
431 mask = GENMASK(19, 16) | MI_CTRL_SP_FMT_MASK;
432 rkisp_unite_set_bits(dev, ISP3X_MI_WR_CTRL, mask, val, false, is_unite);
433
434 mi_frame_end_int_enable(stream);
435 /* set up first buffer */
436 mi_frame_end(stream);
437 return 0;
438 }
439
fbc_config_mi(struct rkisp_stream * stream)440 static int fbc_config_mi(struct rkisp_stream *stream)
441 {
442 /* yuv422 is 16*2, yuv420 is 16*1.5 */
443 u32 mult = stream->out_isp_fmt.write_format ? 32 : 24;
444 u32 h = ALIGN(stream->out_fmt.height, 16);
445 u32 w = ALIGN(stream->out_fmt.width, 16);
446 u32 offs = ALIGN(w * h / 16, RK_MPP_ALIGN);
447 bool is_unite = stream->ispdev->hw_dev->is_unite;
448
449 rkisp_write(stream->ispdev, ISP3X_MPFBC_HEAD_OFFSET, offs, false);
450 rkisp_unite_write(stream->ispdev, ISP3X_MPFBC_VIR_WIDTH, w, false, is_unite);
451 rkisp_unite_write(stream->ispdev, ISP3X_MPFBC_PAYL_WIDTH, w, false, is_unite);
452 rkisp_unite_write(stream->ispdev, ISP3X_MPFBC_VIR_HEIGHT, h, false, is_unite);
453 if (is_unite) {
454 u32 left_w = (stream->out_fmt.width / 2) & ~0xf;
455
456 offs += left_w * mult;
457 rkisp_next_write(stream->ispdev, ISP3X_MPFBC_HEAD_OFFSET, offs, false);
458 }
459 rkisp_unite_set_bits(stream->ispdev, ISP3X_MI_WR_CTRL, 0,
460 CIF_MI_CTRL_INIT_BASE_EN | CIF_MI_CTRL_INIT_OFFSET_EN,
461 false, is_unite);
462 mi_frame_end_int_enable(stream);
463 /* set up first buffer */
464 mi_frame_end(stream);
465 return 0;
466 }
467
bp_config_mi(struct rkisp_stream * stream)468 static int bp_config_mi(struct rkisp_stream *stream)
469 {
470 struct v4l2_pix_format_mplane *out_fmt = &stream->out_fmt;
471 struct rkisp_device *dev = stream->ispdev;
472 bool is_unite = dev->hw_dev->is_unite;
473 u32 val, mask;
474
475 /*
476 * NOTE: plane_fmt[0].sizeimage is total size of all planes for single
477 * memory plane formats, so calculate the size explicitly.
478 */
479 val = out_fmt->plane_fmt[0].bytesperline * out_fmt->height;
480 rkisp_unite_write(dev, stream->config->mi.y_size_init, val, false, is_unite);
481
482 val = out_fmt->plane_fmt[1].sizeimage;
483 rkisp_unite_write(dev, stream->config->mi.cb_size_init, val, false, is_unite);
484
485 val = is_unite ? out_fmt->width / 2 : out_fmt->width;
486 rkisp_unite_write(dev, ISP3X_MI_BP_WR_Y_PIC_WIDTH, val, false, is_unite);
487
488 val = out_fmt->height;
489 rkisp_unite_write(dev, ISP3X_MI_BP_WR_Y_PIC_HEIGHT, val, false, is_unite);
490
491 val = ALIGN(out_fmt->plane_fmt[0].bytesperline, 16);
492 rkisp_unite_write(dev, ISP3X_MI_BP_WR_Y_LLENGTH, val, false, is_unite);
493
494 mask = ISP3X_MPFBC_FORCE_UPD | ISP3X_BP_YUV_MODE;
495 val = rkisp_read_reg_cache(dev, ISP3X_MPFBC_CTRL) & ~mask;
496
497 if (out_fmt->pixelformat == V4L2_PIX_FMT_NV12 ||
498 out_fmt->pixelformat == V4L2_PIX_FMT_NV12M)
499 val |= ISP3X_SEPERATE_YUV_CFG;
500 else
501 val |= ISP3X_SEPERATE_YUV_CFG | ISP3X_BP_YUV_MODE;
502 rkisp_unite_write(dev, ISP3X_MPFBC_CTRL, val, false, is_unite);
503 val = CIF_MI_CTRL_INIT_BASE_EN | CIF_MI_CTRL_INIT_OFFSET_EN;
504 rkisp_unite_set_bits(dev, ISP3X_MI_WR_CTRL, 0, val, false, is_unite);
505 mi_frame_end_int_enable(stream);
506 /* set up first buffer */
507 mi_frame_end(stream);
508 return 0;
509 }
510
mp_enable_mi(struct rkisp_stream * stream)511 static void mp_enable_mi(struct rkisp_stream *stream)
512 {
513 struct capture_fmt *isp_fmt = &stream->out_isp_fmt;
514 u32 mask = CIF_MI_CTRL_MP_ENABLE | CIF_MI_CTRL_RAW_ENABLE;
515 u32 val = CIF_MI_CTRL_MP_ENABLE;
516
517 if (isp_fmt->fmt_type == FMT_BAYER)
518 val = CIF_MI_CTRL_RAW_ENABLE;
519 rkisp_unite_set_bits(stream->ispdev, ISP3X_MI_WR_CTRL, mask, val,
520 false, stream->ispdev->hw_dev->is_unite);
521 }
522
sp_enable_mi(struct rkisp_stream * stream)523 static void sp_enable_mi(struct rkisp_stream *stream)
524 {
525 rkisp_unite_set_bits(stream->ispdev, ISP3X_MI_WR_CTRL, 0,
526 CIF_MI_CTRL_SP_ENABLE, false,
527 stream->ispdev->hw_dev->is_unite);
528 }
529
fbc_enable_mi(struct rkisp_stream * stream)530 static void fbc_enable_mi(struct rkisp_stream *stream)
531 {
532 u32 val, mask = ISP3X_MPFBC_FORCE_UPD | ISP3X_MPFBC_YUV_MASK |
533 ISP3X_MPFBC_SPARSE_MODE;
534 bool is_unite = stream->ispdev->hw_dev->is_unite;
535
536 /* config no effect immediately, read back is shadow, get config value from cache */
537 val = rkisp_read_reg_cache(stream->ispdev, ISP3X_MPFBC_CTRL) & ~mask;
538 val |= stream->out_isp_fmt.write_format | ISP3X_HEAD_OFFSET_EN | ISP3X_MPFBC_EN;
539 rkisp_unite_write(stream->ispdev, ISP3X_MPFBC_CTRL, val, false, is_unite);
540 }
541
bp_enable_mi(struct rkisp_stream * stream)542 static void bp_enable_mi(struct rkisp_stream *stream)
543 {
544 u32 val = stream->out_isp_fmt.write_format |
545 ISP3X_BP_ENABLE | ISP3X_BP_AUTO_UPD;
546
547 rkisp_unite_write(stream->ispdev, ISP3X_MI_BP_WR_CTRL, val, false,
548 stream->ispdev->hw_dev->is_unite);
549 }
550
mp_disable_mi(struct rkisp_stream * stream)551 static void mp_disable_mi(struct rkisp_stream *stream)
552 {
553 u32 mask = CIF_MI_CTRL_MP_ENABLE | CIF_MI_CTRL_RAW_ENABLE;
554
555 rkisp_unite_clear_bits(stream->ispdev, ISP3X_MI_WR_CTRL, mask, false,
556 stream->ispdev->hw_dev->is_unite);
557 }
558
sp_disable_mi(struct rkisp_stream * stream)559 static void sp_disable_mi(struct rkisp_stream *stream)
560 {
561 rkisp_unite_clear_bits(stream->ispdev, ISP3X_MI_WR_CTRL, CIF_MI_CTRL_SP_ENABLE,
562 false, stream->ispdev->hw_dev->is_unite);
563 }
564
fbc_disable_mi(struct rkisp_stream * stream)565 static void fbc_disable_mi(struct rkisp_stream *stream)
566 {
567 u32 mask = ISP3X_MPFBC_FORCE_UPD | ISP3X_MPFBC_EN;
568
569 rkisp_unite_clear_bits(stream->ispdev, ISP3X_MPFBC_CTRL, mask,
570 false, stream->ispdev->hw_dev->is_unite);
571 }
572
bp_disable_mi(struct rkisp_stream * stream)573 static void bp_disable_mi(struct rkisp_stream *stream)
574 {
575 rkisp_unite_clear_bits(stream->ispdev, ISP3X_MI_BP_WR_CTRL, ISP3X_BP_ENABLE,
576 false, stream->ispdev->hw_dev->is_unite);
577 }
578
update_mi(struct rkisp_stream * stream)579 static void update_mi(struct rkisp_stream *stream)
580 {
581 struct rkisp_device *dev = stream->ispdev;
582 struct rkisp_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
583 u32 val, reg;
584
585 if (stream->next_buf) {
586 reg = stream->config->mi.y_base_ad_init;
587 val = stream->next_buf->buff_addr[RKISP_PLANE_Y];
588 rkisp_write(dev, reg, val, false);
589
590 reg = stream->config->mi.cb_base_ad_init;
591 val = stream->next_buf->buff_addr[RKISP_PLANE_CB];
592 rkisp_write(dev, reg, val, false);
593
594 if (stream->id != RKISP_STREAM_FBC && stream->id != RKISP_STREAM_BP) {
595 reg = stream->config->mi.cr_base_ad_init;
596 val = stream->next_buf->buff_addr[RKISP_PLANE_CR];
597 rkisp_write(dev, reg, val, false);
598 }
599
600 if (dev->hw_dev->is_unite) {
601 u32 mult = stream->id != RKISP_STREAM_FBC ? 1 :
602 (stream->out_isp_fmt.write_format ? 32 : 24);
603
604 reg = stream->config->mi.y_base_ad_init;
605 val = stream->next_buf->buff_addr[RKISP_PLANE_Y];
606 val += ((stream->out_fmt.width / 2) & ~0xf);
607 rkisp_next_write(dev, reg, val, false);
608
609 reg = stream->config->mi.cb_base_ad_init;
610 val = stream->next_buf->buff_addr[RKISP_PLANE_CB];
611 val += ((stream->out_fmt.width / 2) & ~0xf) * mult;
612 rkisp_next_write(dev, reg, val, false);
613
614 if (stream->id != RKISP_STREAM_FBC && stream->id != RKISP_STREAM_BP) {
615 reg = stream->config->mi.cr_base_ad_init;
616 val = stream->next_buf->buff_addr[RKISP_PLANE_CR];
617 val += ((stream->out_fmt.width / 2) & ~0xf);
618 rkisp_next_write(dev, reg, val, false);
619 }
620 }
621
622 /* single buf updated at readback for multidevice */
623 if (!dev->hw_dev->is_single) {
624 stream->curr_buf = stream->next_buf;
625 stream->next_buf = NULL;
626 }
627 } else if (dummy_buf->mem_priv) {
628 stream->dbg.frameloss++;
629 val = dummy_buf->dma_addr;
630 reg = stream->config->mi.y_base_ad_init;
631 rkisp_unite_write(dev, reg, val, false, dev->hw_dev->is_unite);
632 reg = stream->config->mi.cb_base_ad_init;
633 rkisp_unite_write(dev, reg, val, false, dev->hw_dev->is_unite);
634 reg = stream->config->mi.cr_base_ad_init;
635 if (stream->id != RKISP_STREAM_FBC && stream->id != RKISP_STREAM_BP)
636 rkisp_unite_write(dev, reg, val, false, dev->hw_dev->is_unite);
637 }
638
639 if (stream->id != RKISP_STREAM_FBC) {
640 reg = stream->config->mi.y_offs_cnt_init;
641 rkisp_unite_write(dev, reg, 0, false, dev->hw_dev->is_unite);
642 reg = stream->config->mi.cb_offs_cnt_init;
643 rkisp_unite_write(dev, reg, 0, false, dev->hw_dev->is_unite);
644 reg = stream->config->mi.cr_offs_cnt_init;
645 if (stream->id != RKISP_STREAM_BP)
646 rkisp_unite_write(dev, reg, 0, false, dev->hw_dev->is_unite);
647 }
648
649 v4l2_dbg(2, rkisp_debug, &dev->v4l2_dev,
650 "%s stream:%d Y:0x%x CB:0x%x | Y_SHD:0x%x\n",
651 __func__, stream->id,
652 rkisp_read(dev, stream->config->mi.y_base_ad_init, false),
653 rkisp_read(dev, stream->config->mi.cb_base_ad_init, false),
654 rkisp_read(dev, stream->config->mi.y_base_ad_shd, true));
655 if (dev->hw_dev->is_unite)
656 v4l2_dbg(2, rkisp_debug, &dev->v4l2_dev,
657 "%s stream:%d Y:0x%x CB:0x%x | Y_SHD:0x%x, right\n",
658 __func__, stream->id,
659 rkisp_next_read(dev, stream->config->mi.y_base_ad_init, false),
660 rkisp_next_read(dev, stream->config->mi.cb_base_ad_init, false),
661 rkisp_next_read(dev, stream->config->mi.y_base_ad_shd, true));
662 }
663
664 static struct streams_ops rkisp_mp_streams_ops = {
665 .config_mi = mp_config_mi,
666 .enable_mi = mp_enable_mi,
667 .disable_mi = mp_disable_mi,
668 .set_data_path = stream_data_path,
669 .is_stream_stopped = mp_is_stream_stopped,
670 .update_mi = update_mi,
671 .frame_end = mi_frame_end,
672 };
673
674 static struct streams_ops rkisp_sp_streams_ops = {
675 .config_mi = sp_config_mi,
676 .enable_mi = sp_enable_mi,
677 .disable_mi = sp_disable_mi,
678 .set_data_path = stream_data_path,
679 .is_stream_stopped = sp_is_stream_stopped,
680 .update_mi = update_mi,
681 .frame_end = mi_frame_end,
682 };
683
684 static struct streams_ops rkisp_fbc_streams_ops = {
685 .config_mi = fbc_config_mi,
686 .enable_mi = fbc_enable_mi,
687 .disable_mi = fbc_disable_mi,
688 .is_stream_stopped = is_fbc_stream_stopped,
689 .update_mi = update_mi,
690 .frame_end = mi_frame_end,
691 };
692
693 static struct streams_ops rkisp_bp_streams_ops = {
694 .config_mi = bp_config_mi,
695 .enable_mi = bp_enable_mi,
696 .disable_mi = bp_disable_mi,
697 .is_stream_stopped = is_bp_stream_stopped,
698 .update_mi = update_mi,
699 .frame_end = mi_frame_end,
700 };
701
702 /*
703 * This function is called when a frame end come. The next frame
704 * is processing and we should set up buffer for next-next frame,
705 * otherwise it will overflow.
706 */
mi_frame_end(struct rkisp_stream * stream)707 static int mi_frame_end(struct rkisp_stream *stream)
708 {
709 struct rkisp_device *dev = stream->ispdev;
710 struct capture_fmt *isp_fmt = &stream->out_isp_fmt;
711 unsigned long lock_flags = 0;
712 int i = 0;
713
714 if (stream->id == RKISP_STREAM_VIR)
715 return 0;
716
717 if (stream->curr_buf) {
718 struct rkisp_stream *vir = &dev->cap_dev.stream[RKISP_STREAM_VIR];
719 struct vb2_buffer *vb2_buf = &stream->curr_buf->vb.vb2_buf;
720 u64 ns = 0;
721
722 /* Dequeue a filled buffer */
723 for (i = 0; i < isp_fmt->mplanes; i++) {
724 u32 payload_size = stream->out_fmt.plane_fmt[i].sizeimage;
725
726 vb2_set_plane_payload(vb2_buf, i, payload_size);
727 }
728
729 rkisp_dmarx_get_frame(dev, &i, NULL, &ns, true);
730 stream->curr_buf->vb.sequence = i;
731 if (!ns)
732 ns = ktime_get_ns();
733 vb2_buf->timestamp = ns;
734
735 ns = ktime_get_ns();
736 stream->dbg.interval = ns - stream->dbg.timestamp;
737 stream->dbg.timestamp = ns;
738 stream->dbg.id = stream->curr_buf->vb.sequence;
739 stream->dbg.delay = ns - dev->isp_sdev.frm_timestamp;
740
741 if (vir->streaming && vir->conn_id == stream->id) {
742
743 spin_lock_irqsave(&vir->vbq_lock, lock_flags);
744 if (vir->streaming)
745 list_add_tail(&stream->curr_buf->queue,
746 &dev->cap_dev.vir_cpy.queue);
747 spin_unlock_irqrestore(&vir->vbq_lock, lock_flags);
748 if (!completion_done(&dev->cap_dev.vir_cpy.cmpl))
749 complete(&dev->cap_dev.vir_cpy.cmpl);
750
751 if (!vir->streaming)
752 vb2_buffer_done(vb2_buf, VB2_BUF_STATE_DONE);
753
754 } else {
755 vb2_buffer_done(vb2_buf, VB2_BUF_STATE_DONE);
756 }
757
758 stream->curr_buf = NULL;
759 }
760
761 stream->curr_buf = stream->next_buf;
762 stream->next_buf = NULL;
763 spin_lock_irqsave(&stream->vbq_lock, lock_flags);
764 if (!list_empty(&stream->buf_queue)) {
765 stream->next_buf = list_first_entry(&stream->buf_queue,
766 struct rkisp_buffer, queue);
767 list_del(&stream->next_buf->queue);
768 }
769 spin_unlock_irqrestore(&stream->vbq_lock, lock_flags);
770
771 stream->ops->update_mi(stream);
772
773 return 0;
774 }
775
776 /***************************** vb2 operations*******************************/
777
778 /*
779 * Set flags and wait, it should stop in interrupt.
780 * If it didn't, stop it by force.
781 */
rkisp_stream_stop(struct rkisp_stream * stream)782 static void rkisp_stream_stop(struct rkisp_stream *stream)
783 {
784 struct rkisp_device *dev = stream->ispdev;
785 struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
786 int ret = 0;
787
788 stream->stopping = true;
789 if (dev->hw_dev->is_single)
790 stream->ops->disable_mi(stream);
791 if (dev->isp_state & ISP_START &&
792 !stream->ops->is_stream_stopped(dev->base_addr)) {
793 ret = wait_event_timeout(stream->done,
794 !stream->streaming,
795 msecs_to_jiffies(500));
796 if (!ret)
797 v4l2_warn(v4l2_dev, "%s id:%d timeout\n",
798 __func__, stream->id);
799 }
800
801 stream->stopping = false;
802 stream->streaming = false;
803 stream->ops->disable_mi(stream);
804 rkisp_disable_dcrop(stream, true);
805 if (stream->id == RKISP_STREAM_MP || stream->id == RKISP_STREAM_SP)
806 rkisp_disable_rsz(stream, true);
807 ret = get_stream_irq_mask(stream);
808 dev->irq_ends_mask &= ~ret;
809
810 stream->burst =
811 CIF_MI_CTRL_BURST_LEN_LUM_16 |
812 CIF_MI_CTRL_BURST_LEN_CHROM_16;
813 stream->interlaced = false;
814 }
815
816 /*
817 * Most of registers inside rockchip isp1 have shadow register since
818 * they must be not changed during processing a frame.
819 * Usually, each sub-module updates its shadow register after
820 * processing the last pixel of a frame.
821 */
rkisp_start(struct rkisp_stream * stream)822 static int rkisp_start(struct rkisp_stream *stream)
823 {
824 struct rkisp_device *dev = stream->ispdev;
825 bool is_update = atomic_read(&dev->cap_dev.refcnt) > 1 ? false : true;
826 int ret;
827
828 if (stream->ops->set_data_path)
829 stream->ops->set_data_path(stream);
830 ret = stream->ops->config_mi(stream);
831 if (ret)
832 return ret;
833
834 stream->ops->enable_mi(stream);
835 if (is_update)
836 dev->irq_ends_mask |= get_stream_irq_mask(stream);
837 stream->streaming = true;
838
839 return 0;
840 }
841
rkisp_queue_setup(struct vb2_queue * queue,unsigned int * num_buffers,unsigned int * num_planes,unsigned int sizes[],struct device * alloc_ctxs[])842 static int rkisp_queue_setup(struct vb2_queue *queue,
843 unsigned int *num_buffers,
844 unsigned int *num_planes,
845 unsigned int sizes[],
846 struct device *alloc_ctxs[])
847 {
848 struct rkisp_stream *stream = queue->drv_priv;
849 struct rkisp_device *dev = stream->ispdev;
850 const struct v4l2_pix_format_mplane *pixm = NULL;
851 const struct capture_fmt *isp_fmt = NULL;
852 u32 i;
853
854 pixm = &stream->out_fmt;
855 isp_fmt = &stream->out_isp_fmt;
856 *num_planes = isp_fmt->mplanes;
857
858 for (i = 0; i < isp_fmt->mplanes; i++) {
859 const struct v4l2_plane_pix_format *plane_fmt;
860
861 plane_fmt = &pixm->plane_fmt[i];
862 /* height to align with 16 when allocating memory
863 * so that Rockchip encoder can use DMA buffer directly
864 */
865 sizes[i] = (isp_fmt->fmt_type == FMT_YUV) ?
866 plane_fmt->sizeimage / pixm->height *
867 ALIGN(pixm->height, 16) :
868 plane_fmt->sizeimage;
869 }
870
871 rkisp_chk_tb_over(dev);
872 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev, "%s %s count %d, size %d\n",
873 stream->vnode.vdev.name, v4l2_type_names[queue->type], *num_buffers, sizes[0]);
874
875 return 0;
876 }
877
878 /*
879 * The vb2_buffer are stored in rkisp_buffer, in order to unify
880 * mplane buffer and none-mplane buffer.
881 */
rkisp_buf_queue(struct vb2_buffer * vb)882 static void rkisp_buf_queue(struct vb2_buffer *vb)
883 {
884 struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
885 struct rkisp_buffer *ispbuf = to_rkisp_buffer(vbuf);
886 struct vb2_queue *queue = vb->vb2_queue;
887 struct rkisp_stream *stream = queue->drv_priv;
888 unsigned long lock_flags = 0;
889 struct v4l2_pix_format_mplane *pixm = &stream->out_fmt;
890 struct capture_fmt *isp_fmt = &stream->out_isp_fmt;
891 struct sg_table *sgt;
892 u32 height, size, offset;
893 int i;
894
895 memset(ispbuf->buff_addr, 0, sizeof(ispbuf->buff_addr));
896 for (i = 0; i < isp_fmt->mplanes; i++) {
897 vb2_plane_vaddr(vb, i);
898 if (stream->ispdev->hw_dev->is_mmu) {
899 sgt = vb2_dma_sg_plane_desc(vb, i);
900 ispbuf->buff_addr[i] = sg_dma_address(sgt->sgl);
901 } else {
902 ispbuf->buff_addr[i] = vb2_dma_contig_plane_dma_addr(vb, i);
903 }
904 }
905 /*
906 * NOTE: plane_fmt[0].sizeimage is total size of all planes for single
907 * memory plane formats, so calculate the size explicitly.
908 */
909 if (isp_fmt->mplanes == 1) {
910 for (i = 0; i < isp_fmt->cplanes - 1; i++) {
911 /* FBC mode calculate payload offset */
912 height = (isp_fmt->fmt_type == FMT_FBC) ?
913 ALIGN(pixm->height, 16) >> 4 : pixm->height;
914 size = (i == 0) ?
915 pixm->plane_fmt[i].bytesperline * height :
916 pixm->plane_fmt[i].sizeimage;
917 offset = (isp_fmt->fmt_type == FMT_FBC) ?
918 ALIGN(size, RK_MPP_ALIGN) : size;
919 ispbuf->buff_addr[i + 1] =
920 ispbuf->buff_addr[i] + offset;
921 }
922 }
923
924 v4l2_dbg(2, rkisp_debug, &stream->ispdev->v4l2_dev,
925 "stream:%d queue buf:0x%x\n",
926 stream->id, ispbuf->buff_addr[0]);
927
928 spin_lock_irqsave(&stream->vbq_lock, lock_flags);
929 list_add_tail(&ispbuf->queue, &stream->buf_queue);
930 spin_unlock_irqrestore(&stream->vbq_lock, lock_flags);
931 }
932
rkisp_create_dummy_buf(struct rkisp_stream * stream)933 static int rkisp_create_dummy_buf(struct rkisp_stream *stream)
934 {
935 return rkisp_alloc_common_dummy_buf(stream->ispdev);
936 }
937
rkisp_destroy_dummy_buf(struct rkisp_stream * stream)938 static void rkisp_destroy_dummy_buf(struct rkisp_stream *stream)
939 {
940 struct rkisp_device *dev = stream->ispdev;
941
942 rkisp_free_common_dummy_buf(dev);
943 }
944
destroy_buf_queue(struct rkisp_stream * stream,enum vb2_buffer_state state)945 static void destroy_buf_queue(struct rkisp_stream *stream,
946 enum vb2_buffer_state state)
947 {
948 unsigned long lock_flags = 0;
949 struct rkisp_buffer *buf;
950
951 spin_lock_irqsave(&stream->vbq_lock, lock_flags);
952 if (stream->curr_buf) {
953 list_add_tail(&stream->curr_buf->queue, &stream->buf_queue);
954 if (stream->curr_buf == stream->next_buf)
955 stream->next_buf = NULL;
956 stream->curr_buf = NULL;
957 }
958 if (stream->next_buf) {
959 list_add_tail(&stream->next_buf->queue, &stream->buf_queue);
960 stream->next_buf = NULL;
961 }
962 while (!list_empty(&stream->buf_queue)) {
963 buf = list_first_entry(&stream->buf_queue,
964 struct rkisp_buffer, queue);
965 list_del(&buf->queue);
966 vb2_buffer_done(&buf->vb.vb2_buf, state);
967 }
968 spin_unlock_irqrestore(&stream->vbq_lock, lock_flags);
969 }
970
rkisp_stop_streaming(struct vb2_queue * queue)971 static void rkisp_stop_streaming(struct vb2_queue *queue)
972 {
973 struct rkisp_stream *stream = queue->drv_priv;
974 struct rkisp_vdev_node *node = &stream->vnode;
975 struct rkisp_device *dev = stream->ispdev;
976 struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
977 int ret;
978
979 mutex_lock(&dev->hw_dev->dev_lock);
980
981 v4l2_dbg(1, rkisp_debug, v4l2_dev, "%s %s %d\n",
982 __func__, node->vdev.name, stream->id);
983
984 if (!stream->streaming)
985 goto end;
986
987 if (stream->id == RKISP_STREAM_VIR) {
988 stream->stopping = true;
989 wait_event_timeout(stream->done,
990 stream->frame_end,
991 msecs_to_jiffies(500));
992 stream->streaming = false;
993 stream->stopping = false;
994 destroy_buf_queue(stream, VB2_BUF_STATE_ERROR);
995
996 if (!completion_done(&dev->cap_dev.vir_cpy.cmpl))
997 complete(&dev->cap_dev.vir_cpy.cmpl);
998 goto end;
999 }
1000
1001 rkisp_stream_stop(stream);
1002 /* call to the other devices */
1003 media_pipeline_stop(&node->vdev.entity);
1004 ret = dev->pipe.set_stream(&dev->pipe, false);
1005 if (ret < 0)
1006 v4l2_err(v4l2_dev, "pipeline stream-off failed:%d\n", ret);
1007
1008 /* release buffers */
1009 destroy_buf_queue(stream, VB2_BUF_STATE_ERROR);
1010
1011 ret = dev->pipe.close(&dev->pipe);
1012 if (ret < 0)
1013 v4l2_err(v4l2_dev, "pipeline close failed error:%d\n", ret);
1014 rkisp_destroy_dummy_buf(stream);
1015 atomic_dec(&dev->cap_dev.refcnt);
1016
1017 end:
1018 mutex_unlock(&dev->hw_dev->dev_lock);
1019 }
1020
vir_cpy_image(struct work_struct * work)1021 static void vir_cpy_image(struct work_struct *work)
1022 {
1023 struct rkisp_vir_cpy *cpy =
1024 container_of(work, struct rkisp_vir_cpy, work);
1025 struct rkisp_stream *vir = cpy->stream;
1026 struct rkisp_buffer *src_buf = NULL;
1027 unsigned long lock_flags = 0;
1028 u32 i;
1029
1030 v4l2_dbg(1, rkisp_debug, &vir->ispdev->v4l2_dev,
1031 "%s enter\n", __func__);
1032
1033 vir->streaming = true;
1034 spin_lock_irqsave(&vir->vbq_lock, lock_flags);
1035 if (!list_empty(&cpy->queue)) {
1036 src_buf = list_first_entry(&cpy->queue,
1037 struct rkisp_buffer, queue);
1038 list_del(&src_buf->queue);
1039 }
1040 spin_unlock_irqrestore(&vir->vbq_lock, lock_flags);
1041
1042 while (src_buf || vir->streaming) {
1043 if (vir->stopping || !vir->streaming)
1044 goto end;
1045
1046 if (!src_buf)
1047 wait_for_completion(&cpy->cmpl);
1048
1049 vir->frame_end = false;
1050 spin_lock_irqsave(&vir->vbq_lock, lock_flags);
1051
1052 if (!src_buf && !list_empty(&cpy->queue)) {
1053 src_buf = list_first_entry(&cpy->queue,
1054 struct rkisp_buffer, queue);
1055 list_del(&src_buf->queue);
1056 }
1057
1058 if (src_buf && !vir->curr_buf && !list_empty(&vir->buf_queue)) {
1059 vir->curr_buf = list_first_entry(&vir->buf_queue,
1060 struct rkisp_buffer, queue);
1061 list_del(&vir->curr_buf->queue);
1062 }
1063 spin_unlock_irqrestore(&vir->vbq_lock, lock_flags);
1064
1065 if (!vir->curr_buf || !src_buf)
1066 goto end;
1067
1068 for (i = 0; i < vir->out_isp_fmt.mplanes; i++) {
1069 u32 payload_size = vir->out_fmt.plane_fmt[i].sizeimage;
1070 void *src = vb2_plane_vaddr(&src_buf->vb.vb2_buf, i);
1071 void *dst = vb2_plane_vaddr(&vir->curr_buf->vb.vb2_buf, i);
1072
1073 if (!src || !dst)
1074 break;
1075 vb2_set_plane_payload(&vir->curr_buf->vb.vb2_buf, i, payload_size);
1076 memcpy(dst, src, payload_size);
1077 }
1078
1079 vir->curr_buf->vb.sequence = src_buf->vb.sequence;
1080 vir->curr_buf->vb.vb2_buf.timestamp = src_buf->vb.vb2_buf.timestamp;
1081 vb2_buffer_done(&vir->curr_buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
1082 vir->curr_buf = NULL;
1083 end:
1084 if (src_buf)
1085 vb2_buffer_done(&src_buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
1086 src_buf = NULL;
1087 spin_lock_irqsave(&vir->vbq_lock, lock_flags);
1088
1089 if (!list_empty(&cpy->queue)) {
1090 src_buf = list_first_entry(&cpy->queue,
1091 struct rkisp_buffer, queue);
1092 list_del(&src_buf->queue);
1093 } else if (vir->stopping) {
1094 vir->streaming = false;
1095 }
1096
1097 spin_unlock_irqrestore(&vir->vbq_lock, lock_flags);
1098 }
1099
1100 vir->frame_end = true;
1101
1102 if (vir->stopping) {
1103 vir->stopping = false;
1104 vir->streaming = false;
1105 wake_up(&vir->done);
1106 }
1107
1108 v4l2_dbg(1, rkisp_debug, &vir->ispdev->v4l2_dev,
1109 "%s exit\n", __func__);
1110 }
1111
rkisp_stream_start(struct rkisp_stream * stream)1112 static int rkisp_stream_start(struct rkisp_stream *stream)
1113 {
1114 struct v4l2_device *v4l2_dev = &stream->ispdev->v4l2_dev;
1115 struct rkisp_device *dev = stream->ispdev;
1116 bool async = false;
1117 int ret;
1118
1119 async = (stream->id == RKISP_STREAM_MP) ?
1120 dev->cap_dev.stream[RKISP_STREAM_SP].streaming :
1121 dev->cap_dev.stream[RKISP_STREAM_MP].streaming;
1122
1123 /*
1124 * can't be async now, otherwise the latter started stream fails to
1125 * produce mi interrupt.
1126 */
1127 ret = rkisp_stream_config_dcrop(stream, false);
1128 if (ret < 0) {
1129 v4l2_err(v4l2_dev, "config dcrop failed with error %d\n", ret);
1130 return ret;
1131 }
1132
1133 if (stream->id == RKISP_STREAM_FBC || stream->id == RKISP_STREAM_BP)
1134 goto end;
1135
1136 ret = rkisp_stream_config_rsz(stream, async);
1137 if (ret < 0) {
1138 v4l2_err(v4l2_dev, "config rsz failed with error %d\n", ret);
1139 return ret;
1140 }
1141 end:
1142 return rkisp_start(stream);
1143 }
1144
1145 static int
rkisp_start_streaming(struct vb2_queue * queue,unsigned int count)1146 rkisp_start_streaming(struct vb2_queue *queue, unsigned int count)
1147 {
1148 struct rkisp_stream *stream = queue->drv_priv;
1149 struct rkisp_vdev_node *node = &stream->vnode;
1150 struct rkisp_device *dev = stream->ispdev;
1151 struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
1152 int ret = -1;
1153
1154 mutex_lock(&dev->hw_dev->dev_lock);
1155
1156 v4l2_dbg(1, rkisp_debug, v4l2_dev, "%s %s id:%d\n",
1157 __func__, node->vdev.name, stream->id);
1158
1159 if (WARN_ON(stream->streaming)) {
1160 mutex_unlock(&dev->hw_dev->dev_lock);
1161 return -EBUSY;
1162 }
1163
1164 if (stream->id == RKISP_STREAM_VIR) {
1165 struct rkisp_stream *t = &dev->cap_dev.stream[stream->conn_id];
1166
1167 if (t->streaming) {
1168 INIT_WORK(&dev->cap_dev.vir_cpy.work, vir_cpy_image);
1169 init_completion(&dev->cap_dev.vir_cpy.cmpl);
1170 INIT_LIST_HEAD(&dev->cap_dev.vir_cpy.queue);
1171 dev->cap_dev.vir_cpy.stream = stream;
1172 schedule_work(&dev->cap_dev.vir_cpy.work);
1173 ret = 0;
1174 } else {
1175 v4l2_err(&dev->v4l2_dev,
1176 "no stream enable for iqtool\n");
1177 destroy_buf_queue(stream, VB2_BUF_STATE_QUEUED);
1178 ret = -EINVAL;
1179 }
1180
1181 mutex_unlock(&dev->hw_dev->dev_lock);
1182
1183 return ret;
1184 }
1185
1186 memset(&stream->dbg, 0, sizeof(stream->dbg));
1187 atomic_inc(&dev->cap_dev.refcnt);
1188 if (!dev->isp_inp || !stream->linked) {
1189 v4l2_err(v4l2_dev, "check %s link or isp input\n", node->vdev.name);
1190 goto buffer_done;
1191 }
1192
1193 if (atomic_read(&dev->cap_dev.refcnt) == 1 &&
1194 (dev->isp_inp & INP_CIF)) {
1195 /* update sensor info when first streaming */
1196 ret = rkisp_update_sensor_info(dev);
1197 if (ret < 0) {
1198 v4l2_err(v4l2_dev, "update sensor info failed %d\n", ret);
1199 goto buffer_done;
1200 }
1201 }
1202
1203 ret = rkisp_create_dummy_buf(stream);
1204 if (ret < 0)
1205 goto buffer_done;
1206
1207 /* enable clocks/power-domains */
1208 ret = dev->pipe.open(&dev->pipe, &node->vdev.entity, true);
1209 if (ret < 0) {
1210 v4l2_err(v4l2_dev, "open isp pipeline failed %d\n", ret);
1211 goto destroy_dummy_buf;
1212 }
1213
1214 /* configure stream hardware to start */
1215 ret = rkisp_stream_start(stream);
1216 if (ret < 0) {
1217 v4l2_err(v4l2_dev, "start %s failed\n", node->vdev.name);
1218 goto close_pipe;
1219 }
1220
1221 /* start sub-devices */
1222 ret = dev->pipe.set_stream(&dev->pipe, true);
1223 if (ret < 0)
1224 goto stop_stream;
1225
1226 ret = media_pipeline_start(&node->vdev.entity, &dev->pipe.pipe);
1227 if (ret < 0) {
1228 v4l2_err(v4l2_dev, "start pipeline failed %d\n", ret);
1229 goto pipe_stream_off;
1230 }
1231
1232 mutex_unlock(&dev->hw_dev->dev_lock);
1233 return 0;
1234
1235 pipe_stream_off:
1236 dev->pipe.set_stream(&dev->pipe, false);
1237 stop_stream:
1238 rkisp_stream_stop(stream);
1239 close_pipe:
1240 dev->pipe.close(&dev->pipe);
1241 destroy_dummy_buf:
1242 rkisp_destroy_dummy_buf(stream);
1243 buffer_done:
1244 destroy_buf_queue(stream, VB2_BUF_STATE_QUEUED);
1245 atomic_dec(&dev->cap_dev.refcnt);
1246 stream->streaming = false;
1247 mutex_unlock(&dev->hw_dev->dev_lock);
1248 return ret;
1249 }
1250
1251 static struct vb2_ops rkisp_vb2_ops = {
1252 .queue_setup = rkisp_queue_setup,
1253 .buf_queue = rkisp_buf_queue,
1254 .wait_prepare = vb2_ops_wait_prepare,
1255 .wait_finish = vb2_ops_wait_finish,
1256 .stop_streaming = rkisp_stop_streaming,
1257 .start_streaming = rkisp_start_streaming,
1258 };
1259
rkisp_init_vb2_queue(struct vb2_queue * q,struct rkisp_stream * stream,enum v4l2_buf_type buf_type)1260 static int rkisp_init_vb2_queue(struct vb2_queue *q,
1261 struct rkisp_stream *stream,
1262 enum v4l2_buf_type buf_type)
1263 {
1264 q->type = buf_type;
1265 q->io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF;
1266 q->drv_priv = stream;
1267 q->ops = &rkisp_vb2_ops;
1268 q->mem_ops = stream->ispdev->hw_dev->mem_ops;
1269 q->buf_struct_size = sizeof(struct rkisp_buffer);
1270 q->min_buffers_needed = CIF_ISP_REQ_BUFS_MIN;
1271 q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
1272 q->lock = &stream->apilock;
1273 q->dev = stream->ispdev->hw_dev->dev;
1274 q->allow_cache_hints = 1;
1275 q->bidirectional = 1;
1276 if (stream->ispdev->hw_dev->is_dma_contig)
1277 q->dma_attrs = DMA_ATTR_FORCE_CONTIGUOUS;
1278 q->gfp_flags = GFP_DMA32;
1279 return vb2_queue_init(q);
1280 }
1281
rkisp_stream_init(struct rkisp_device * dev,u32 id)1282 static int rkisp_stream_init(struct rkisp_device *dev, u32 id)
1283 {
1284 struct rkisp_capture_device *cap_dev = &dev->cap_dev;
1285 struct rkisp_stream *stream;
1286 struct video_device *vdev;
1287 struct rkisp_vdev_node *node;
1288 int ret = 0;
1289
1290 stream = &cap_dev->stream[id];
1291 stream->id = id;
1292 stream->ispdev = dev;
1293 vdev = &stream->vnode.vdev;
1294
1295 INIT_LIST_HEAD(&stream->buf_queue);
1296 init_waitqueue_head(&stream->done);
1297 spin_lock_init(&stream->vbq_lock);
1298 stream->linked = true;
1299
1300 switch (id) {
1301 case RKISP_STREAM_SP:
1302 strscpy(vdev->name, SP_VDEV_NAME, sizeof(vdev->name));
1303 stream->ops = &rkisp_sp_streams_ops;
1304 stream->config = &rkisp_sp_stream_config;
1305 break;
1306 case RKISP_STREAM_FBC:
1307 strscpy(vdev->name, FBC_VDEV_NAME, sizeof(vdev->name));
1308 stream->ops = &rkisp_fbc_streams_ops;
1309 stream->config = &rkisp_fbc_stream_config;
1310 break;
1311 case RKISP_STREAM_BP:
1312 strscpy(vdev->name, BP_VDEV_NAME, sizeof(vdev->name));
1313 stream->ops = &rkisp_bp_streams_ops;
1314 stream->config = &rkisp_bp_stream_config;
1315 break;
1316 case RKISP_STREAM_VIR:
1317 strscpy(vdev->name, VIR_VDEV_NAME, sizeof(vdev->name));
1318 stream->ops = NULL;
1319 stream->config = &rkisp_mp_stream_config;
1320 break;
1321 default:
1322 strscpy(vdev->name, MP_VDEV_NAME, sizeof(vdev->name));
1323 stream->ops = &rkisp_mp_streams_ops;
1324 stream->config = &rkisp_mp_stream_config;
1325 if (dev->br_dev.linked)
1326 stream->linked = false;
1327 }
1328
1329 node = vdev_to_node(vdev);
1330 rkisp_init_vb2_queue(&node->buf_queue, stream,
1331 V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
1332 ret = rkisp_register_stream_vdev(stream);
1333 if (ret < 0)
1334 return ret;
1335
1336 stream->streaming = false;
1337 stream->interlaced = false;
1338 stream->burst =
1339 CIF_MI_CTRL_BURST_LEN_LUM_16 |
1340 CIF_MI_CTRL_BURST_LEN_CHROM_16;
1341 atomic_set(&stream->sequence, 0);
1342 return 0;
1343 }
1344
rkisp_register_stream_v30(struct rkisp_device * dev)1345 int rkisp_register_stream_v30(struct rkisp_device *dev)
1346 {
1347 struct rkisp_capture_device *cap_dev = &dev->cap_dev;
1348 int ret;
1349
1350 ret = rkisp_stream_init(dev, RKISP_STREAM_MP);
1351 if (ret < 0)
1352 goto err;
1353 ret = rkisp_stream_init(dev, RKISP_STREAM_SP);
1354 if (ret < 0)
1355 goto err_free_mp;
1356 ret = rkisp_stream_init(dev, RKISP_STREAM_FBC);
1357 if (ret < 0)
1358 goto err_free_sp;
1359 ret = rkisp_stream_init(dev, RKISP_STREAM_VIR);
1360 if (ret < 0)
1361 goto err_free_fbc;
1362 #ifdef RKISP_STREAM_BP_EN
1363 ret = rkisp_stream_init(dev, RKISP_STREAM_BP);
1364 if (ret < 0)
1365 goto err_free_vir;
1366 #endif
1367 return 0;
1368 #ifdef RKISP_STREAM_BP_EN
1369 err_free_vir:
1370 rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_VIR]);
1371 #endif
1372 err_free_fbc:
1373 rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_FBC]);
1374 err_free_sp:
1375 rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_SP]);
1376 err_free_mp:
1377 rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_MP]);
1378 err:
1379 return ret;
1380 }
1381
rkisp_unregister_stream_v30(struct rkisp_device * dev)1382 void rkisp_unregister_stream_v30(struct rkisp_device *dev)
1383 {
1384 struct rkisp_capture_device *cap_dev = &dev->cap_dev;
1385 struct rkisp_stream *stream;
1386
1387 stream = &cap_dev->stream[RKISP_STREAM_MP];
1388 rkisp_unregister_stream_vdev(stream);
1389 stream = &cap_dev->stream[RKISP_STREAM_SP];
1390 rkisp_unregister_stream_vdev(stream);
1391 stream = &cap_dev->stream[RKISP_STREAM_FBC];
1392 rkisp_unregister_stream_vdev(stream);
1393 stream = &cap_dev->stream[RKISP_STREAM_VIR];
1394 rkisp_unregister_stream_vdev(stream);
1395 #ifdef RKISP_STREAM_BP_EN
1396 stream = &cap_dev->stream[RKISP_STREAM_BP];
1397 rkisp_unregister_stream_vdev(stream);
1398 #endif
1399 }
1400
1401 /**************** Interrupter Handler ****************/
1402
rkisp_mi_v30_isr(u32 mis_val,struct rkisp_device * dev)1403 void rkisp_mi_v30_isr(u32 mis_val, struct rkisp_device *dev)
1404 {
1405 struct rkisp_stream *stream;
1406 unsigned int i;
1407
1408 if (dev->hw_dev->is_unite) {
1409 u32 val = rkisp_read(dev, ISP3X_MI_RIS, true);
1410
1411 if (val) {
1412 rkisp_write(dev, ISP3X_MI_ICR, val, true);
1413 v4l2_dbg(3, rkisp_debug, &dev->v4l2_dev,
1414 "left mi isr:0x%x\n", val);
1415 }
1416 }
1417 v4l2_dbg(3, rkisp_debug, &dev->v4l2_dev,
1418 "mi isr:0x%x\n", mis_val);
1419
1420 rkisp_bridge_isr(&mis_val, dev);
1421
1422 for (i = 0; i < RKISP_MAX_STREAM; ++i) {
1423 stream = &dev->cap_dev.stream[i];
1424
1425 if (!(mis_val & CIF_MI_FRAME(stream)) ||
1426 stream->id == RKISP_STREAM_VIR)
1427 continue;
1428
1429 mi_frame_end_int_clear(stream);
1430
1431 if (stream->stopping) {
1432 /*
1433 * Make sure stream is actually stopped, whose state
1434 * can be read from the shadow register, before
1435 * wake_up() thread which would immediately free all
1436 * frame buffers. disable_mi() takes effect at the next
1437 * frame end that sync the configurations to shadow
1438 * regs.
1439 */
1440 if (!dev->hw_dev->is_single) {
1441 stream->stopping = false;
1442 stream->streaming = false;
1443 stream->ops->disable_mi(stream);
1444 wake_up(&stream->done);
1445 } else if (stream->ops->is_stream_stopped(dev->base_addr)) {
1446 stream->stopping = false;
1447 stream->streaming = false;
1448 wake_up(&stream->done);
1449 }
1450 } else {
1451 mi_frame_end(stream);
1452 }
1453 }
1454
1455 if (mis_val & ISP3X_MI_MP_FRAME) {
1456 stream = &dev->cap_dev.stream[RKISP_STREAM_MP];
1457 if (!stream->streaming)
1458 dev->irq_ends_mask &= ~ISP_FRAME_MP;
1459 else
1460 dev->irq_ends_mask |= ISP_FRAME_MP;
1461 rkisp_check_idle(dev, ISP_FRAME_MP);
1462 }
1463 if (mis_val & ISP3X_MI_SP_FRAME) {
1464 stream = &dev->cap_dev.stream[RKISP_STREAM_SP];
1465 if (!stream->streaming)
1466 dev->irq_ends_mask &= ~ISP_FRAME_SP;
1467 else
1468 dev->irq_ends_mask |= ISP_FRAME_SP;
1469 rkisp_check_idle(dev, ISP_FRAME_SP);
1470 }
1471 if (mis_val & ISP3X_MI_MPFBC_FRAME) {
1472 stream = &dev->cap_dev.stream[RKISP_STREAM_FBC];
1473 if (!stream->streaming)
1474 dev->irq_ends_mask &= ~ISP_FRAME_MPFBC;
1475 else
1476 dev->irq_ends_mask |= ISP_FRAME_MPFBC;
1477 rkisp_check_idle(dev, ISP_FRAME_MPFBC);
1478 }
1479 if (mis_val & ISP3X_MI_BP_FRAME) {
1480 stream = &dev->cap_dev.stream[RKISP_STREAM_BP];
1481 if (!stream->streaming)
1482 dev->irq_ends_mask &= ~ISP_FRAME_BP;
1483 else
1484 dev->irq_ends_mask |= ISP_FRAME_BP;
1485 rkisp_check_idle(dev, ISP_FRAME_BP);
1486 }
1487 }
1488
rkisp_mipi_v30_isr(unsigned int phy,unsigned int packet,unsigned int overflow,unsigned int state,struct rkisp_device * dev)1489 void rkisp_mipi_v30_isr(unsigned int phy, unsigned int packet,
1490 unsigned int overflow, unsigned int state,
1491 struct rkisp_device *dev)
1492 {
1493 if (state & GENMASK(19, 17))
1494 v4l2_warn(&dev->v4l2_dev, "RD_SIZE_ERR:0x%08x\n", state);
1495 }
1496